From 6c345f9fa2d43ec762f66b745ce0df17265cd823 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Wed, 21 Mar 2018 02:27:10 +0100 Subject: [PATCH 001/143] First stab: p2p/conn --- Gopkg.lock | 74 +++++---- Gopkg.toml | 9 +- p2p/conn/connection.go | 239 +++++++++++++++-------------- p2p/conn/connection_test.go | 169 ++++++++++---------- p2p/conn/secret_connection.go | 104 ++++++++----- p2p/conn/secret_connection_test.go | 112 +++++++++----- p2p/conn/wire.go | 14 ++ wire/wire.go | 60 -------- 8 files changed, 409 insertions(+), 372 deletions(-) create mode 100644 p2p/conn/wire.go delete mode 100644 wire/wire.go diff --git a/Gopkg.lock b/Gopkg.lock index 91e0b41e2..9c9ffd416 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -2,9 +2,10 @@ [[projects]] + branch = "master" name = "github.com/btcsuite/btcd" packages = ["btcec"] - revision = "50de9da05b50eb15658bb350f6ea24368a111ab7" + revision = "2be2f12b358dc57d70b8f501b00be450192efbc3" [[projects]] name = "github.com/davecgh/go-spew" @@ -19,10 +20,10 @@ revision = "95f809107225be108efcf10a3509e4ea6ceef3c4" [[projects]] - branch = "master" name = "github.com/fortytw2/leaktest" packages = ["."] - revision = "3b724c3d7b8729a35bf4e577f71653aec6e53513" + revision = "a5ef70473c97b71626b9abeda80ee92ba2a7de9e" + version = "v1.2.0" [[projects]] name = "github.com/fsnotify/fsnotify" @@ -96,6 +97,7 @@ ".", "hcl/ast", "hcl/parser", + "hcl/printer", "hcl/scanner", "hcl/strconv", "hcl/token", @@ -103,7 +105,7 @@ "json/scanner", "json/token" ] - revision = "23c074d0eceb2b8a5bfdbb271ab780cde70f05a8" + revision = "f40e974e75af4e271d97ce0fc917af5898ae7bda" [[projects]] name = "github.com/inconshreveable/mousetrap" @@ -126,12 +128,14 @@ [[projects]] name = "github.com/magiconair/properties" packages = ["."] - revision = "49d762b9817ba1c2e9d0c69183c2b4a8b8f1d934" + revision = "c3beff4c2358b44d0493c7dda585e7db7ff28ae6" + version = "v1.7.6" [[projects]] + branch = "master" name = "github.com/mitchellh/mapstructure" packages = ["."] - revision = "b4575eea38cca1123ec2dc90c26529b5c5acfcff" + revision = "00c29f56e2386353d58c599509e8dc3801b0d716" [[projects]] name = "github.com/pelletier/go-toml" @@ -169,8 +173,8 @@ [[projects]] name = "github.com/spf13/cast" packages = ["."] - revision = "acbeb36b902d72a7a4c18e8f3241075e7ab763e4" - version = "v1.1.0" + revision = "8965335b8c7107321228e3e3702cab9832751bac" + version = "v1.2.0" [[projects]] name = "github.com/spf13/cobra" @@ -193,8 +197,8 @@ [[projects]] name = "github.com/spf13/viper" packages = ["."] - revision = "25b30aa063fc18e48662b86996252eabdcf2f0c7" - version = "v1.0.0" + revision = "b5e8006cbee93ec955a89ab31e0e3ce3204f3736" + version = "v1.0.2" [[projects]] name = "github.com/stretchr/testify" @@ -206,6 +210,7 @@ version = "v1.2.1" [[projects]] + branch = "master" name = "github.com/syndtr/goleveldb" packages = [ "leveldb", @@ -221,7 +226,7 @@ "leveldb/table", "leveldb/util" ] - revision = "34011bf325bce385408353a30b101fe5e923eb6e" + revision = "169b1b37be738edb2813dab48c97a549bcf99bb5" [[projects]] branch = "develop" @@ -234,7 +239,7 @@ "server", "types" ] - revision = "9e0e00bef42aebf6b402f66bf0f3dc607de8a6f3" + revision = "4e0218467649fecf17ebc5e8161f1c888fc8ff22" [[projects]] branch = "master" @@ -247,10 +252,16 @@ revision = "d8387025d2b9d158cf4efb07e7ebf814bcce2057" [[projects]] + branch = "develop" + name = "github.com/tendermint/go-amino" + packages = ["."] + revision = "b1f32ee20e73716d8bfe695365c0a812b2bd8ef9" + +[[projects]] + branch = "develop" name = "github.com/tendermint/go-crypto" packages = ["."] - revision = "c3e19f3ea26f5c3357e0bcbb799b0761ef923755" - version = "v0.5.0" + revision = "a3800da0a15c8272cbd3c155e024bff28fe9692c" [[projects]] name = "github.com/tendermint/go-wire" @@ -259,10 +270,10 @@ "data" ] revision = "fa721242b042ecd4c6ed1a934ee740db4f74e45c" - source = "github.com/tendermint/go-amino" version = "v0.7.3" [[projects]] + branch = "develop" name = "github.com/tendermint/tmlibs" packages = [ "autofile", @@ -278,10 +289,10 @@ "pubsub/query", "test" ] - revision = "1b9b5652a199ab0be2e781393fb275b66377309d" - version = "v0.7.0" + revision = "4e5c655944c9a636eaed549e6ad8fd8011fb4d42" [[projects]] + branch = "master" name = "golang.org/x/crypto" packages = [ "curve25519", @@ -293,7 +304,7 @@ "ripemd160", "salsa20/salsa" ] - revision = "1875d0a70c90e57f11972aefd42276df65e895b9" + revision = "c3a3ad6d03f7a915c0f7e194b7152974bb73d287" [[projects]] branch = "master" @@ -307,12 +318,13 @@ "lex/httplex", "trace" ] - revision = "cbe0f9307d0156177f9dd5dc85da1a31abc5f2fb" + revision = "6078986fec03a1dcc236c34816c71b0e05018fda" [[projects]] + branch = "master" name = "golang.org/x/sys" packages = ["unix"] - revision = "37707fdb30a5b38865cfb95e5aab41707daec7fd" + revision = "7ceb54c8418b8f9cdf0177b511d5cbb06e9fae39" [[projects]] name = "golang.org/x/text" @@ -332,21 +344,27 @@ "unicode/norm", "unicode/rangetable" ] - revision = "e19ae1496984b1c655b8044a65c0300a3c878dd3" + revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" + version = "v0.3.0" [[projects]] + branch = "master" name = "google.golang.org/genproto" packages = ["googleapis/rpc/status"] - revision = "4eb30f4778eed4c258ba66527a0d4f9ec8a36c45" + revision = "f8c8703595236ae70fdf8789ecb656ea0bcdcf46" [[projects]] name = "google.golang.org/grpc" packages = [ ".", "balancer", + "balancer/base", + "balancer/roundrobin", "codes", "connectivity", "credentials", + "encoding", + "encoding/proto", "grpclb/grpc_lb_v1/messages", "grpclog", "internal", @@ -355,23 +373,25 @@ "naming", "peer", "resolver", + "resolver/dns", + "resolver/passthrough", "stats", "status", "tap", "transport" ] - revision = "401e0e00e4bb830a10496d64cd95e068c5bf50de" - version = "v1.7.3" + revision = "8e4536a86ab602859c20df5ebfd0bd4228d08655" + version = "v1.10.0" [[projects]] name = "gopkg.in/yaml.v2" packages = ["."] - revision = "d670f9405373e636a5a2765eea47fac0c9bc91a4" - version = "v2.0.0" + revision = "7f97868eec74b32b0982dd158a51a446d1da7eb5" + version = "v2.1.1" [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "ed9db0be72a900f4812675f683db20eff9d64ef4511dc00ad29a810da65909c2" + inputs-digest = "6da81f319b092e227b5d2c9de3b10296e9bb7287c02adb38fe547147e9e5e447" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 61406ad66..ce1698145 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -75,16 +75,15 @@ [[constraint]] name = "github.com/tendermint/go-crypto" - version = "0.5.0" + branch = "develop" [[constraint]] - name = "github.com/tendermint/go-wire" - source = "github.com/tendermint/go-amino" - version = "0.7.3" + name = "github.com/tendermint/go-amino" + branch = "develop" [[constraint]] name = "github.com/tendermint/tmlibs" - version = "0.7.0" + branch = "develop" [[constraint]] name = "google.golang.org/grpc" diff --git a/p2p/conn/connection.go b/p2p/conn/connection.go index 9acaf6175..1c1a696f7 100644 --- a/p2p/conn/connection.go +++ b/p2p/conn/connection.go @@ -7,18 +7,22 @@ import ( "io" "math" "net" + "reflect" "runtime/debug" "sync/atomic" "time" - wire "github.com/tendermint/go-wire" + amino "github.com/tendermint/go-amino" cmn "github.com/tendermint/tmlibs/common" flow "github.com/tendermint/tmlibs/flowrate" "github.com/tendermint/tmlibs/log" ) const ( - numBatchMsgPackets = 10 + defaultMaxPacketMsgPayloadSize = 1024 + maxPacketMsgOverheadSize = 10 // It's actually lower but good enough + + numBatchPacketMsgs = 10 minReadBufferSize = 1024 minWriteBufferSize = 65536 updateStats = 2 * time.Second @@ -58,8 +62,7 @@ There are two methods for sending messages: `Send(chID, msg)` is a blocking call that waits until `msg` is successfully queued for the channel with the given id byte `chID`, or until the request times out. -The message `msg` is serialized using the `tendermint/wire` submodule's -`WriteBinary()` reflection routine. +The message `msg` is serialized using Go-Amino. `TrySend(chID, msg)` is a nonblocking call that returns false if the channel's queue is full. @@ -69,19 +72,19 @@ Inbound message bytes are handled with an onReceive callback function. type MConnection struct { cmn.BaseService - conn net.Conn - bufReader *bufio.Reader - bufWriter *bufio.Writer - sendMonitor *flow.Monitor - recvMonitor *flow.Monitor - send chan struct{} - pong chan struct{} - channels []*Channel - channelsIdx map[byte]*Channel - onReceive receiveCbFunc - onError errorCbFunc - errored uint32 - config *MConnConfig + conn net.Conn + bufConnReader *bufio.Reader + bufConnWriter *bufio.Writer + sendMonitor *flow.Monitor + recvMonitor *flow.Monitor + send chan struct{} + pong chan struct{} + channels []*Channel + channelsIdx map[byte]*Channel + onReceive receiveCbFunc + onError errorCbFunc + errored uint32 + config *MConnConfig quit chan struct{} flushTimer *cmn.ThrottleTimer // flush writes as necessary but throttled. @@ -102,7 +105,7 @@ type MConnConfig struct { RecvRate int64 `mapstructure:"recv_rate"` // Maximum payload size - MaxMsgPacketPayloadSize int `mapstructure:"max_msg_packet_payload_size"` + MaxPacketMsgPayloadSize int `mapstructure:"max_msg_packet_payload_size"` // Interval to flush writes (throttled) FlushThrottle time.Duration `mapstructure:"flush_throttle"` @@ -114,8 +117,8 @@ type MConnConfig struct { PongTimeout time.Duration `mapstructure:"pong_timeout"` } -func (cfg *MConnConfig) maxMsgPacketTotalSize() int { - return cfg.MaxMsgPacketPayloadSize + maxMsgPacketOverheadSize +func (cfg *MConnConfig) maxPacketMsgTotalSize() int { + return cfg.MaxPacketMsgPayloadSize + maxPacketMsgOverheadSize } // DefaultMConnConfig returns the default config. @@ -123,7 +126,7 @@ func DefaultMConnConfig() *MConnConfig { return &MConnConfig{ SendRate: defaultSendRate, RecvRate: defaultRecvRate, - MaxMsgPacketPayloadSize: defaultMaxMsgPacketPayloadSize, + MaxPacketMsgPayloadSize: defaultMaxPacketMsgPayloadSize, FlushThrottle: defaultFlushThrottle, PingInterval: defaultPingInterval, PongTimeout: defaultPongTimeout, @@ -147,16 +150,16 @@ func NewMConnectionWithConfig(conn net.Conn, chDescs []*ChannelDescriptor, onRec } mconn := &MConnection{ - conn: conn, - bufReader: bufio.NewReaderSize(conn, minReadBufferSize), - bufWriter: bufio.NewWriterSize(conn, minWriteBufferSize), - sendMonitor: flow.New(0, 0), - recvMonitor: flow.New(0, 0), - send: make(chan struct{}, 1), - pong: make(chan struct{}, 1), - onReceive: onReceive, - onError: onError, - config: config, + conn: conn, + bufConnReader: bufio.NewReaderSize(conn, minReadBufferSize), + bufConnWriter: bufio.NewWriterSize(conn, minWriteBufferSize), + sendMonitor: flow.New(0, 0), + recvMonitor: flow.New(0, 0), + send: make(chan struct{}, 1), + pong: make(chan struct{}, 1), + onReceive: onReceive, + onError: onError, + config: config, } // Create channels @@ -221,7 +224,7 @@ func (c *MConnection) String() string { func (c *MConnection) flush() { c.Logger.Debug("Flush", "conn", c) - err := c.bufWriter.Flush() + err := c.bufConnWriter.Flush() if err != nil { c.Logger.Error("MConnection flush failed", "err", err) } @@ -251,7 +254,7 @@ func (c *MConnection) Send(chID byte, msg interface{}) bool { return false } - c.Logger.Debug("Send", "channel", chID, "conn", c, "msg", msg) //, "bytes", wire.BinaryBytes(msg)) + c.Logger.Debug("Send", "channel", chID, "conn", c, "msg", msg) // Send message to channel. channel, ok := c.channelsIdx[chID] @@ -260,7 +263,7 @@ func (c *MConnection) Send(chID byte, msg interface{}) bool { return false } - success := channel.sendBytes(wire.BinaryBytes(msg)) + success := channel.sendBytes(cdc.MustMarshalBinary(msg)) if success { // Wake up sendRoutine if necessary select { @@ -289,7 +292,7 @@ func (c *MConnection) TrySend(chID byte, msg interface{}) bool { return false } - ok = channel.trySendBytes(wire.BinaryBytes(msg)) + ok = channel.trySendBytes(cdc.MustMarshalBinary(msg)) if ok { // Wake up sendRoutine if necessary select { @@ -322,12 +325,13 @@ func (c *MConnection) sendRoutine() { FOR_LOOP: for { - var n int + var _n int64 var err error + SELECTION: select { case <-c.flushTimer.Ch: // NOTE: flushTimer.Set() must be called every time - // something is written to .bufWriter. + // something is written to .bufConnWriter. c.flush() case <-c.chStatsTimer.Chan(): for _, channel := range c.channels { @@ -335,8 +339,11 @@ FOR_LOOP: } case <-c.pingTimer.Chan(): c.Logger.Debug("Send Ping") - wire.WriteByte(packetTypePing, c.bufWriter, &n, &err) - c.sendMonitor.Update(int(n)) + _n, err = cdc.MarshalBinaryWriter(c.bufConnWriter, PacketPing{}) + if err != nil { + break SELECTION + } + c.sendMonitor.Update(int(_n)) c.Logger.Debug("Starting pong timer", "dur", c.config.PongTimeout) c.pongTimer = time.AfterFunc(c.config.PongTimeout, func() { select { @@ -354,14 +361,17 @@ FOR_LOOP: } case <-c.pong: c.Logger.Debug("Send Pong") - wire.WriteByte(packetTypePong, c.bufWriter, &n, &err) - c.sendMonitor.Update(int(n)) + _n, err = cdc.MarshalBinaryWriter(c.bufConnWriter, PacketPong{}) + if err != nil { + break SELECTION + } + c.sendMonitor.Update(int(_n)) c.flush() case <-c.quit: break FOR_LOOP case <-c.send: - // Send some msgPackets - eof := c.sendSomeMsgPackets() + // Send some PacketMsgs + eof := c.sendSomePacketMsgs() if !eof { // Keep sendRoutine awake. select { @@ -387,15 +397,15 @@ FOR_LOOP: // Returns true if messages from channels were exhausted. // Blocks in accordance to .sendMonitor throttling. -func (c *MConnection) sendSomeMsgPackets() bool { +func (c *MConnection) sendSomePacketMsgs() bool { // Block until .sendMonitor says we can write. // Once we're ready we send more than we asked for, // but amortized it should even out. - c.sendMonitor.Limit(c.config.maxMsgPacketTotalSize(), atomic.LoadInt64(&c.config.SendRate), true) + c.sendMonitor.Limit(c.config.maxPacketMsgTotalSize(), atomic.LoadInt64(&c.config.SendRate), true) - // Now send some msgPackets. - for i := 0; i < numBatchMsgPackets; i++ { - if c.sendMsgPacket() { + // Now send some PacketMsgs. + for i := 0; i < numBatchPacketMsgs; i++ { + if c.sendPacketMsg() { return true } } @@ -403,8 +413,8 @@ func (c *MConnection) sendSomeMsgPackets() bool { } // Returns true if messages from channels were exhausted. -func (c *MConnection) sendMsgPacket() bool { - // Choose a channel to create a msgPacket from. +func (c *MConnection) sendPacketMsg() bool { + // Choose a channel to create a PacketMsg from. // The chosen channel will be the one whose recentlySent/priority is the least. var leastRatio float32 = math.MaxFloat32 var leastChannel *Channel @@ -425,22 +435,22 @@ func (c *MConnection) sendMsgPacket() bool { if leastChannel == nil { return true } else { - // c.Logger.Info("Found a msgPacket to send") + // c.Logger.Info("Found a PacketMsg to send") } - // Make & send a msgPacket from this channel - n, err := leastChannel.writeMsgPacketTo(c.bufWriter) + // Make & send a PacketMsg from this channel + _n, err := leastChannel.writePacketMsgTo(c.bufConnWriter) if err != nil { - c.Logger.Error("Failed to write msgPacket", "err", err) + c.Logger.Error("Failed to write PacketMsg", "err", err) c.stopForError(err) return true } - c.sendMonitor.Update(int(n)) + c.sendMonitor.Update(int(_n)) c.flushTimer.Set() return false } -// recvRoutine reads msgPackets and reconstructs the message using the channels' "recving" buffer. +// recvRoutine reads PacketMsgs and reconstructs the message using the channels' "recving" buffer. // After a whole message has been assembled, it's pushed to onReceive(). // Blocks depending on how the connection is throttled. // Otherwise, it never blocks. @@ -450,13 +460,13 @@ func (c *MConnection) recvRoutine() { FOR_LOOP: for { // Block until .recvMonitor says we can read. - c.recvMonitor.Limit(c.config.maxMsgPacketTotalSize(), atomic.LoadInt64(&c.config.RecvRate), true) + c.recvMonitor.Limit(c.config.maxPacketMsgTotalSize(), atomic.LoadInt64(&c.config.RecvRate), true) /* - // Peek into bufReader for debugging - if numBytes := c.bufReader.Buffered(); numBytes > 0 { + // Peek into bufConnReader for debugging + if numBytes := c.bufConnReader.Buffered(); numBytes > 0 { log.Info("Peek connection buffer", "numBytes", numBytes, "bytes", log15.Lazy{func() []byte { - bytes, err := c.bufReader.Peek(cmn.MinInt(numBytes, 100)) + bytes, err := c.bufConnReader.Peek(cmn.MinInt(numBytes, 100)) if err == nil { return bytes } else { @@ -468,10 +478,11 @@ FOR_LOOP: */ // Read packet type - var n int + var packet Packet + var _n int64 var err error - pktType := wire.ReadByte(c.bufReader, &n, &err) - c.recvMonitor.Update(int(n)) + _n, err = cdc.UnmarshalBinaryReader(c.bufConnReader, &packet, int64(c.config.maxPacketMsgTotalSize())) + c.recvMonitor.Update(int(_n)) if err != nil { if c.IsRunning() { c.Logger.Error("Connection failed @ recvRoutine (reading byte)", "conn", c, "err", err) @@ -481,8 +492,8 @@ FOR_LOOP: } // Read more depending on packet type. - switch pktType { - case packetTypePing: + switch pkt := packet.(type) { + case PacketPing: // TODO: prevent abuse, as they cause flush()'s. // https://github.com/tendermint/tendermint/issues/1190 c.Logger.Debug("Receive Ping") @@ -491,24 +502,14 @@ FOR_LOOP: default: // never block } - case packetTypePong: + case PacketPong: c.Logger.Debug("Receive Pong") select { case c.pongTimeoutCh <- false: default: // never block } - case packetTypeMsg: - pkt, n, err := msgPacket{}, int(0), error(nil) - wire.ReadBinaryPtr(&pkt, c.bufReader, c.config.maxMsgPacketTotalSize(), &n, &err) - c.recvMonitor.Update(int(n)) - if err != nil { - if c.IsRunning() { - c.Logger.Error("Connection failed @ recvRoutine", "conn", c, "err", err) - c.stopForError(err) - } - break FOR_LOOP - } + case PacketMsg: channel, ok := c.channelsIdx[pkt.ChannelID] if !ok || channel == nil { err := fmt.Errorf("Unknown channel %X", pkt.ChannelID) @@ -517,7 +518,7 @@ FOR_LOOP: break FOR_LOOP } - msgBytes, err := channel.recvMsgPacket(pkt) + msgBytes, err := channel.recvPacketMsg(pkt) if err != nil { if c.IsRunning() { c.Logger.Error("Connection failed @ recvRoutine", "conn", c, "err", err) @@ -531,7 +532,7 @@ FOR_LOOP: c.onReceive(pkt.ChannelID, msgBytes) } default: - err := fmt.Errorf("Unknown message type %X", pktType) + err := fmt.Errorf("Unknown message type %v", reflect.TypeOf(packet)) c.Logger.Error("Connection failed @ recvRoutine", "conn", c, "err", err) c.stopForError(err) break FOR_LOOP @@ -623,7 +624,7 @@ type Channel struct { sending []byte recentlySent int64 // exponential moving average - maxMsgPacketPayloadSize int + maxPacketMsgPayloadSize int Logger log.Logger } @@ -638,7 +639,7 @@ func newChannel(conn *MConnection, desc ChannelDescriptor) *Channel { desc: desc, sendQueue: make(chan []byte, desc.SendQueueCapacity), recving: make([]byte, 0, desc.RecvBufferCapacity), - maxMsgPacketPayloadSize: conn.config.MaxMsgPacketPayloadSize, + maxPacketMsgPayloadSize: conn.config.MaxPacketMsgPayloadSize, } } @@ -683,8 +684,8 @@ func (ch *Channel) canSend() bool { return ch.loadSendQueueSize() < defaultSendQueueCapacity } -// Returns true if any msgPackets are pending to be sent. -// Call before calling nextMsgPacket() +// Returns true if any PacketMsgs are pending to be sent. +// Call before calling nextPacketMsg() // Goroutine-safe func (ch *Channel) isSendPending() bool { if len(ch.sending) == 0 { @@ -696,12 +697,12 @@ func (ch *Channel) isSendPending() bool { return true } -// Creates a new msgPacket to send. +// Creates a new PacketMsg to send. // Not goroutine-safe -func (ch *Channel) nextMsgPacket() msgPacket { - packet := msgPacket{} +func (ch *Channel) nextPacketMsg() PacketMsg { + packet := PacketMsg{} packet.ChannelID = byte(ch.desc.ID) - maxSize := ch.maxMsgPacketPayloadSize + maxSize := ch.maxPacketMsgPayloadSize packet.Bytes = ch.sending[:cmn.MinInt(maxSize, len(ch.sending))] if len(ch.sending) <= maxSize { packet.EOF = byte(0x01) @@ -714,30 +715,24 @@ func (ch *Channel) nextMsgPacket() msgPacket { return packet } -// Writes next msgPacket to w. +// Writes next PacketMsg to w and updates c.recentlySent. // Not goroutine-safe -func (ch *Channel) writeMsgPacketTo(w io.Writer) (n int, err error) { - packet := ch.nextMsgPacket() +func (ch *Channel) writePacketMsgTo(w io.Writer) (n int64, err error) { + var packet = ch.nextPacketMsg() ch.Logger.Debug("Write Msg Packet", "conn", ch.conn, "packet", packet) - writeMsgPacketTo(packet, w, &n, &err) - if err == nil { - ch.recentlySent += int64(n) - } + n, err = cdc.MarshalBinaryWriter(w, packet) + ch.recentlySent += n return } -func writeMsgPacketTo(packet msgPacket, w io.Writer, n *int, err *error) { - wire.WriteByte(packetTypeMsg, w, n, err) - wire.WriteBinary(packet, w, n, err) -} - -// Handles incoming msgPackets. It returns a message bytes if message is -// complete. NOTE message bytes may change on next call to recvMsgPacket. +// Handles incoming PacketMsgs. It returns a message bytes if message is +// complete. NOTE message bytes may change on next call to recvPacketMsg. // Not goroutine-safe -func (ch *Channel) recvMsgPacket(packet msgPacket) ([]byte, error) { +func (ch *Channel) recvPacketMsg(packet PacketMsg) ([]byte, error) { ch.Logger.Debug("Read Msg Packet", "conn", ch.conn, "packet", packet) - if ch.desc.RecvMessageCapacity < len(ch.recving)+len(packet.Bytes) { - return nil, wire.ErrBinaryReadOverflow + var recvCap, recvReceived = ch.desc.RecvMessageCapacity, len(ch.recving) + len(packet.Bytes) + if recvCap < recvReceived { + return nil, fmt.Errorf("Received message exceeds available capacity: %v < %v", recvCap, recvReceived) } ch.recving = append(ch.recving, packet.Bytes...) if packet.EOF == byte(0x01) { @@ -761,24 +756,36 @@ func (ch *Channel) updateStats() { ch.recentlySent = int64(float64(ch.recentlySent) * 0.8) } -//----------------------------------------------------------------------------- +//---------------------------------------- +// Packet -const ( - defaultMaxMsgPacketPayloadSize = 1024 +type Packet interface { + AssertIsPacket() +} - maxMsgPacketOverheadSize = 10 // It's actually lower but good enough - packetTypePing = byte(0x01) - packetTypePong = byte(0x02) - packetTypeMsg = byte(0x03) -) +func RegisterPacket(cdc *amino.Codec) { + cdc.RegisterInterface((*Packet)(nil), nil) + cdc.RegisterConcrete(PacketPing{}, "tendermint/p2p/PacketPing", nil) + cdc.RegisterConcrete(PacketPong{}, "tendermint/p2p/PacketPong", nil) + cdc.RegisterConcrete(PacketMsg{}, "tendermint/p2p/PacketMsg", nil) +} -// Messages in channels are chopped into smaller msgPackets for multiplexing. -type msgPacket struct { +func (_ PacketPing) AssertIsPacket() {} +func (_ PacketPong) AssertIsPacket() {} +func (_ PacketMsg) AssertIsPacket() {} + +type PacketPing struct { +} + +type PacketPong struct { +} + +type PacketMsg struct { ChannelID byte EOF byte // 1 means message ends here. Bytes []byte } -func (p msgPacket) String() string { - return fmt.Sprintf("MsgPacket{%X:%X T:%X}", p.ChannelID, p.Bytes, p.EOF) +func (mp PacketMsg) String() string { + return fmt.Sprintf("PacketMsg{%X:%X T:%X}", mp.ChannelID, mp.Bytes, mp.EOF) } diff --git a/p2p/conn/connection_test.go b/p2p/conn/connection_test.go index d308ea61a..193b77b26 100644 --- a/p2p/conn/connection_test.go +++ b/p2p/conn/connection_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - wire "github.com/tendermint/go-wire" + "github.com/tendermint/go-amino" "github.com/tendermint/tmlibs/log" ) @@ -32,41 +32,37 @@ func createMConnectionWithCallbacks(conn net.Conn, onReceive func(chID byte, msg } func TestMConnectionSend(t *testing.T) { - assert, require := assert.New(t), require.New(t) - server, client := NetPipe() defer server.Close() // nolint: errcheck defer client.Close() // nolint: errcheck mconn := createTestMConnection(client) err := mconn.Start() - require.Nil(err) + require.Nil(t, err) defer mconn.Stop() msg := "Ant-Man" - assert.True(mconn.Send(0x01, msg)) + assert.True(t, mconn.Send(0x01, msg)) // Note: subsequent Send/TrySend calls could pass because we are reading from // the send queue in a separate goroutine. _, err = server.Read(make([]byte, len(msg))) if err != nil { t.Error(err) } - assert.True(mconn.CanSend(0x01)) + assert.True(t, mconn.CanSend(0x01)) msg = "Spider-Man" - assert.True(mconn.TrySend(0x01, msg)) + assert.True(t, mconn.TrySend(0x01, msg)) _, err = server.Read(make([]byte, len(msg))) if err != nil { t.Error(err) } - assert.False(mconn.CanSend(0x05), "CanSend should return false because channel is unknown") - assert.False(mconn.Send(0x05, "Absorbing Man"), "Send should return false because channel is unknown") + assert.False(t, mconn.CanSend(0x05), "CanSend should return false because channel is unknown") + assert.False(t, mconn.Send(0x05, "Absorbing Man"), "Send should return false because channel is unknown") } func TestMConnectionReceive(t *testing.T) { - assert, require := assert.New(t), require.New(t) - server, client := NetPipe() defer server.Close() // nolint: errcheck defer client.Close() // nolint: errcheck @@ -81,20 +77,20 @@ func TestMConnectionReceive(t *testing.T) { } mconn1 := createMConnectionWithCallbacks(client, onReceive, onError) err := mconn1.Start() - require.Nil(err) + require.Nil(t, err) defer mconn1.Stop() mconn2 := createTestMConnection(server) err = mconn2.Start() - require.Nil(err) + require.Nil(t, err) defer mconn2.Stop() msg := "Cyclops" - assert.True(mconn2.Send(0x01, msg)) + assert.True(t, mconn2.Send(0x01, msg)) select { case receivedBytes := <-receivedCh: - assert.Equal([]byte(msg), receivedBytes[2:]) // first 3 bytes are internal + assert.Equal(t, []byte(msg), receivedBytes[2:]) // first 3 bytes are internal case err := <-errorsCh: t.Fatalf("Expected %s, got %+v", msg, err) case <-time.After(500 * time.Millisecond): @@ -103,20 +99,18 @@ func TestMConnectionReceive(t *testing.T) { } func TestMConnectionStatus(t *testing.T) { - assert, require := assert.New(t), require.New(t) - server, client := NetPipe() defer server.Close() // nolint: errcheck defer client.Close() // nolint: errcheck mconn := createTestMConnection(client) err := mconn.Start() - require.Nil(err) + require.Nil(t, err) defer mconn.Stop() status := mconn.Status() - assert.NotNil(status) - assert.Zero(status.Channels[0].SendQueueSize) + assert.NotNil(t, status) + assert.Zero(t, status.Channels[0].SendQueueSize) } func TestMConnectionPongTimeoutResultsInError(t *testing.T) { @@ -140,7 +134,9 @@ func TestMConnectionPongTimeoutResultsInError(t *testing.T) { serverGotPing := make(chan struct{}) go func() { // read ping - server.Read(make([]byte, 1)) + var pkt PacketPing + _, err = cdc.UnmarshalBinaryReader(server, &pkt, 1024) + assert.Nil(t, err) serverGotPing <- struct{}{} }() <-serverGotPing @@ -175,21 +171,22 @@ func TestMConnectionMultiplePongsInTheBeginning(t *testing.T) { defer mconn.Stop() // sending 3 pongs in a row (abuse) - _, err = server.Write([]byte{packetTypePong}) + _, err = server.Write(cdc.MustMarshalBinary(PacketPong{})) require.Nil(t, err) - _, err = server.Write([]byte{packetTypePong}) + _, err = server.Write(cdc.MustMarshalBinary(PacketPong{})) require.Nil(t, err) - _, err = server.Write([]byte{packetTypePong}) + _, err = server.Write(cdc.MustMarshalBinary(PacketPong{})) require.Nil(t, err) serverGotPing := make(chan struct{}) go func() { // read ping (one byte) - _, err = server.Read(make([]byte, 1)) + var packet, err = Packet(nil), error(nil) + _, err = cdc.UnmarshalBinaryReader(server, &packet, 1024) require.Nil(t, err) serverGotPing <- struct{}{} // respond with pong - _, err = server.Write([]byte{packetTypePong}) + _, err = server.Write(cdc.MustMarshalBinary(PacketPong{})) require.Nil(t, err) }() <-serverGotPing @@ -225,17 +222,18 @@ func TestMConnectionMultiplePings(t *testing.T) { // sending 3 pings in a row (abuse) // see https://github.com/tendermint/tendermint/issues/1190 - _, err = server.Write([]byte{packetTypePing}) + _, err = server.Write(cdc.MustMarshalBinary(PacketPing{})) require.Nil(t, err) - _, err = server.Read(make([]byte, 1)) + var pkt PacketPong + _, err = cdc.UnmarshalBinaryReader(server, &pkt, 1024) require.Nil(t, err) - _, err = server.Write([]byte{packetTypePing}) + _, err = server.Write(cdc.MustMarshalBinary(PacketPing{})) require.Nil(t, err) - _, err = server.Read(make([]byte, 1)) + _, err = cdc.UnmarshalBinaryReader(server, &pkt, 1024) require.Nil(t, err) - _, err = server.Write([]byte{packetTypePing}) + _, err = server.Write(cdc.MustMarshalBinary(PacketPing{})) require.Nil(t, err) - _, err = server.Read(make([]byte, 1)) + _, err = cdc.UnmarshalBinaryReader(server, &pkt, 1024) require.Nil(t, err) assert.True(t, mconn.IsRunning()) @@ -262,18 +260,21 @@ func TestMConnectionPingPongs(t *testing.T) { serverGotPing := make(chan struct{}) go func() { // read ping - server.Read(make([]byte, 1)) + var pkt PacketPing + _, err = cdc.UnmarshalBinaryReader(server, &pkt, 1024) + require.Nil(t, err) serverGotPing <- struct{}{} // respond with pong - _, err = server.Write([]byte{packetTypePong}) + _, err = server.Write(cdc.MustMarshalBinary(PacketPong{})) require.Nil(t, err) time.Sleep(mconn.config.PingInterval) // read ping - server.Read(make([]byte, 1)) + _, err = cdc.UnmarshalBinaryReader(server, &pkt, 1024) + require.Nil(t, err) // respond with pong - _, err = server.Write([]byte{packetTypePong}) + _, err = server.Write(cdc.MustMarshalBinary(PacketPong{})) require.Nil(t, err) }() <-serverGotPing @@ -290,8 +291,6 @@ func TestMConnectionPingPongs(t *testing.T) { } func TestMConnectionStopsAndReturnsError(t *testing.T) { - assert, require := assert.New(t), require.New(t) - server, client := NetPipe() defer server.Close() // nolint: errcheck defer client.Close() // nolint: errcheck @@ -306,7 +305,7 @@ func TestMConnectionStopsAndReturnsError(t *testing.T) { } mconn := createMConnectionWithCallbacks(client, onReceive, onError) err := mconn.Start() - require.Nil(err) + require.Nil(t, err) defer mconn.Stop() if err := client.Close(); err != nil { @@ -317,14 +316,14 @@ func TestMConnectionStopsAndReturnsError(t *testing.T) { case receivedBytes := <-receivedCh: t.Fatalf("Expected error, got %v", receivedBytes) case err := <-errorsCh: - assert.NotNil(err) - assert.False(mconn.IsRunning()) + assert.NotNil(t, err) + assert.False(t, mconn.IsRunning()) case <-time.After(500 * time.Millisecond): t.Fatal("Did not receive error in 500ms") } } -func newClientAndServerConnsForReadErrors(require *require.Assertions, chOnErr chan struct{}) (*MConnection, *MConnection) { +func newClientAndServerConnsForReadErrors(t *testing.T, chOnErr chan struct{}) (*MConnection, *MConnection) { server, client := NetPipe() onReceive := func(chID byte, msgBytes []byte) {} @@ -338,7 +337,7 @@ func newClientAndServerConnsForReadErrors(require *require.Assertions, chOnErr c mconnClient := NewMConnection(client, chDescs, onReceive, onError) mconnClient.SetLogger(log.TestingLogger().With("module", "client")) err := mconnClient.Start() - require.Nil(err) + require.Nil(t, err) // create server conn with 1 channel // it fires on chOnErr when there's an error @@ -349,7 +348,7 @@ func newClientAndServerConnsForReadErrors(require *require.Assertions, chOnErr c mconnServer := createMConnectionWithCallbacks(server, onReceive, onError) mconnServer.SetLogger(serverLogger) err = mconnServer.Start() - require.Nil(err) + require.Nil(t, err) return mconnClient, mconnServer } @@ -364,50 +363,45 @@ func expectSend(ch chan struct{}) bool { } func TestMConnectionReadErrorBadEncoding(t *testing.T) { - assert, require := assert.New(t), require.New(t) - chOnErr := make(chan struct{}) - mconnClient, mconnServer := newClientAndServerConnsForReadErrors(require, chOnErr) + mconnClient, mconnServer := newClientAndServerConnsForReadErrors(t, chOnErr) defer mconnClient.Stop() defer mconnServer.Stop() client := mconnClient.conn - msg := "Ant-Man" // send badly encoded msgPacket - var n int - var err error - wire.WriteByte(packetTypeMsg, client, &n, &err) - wire.WriteByteSlice([]byte(msg), client, &n, &err) - assert.True(expectSend(chOnErr), "badly encoded msgPacket") + bz := cdc.MustMarshalBinary(PacketMsg{}) + bz[4] += 0x01 // Invalid prefix bytes. + + // Write it. + _, err := client.Write(bz) + assert.Nil(t, err) + assert.True(t, expectSend(chOnErr), "badly encoded msgPacket") } func TestMConnectionReadErrorUnknownChannel(t *testing.T) { - assert, require := assert.New(t), require.New(t) - chOnErr := make(chan struct{}) - mconnClient, mconnServer := newClientAndServerConnsForReadErrors(require, chOnErr) + mconnClient, mconnServer := newClientAndServerConnsForReadErrors(t, chOnErr) defer mconnClient.Stop() defer mconnServer.Stop() msg := "Ant-Man" // fail to send msg on channel unknown by client - assert.False(mconnClient.Send(0x03, msg)) + assert.False(t, mconnClient.Send(0x03, msg)) // send msg on channel unknown by the server. // should cause an error - assert.True(mconnClient.Send(0x02, msg)) - assert.True(expectSend(chOnErr), "unknown channel") + assert.True(t, mconnClient.Send(0x02, msg)) + assert.True(t, expectSend(chOnErr), "unknown channel") } func TestMConnectionReadErrorLongMessage(t *testing.T) { - assert, require := assert.New(t), require.New(t) - chOnErr := make(chan struct{}) chOnRcv := make(chan struct{}) - mconnClient, mconnServer := newClientAndServerConnsForReadErrors(require, chOnErr) + mconnClient, mconnServer := newClientAndServerConnsForReadErrors(t, chOnErr) defer mconnClient.Stop() defer mconnServer.Stop() @@ -418,65 +412,64 @@ func TestMConnectionReadErrorLongMessage(t *testing.T) { client := mconnClient.conn // send msg thats just right - var n int var err error - packet := msgPacket{ + var packet = PacketMsg{ ChannelID: 0x01, - Bytes: make([]byte, mconnClient.config.maxMsgPacketTotalSize()-5), + Bytes: make([]byte, mconnClient.config.maxPacketMsgTotalSize()-12), EOF: 1, } - writeMsgPacketTo(packet, client, &n, &err) - assert.True(expectSend(chOnRcv), "msg just right") + _, err = cdc.MarshalBinaryWriter(client, packet) + assert.Nil(t, err) + assert.True(t, expectSend(chOnRcv), "msg just right") // send msg thats too long - packet = msgPacket{ + packet = PacketMsg{ ChannelID: 0x01, - Bytes: make([]byte, mconnClient.config.maxMsgPacketTotalSize()-4), + Bytes: make([]byte, mconnClient.config.maxPacketMsgTotalSize()-11), EOF: 1, } - writeMsgPacketTo(packet, client, &n, &err) - assert.True(expectSend(chOnErr), "msg too long") + _, err = cdc.MarshalBinaryWriter(client, packet) + assert.Nil(t, err) + assert.True(t, expectSend(chOnErr), "msg too long") } func TestMConnectionReadErrorUnknownMsgType(t *testing.T) { - assert, require := assert.New(t), require.New(t) - chOnErr := make(chan struct{}) - mconnClient, mconnServer := newClientAndServerConnsForReadErrors(require, chOnErr) + mconnClient, mconnServer := newClientAndServerConnsForReadErrors(t, chOnErr) defer mconnClient.Stop() defer mconnServer.Stop() // send msg with unknown msg type - var n int - var err error - wire.WriteByte(0x04, mconnClient.conn, &n, &err) - assert.True(expectSend(chOnErr), "unknown msg type") + err := error(nil) + err = amino.EncodeUvarint(mconnClient.conn, 4) + assert.Nil(t, err) + _, err = mconnClient.conn.Write([]byte{0xFF, 0xFF, 0xFF, 0xFF}) + assert.Nil(t, err) + assert.True(t, expectSend(chOnErr), "unknown msg type") } func TestMConnectionTrySend(t *testing.T) { - assert, require := assert.New(t), require.New(t) - server, client := NetPipe() defer server.Close() defer client.Close() mconn := createTestMConnection(client) err := mconn.Start() - require.Nil(err) + require.Nil(t, err) defer mconn.Stop() msg := "Semicolon-Woman" resultCh := make(chan string, 2) - assert.True(mconn.TrySend(0x01, msg)) + assert.True(t, mconn.TrySend(0x01, msg)) server.Read(make([]byte, len(msg))) - assert.True(mconn.CanSend(0x01)) - assert.True(mconn.TrySend(0x01, msg)) - assert.False(mconn.CanSend(0x01)) + assert.True(t, mconn.CanSend(0x01)) + assert.True(t, mconn.TrySend(0x01, msg)) + assert.False(t, mconn.CanSend(0x01)) go func() { mconn.TrySend(0x01, msg) resultCh <- "TrySend" }() - assert.False(mconn.CanSend(0x01)) - assert.False(mconn.TrySend(0x01, msg)) - assert.Equal("TrySend", <-resultCh) + assert.False(t, mconn.CanSend(0x01)) + assert.False(t, mconn.TrySend(0x01, msg)) + assert.Equal(t, "TrySend", <-resultCh) } diff --git a/p2p/conn/secret_connection.go b/p2p/conn/secret_connection.go index aa6db05bb..58d2e366f 100644 --- a/p2p/conn/secret_connection.go +++ b/p2p/conn/secret_connection.go @@ -12,6 +12,7 @@ import ( "crypto/sha256" "encoding/binary" "errors" + "fmt" "io" "net" "time" @@ -21,16 +22,14 @@ import ( "golang.org/x/crypto/ripemd160" "github.com/tendermint/go-crypto" - "github.com/tendermint/go-wire" cmn "github.com/tendermint/tmlibs/common" ) -// 2 + 1024 == 1026 total frame size -const dataLenSize = 2 // uint16 to describe the length, is <= dataMaxSize +// 4 + 1024 == 1028 total frame size +const dataLenSize = 4 const dataMaxSize = 1024 const totalFrameSize = dataMaxSize + dataLenSize const sealedFrameSize = totalFrameSize + secretbox.Overhead -const authSigMsgSize = (32 + 1) + (64 + 1) // fixed size (length prefixed) byte arrays // Implements net.Conn type SecretConnection struct { @@ -92,6 +91,7 @@ func MakeSecretConnection(conn io.ReadWriteCloser, locPrivKey crypto.PrivKey) (* // Share (in secret) each other's pubkey & challenge signature authSigMsg, err := shareAuthSignature(sc, locPubKey, locSignature) if err != nil { + fmt.Println(">>>", err) return nil, err } remPubKey, remSignature := authSigMsg.Key, authSigMsg.Sig @@ -123,7 +123,7 @@ func (sc *SecretConnection) Write(data []byte) (n int, err error) { data = nil } chunkLength := len(chunk) - binary.BigEndian.PutUint16(frame, uint16(chunkLength)) + binary.BigEndian.PutUint32(frame, uint32(chunkLength)) copy(frame[dataLenSize:], chunk) // encrypt the frame @@ -167,7 +167,7 @@ func (sc *SecretConnection) Read(data []byte) (n int, err error) { incr2Nonce(sc.recvNonce) // end decryption - var chunkLength = binary.BigEndian.Uint16(frame) // read the first two bytes + var chunkLength = binary.BigEndian.Uint32(frame) // read the first two bytes if chunkLength > dataMaxSize { return 0, errors.New("chunkLength is greater than dataMaxSize") } @@ -200,26 +200,41 @@ func genEphKeys() (ephPub, ephPriv *[32]byte) { } func shareEphPubKey(conn io.ReadWriteCloser, locEphPub *[32]byte) (remEphPub *[32]byte, err error) { - var err1, err2 error - cmn.Parallel( - func() { - _, err1 = conn.Write(locEphPub[:]) + // Send our pubkey and receive theirs in tandem. + var trs, _ = cmn.Parallel( + func(_ int) (val interface{}, err error, abort bool) { + var _, err1 = cdc.MarshalBinaryWriter(conn, locEphPub) + if err1 != nil { + return nil, err1, true // abort + } else { + return nil, nil, false + } }, - func() { - remEphPub = new([32]byte) - _, err2 = io.ReadFull(conn, remEphPub[:]) + func(_ int) (val interface{}, err error, abort bool) { + var _remEphPub [32]byte + var _, err2 = cdc.UnmarshalBinaryReader(conn, &_remEphPub, 1024*1024) // TODO + if err2 != nil { + return nil, err2, true // abort + } else { + return _remEphPub, nil, false + } }, ) - if err1 != nil { - return nil, err1 - } - if err2 != nil { - return nil, err2 + // If error: + if trs.FirstError() != nil { + err = trs.FirstError() + return + } else if trs.FirstPanic() != nil { + err = fmt.Errorf("Panic: %v", trs.FirstPanic()) + return } - return remEphPub, nil + // Otherwise: + var _remEphPub = trs.FirstValue().([32]byte) + return &_remEphPub, nil + } func computeSharedSecret(remPubKey, locPrivKey *[32]byte) (shrSecret *[32]byte) { @@ -268,33 +283,40 @@ type authSigMessage struct { Sig crypto.Signature } -func shareAuthSignature(sc *SecretConnection, pubKey crypto.PubKey, signature crypto.Signature) (*authSigMessage, error) { - var recvMsg authSigMessage - var err1, err2 error +func shareAuthSignature(sc *SecretConnection, pubKey crypto.PubKey, signature crypto.Signature) (recvMsg authSigMessage, err error) { - cmn.Parallel( - func() { - msgBytes := wire.BinaryBytes(authSigMessage{pubKey.Wrap(), signature.Wrap()}) - _, err1 = sc.Write(msgBytes) - }, - func() { - readBuffer := make([]byte, authSigMsgSize) - _, err2 = io.ReadFull(sc, readBuffer) - if err2 != nil { - return + // Send our info and receive theirs in tandem. + var trs, _ = cmn.Parallel( + func(_ int) (val interface{}, err error, abort bool) { + var _, err1 = cdc.MarshalBinaryWriter(sc, authSigMessage{pubKey, signature}) + if err1 != nil { + return nil, err1, true // abort + } else { + return nil, nil, false } - n := int(0) // not used. - recvMsg = wire.ReadBinary(authSigMessage{}, bytes.NewBuffer(readBuffer), authSigMsgSize, &n, &err2).(authSigMessage) - }) + }, + func(_ int) (val interface{}, err error, abort bool) { + var _recvMsg authSigMessage + var _, err2 = cdc.UnmarshalBinaryReader(sc, &_recvMsg, 1024*1024) // TODO + if err2 != nil { + return nil, err2, true // abort + } else { + return _recvMsg, nil, false + } + }, + ) - if err1 != nil { - return nil, err1 - } - if err2 != nil { - return nil, err2 + // If error: + if trs.FirstError() != nil { + err = trs.FirstError() + return + } else if trs.FirstPanic() != nil { + err = fmt.Errorf("Panic: %v", trs.FirstPanic()) + return } - return &recvMsg, nil + var _recvMsg = trs.FirstValue().(authSigMessage) + return _recvMsg, nil } //-------------------------------------------------------------------------------- diff --git a/p2p/conn/secret_connection_test.go b/p2p/conn/secret_connection_test.go index c81b3b285..179413ddc 100644 --- a/p2p/conn/secret_connection_test.go +++ b/p2p/conn/secret_connection_test.go @@ -1,9 +1,12 @@ package conn import ( + "fmt" "io" "testing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" crypto "github.com/tendermint/go-crypto" cmn "github.com/tendermint/tmlibs/common" ) @@ -30,39 +33,49 @@ func makeKVStoreConnPair() (fooConn, barConn kvstoreConn) { } func makeSecretConnPair(tb testing.TB) (fooSecConn, barSecConn *SecretConnection) { - fooConn, barConn := makeKVStoreConnPair() - fooPrvKey := crypto.GenPrivKeyEd25519().Wrap() - fooPubKey := fooPrvKey.PubKey() - barPrvKey := crypto.GenPrivKeyEd25519().Wrap() - barPubKey := barPrvKey.PubKey() - cmn.Parallel( - func() { - var err error + var fooConn, barConn = makeKVStoreConnPair() + var fooPrvKey = crypto.GenPrivKeyEd25519() + var fooPubKey = fooPrvKey.PubKey() + var barPrvKey = crypto.GenPrivKeyEd25519() + var barPubKey = barPrvKey.PubKey() + + // Make connections from both sides in parallel. + var trs, ok = cmn.Parallel( + func(_ int) (val interface{}, err error, abort bool) { fooSecConn, err = MakeSecretConnection(fooConn, fooPrvKey) if err != nil { tb.Errorf("Failed to establish SecretConnection for foo: %v", err) - return + return nil, err, true } remotePubBytes := fooSecConn.RemotePubKey() if !remotePubBytes.Equals(barPubKey) { - tb.Errorf("Unexpected fooSecConn.RemotePubKey. Expected %v, got %v", + err = fmt.Errorf("Unexpected fooSecConn.RemotePubKey. Expected %v, got %v", barPubKey, fooSecConn.RemotePubKey()) + tb.Error(err) + return nil, err, false } + return nil, nil, false }, - func() { - var err error + func(_ int) (val interface{}, err error, abort bool) { barSecConn, err = MakeSecretConnection(barConn, barPrvKey) if barSecConn == nil { tb.Errorf("Failed to establish SecretConnection for bar: %v", err) - return + return nil, err, true } remotePubBytes := barSecConn.RemotePubKey() if !remotePubBytes.Equals(fooPubKey) { - tb.Errorf("Unexpected barSecConn.RemotePubKey. Expected %v, got %v", + err = fmt.Errorf("Unexpected barSecConn.RemotePubKey. Expected %v, got %v", fooPubKey, barSecConn.RemotePubKey()) + tb.Error(err) + return nil, nil, false } - }) + return nil, nil, false + }, + ) + require.Nil(tb, trs.FirstPanic()) + require.Nil(tb, trs.FirstError()) + require.True(tb, ok, "Unexpected task abortion") return } @@ -89,59 +102,80 @@ func TestSecretConnectionReadWrite(t *testing.T) { } // A helper that will run with (fooConn, fooWrites, fooReads) and vice versa - genNodeRunner := func(nodeConn kvstoreConn, nodeWrites []string, nodeReads *[]string) func() { - return func() { - // Node handskae - nodePrvKey := crypto.GenPrivKeyEd25519().Wrap() + genNodeRunner := func(id string, nodeConn kvstoreConn, nodeWrites []string, nodeReads *[]string) cmn.Task { + return func(_ int) (interface{}, error, bool) { + // Initiate cryptographic private key and secret connection trhough nodeConn. + nodePrvKey := crypto.GenPrivKeyEd25519() nodeSecretConn, err := MakeSecretConnection(nodeConn, nodePrvKey) if err != nil { t.Errorf("Failed to establish SecretConnection for node: %v", err) - return + return nil, err, true } - // In parallel, handle reads and writes - cmn.Parallel( - func() { - // Node writes + // In parallel, handle some reads and writes. + var trs, ok = cmn.Parallel( + func(_ int) (interface{}, error, bool) { + // Node writes: for _, nodeWrite := range nodeWrites { n, err := nodeSecretConn.Write([]byte(nodeWrite)) if err != nil { t.Errorf("Failed to write to nodeSecretConn: %v", err) - return + return nil, err, true } if n != len(nodeWrite) { - t.Errorf("Failed to write all bytes. Expected %v, wrote %v", len(nodeWrite), n) - return + err = fmt.Errorf("Failed to write all bytes. Expected %v, wrote %v", len(nodeWrite), n) + t.Error(err) + return nil, err, true } } if err := nodeConn.PipeWriter.Close(); err != nil { t.Error(err) + return nil, err, true } + return nil, nil, false }, - func() { - // Node reads + func(_ int) (interface{}, error, bool) { + // Node reads: readBuffer := make([]byte, dataMaxSize) for { n, err := nodeSecretConn.Read(readBuffer) if err == io.EOF { - return + return nil, nil, false } else if err != nil { t.Errorf("Failed to read from nodeSecretConn: %v", err) - return + return nil, err, true } *nodeReads = append(*nodeReads, string(readBuffer[:n])) } if err := nodeConn.PipeReader.Close(); err != nil { t.Error(err) + return nil, err, true } - }) + return nil, nil, false + }, + ) + assert.True(t, ok, "Unexpected task abortion") + + // If error: + if trs.FirstError() != nil { + return nil, trs.FirstError(), true + } else if trs.FirstPanic() != nil { + err = fmt.Errorf("Panic in task: %v", trs.FirstPanic()) + return nil, err, true + } + + // Otherwise: + return nil, nil, false } } // Run foo & bar in parallel - cmn.Parallel( - genNodeRunner(fooConn, fooWrites, &fooReads), - genNodeRunner(barConn, barWrites, &barReads), + var trs, ok = cmn.Parallel( + genNodeRunner("foo", fooConn, fooWrites, &fooReads), + genNodeRunner("bar", barConn, barWrites, &barReads), ) + require.Nil(t, trs.FirstPanic()) + require.Nil(t, trs.FirstError()) + require.True(t, ok, "unexpected task abortion") // A helper to ensure that the writes and reads match. // Additionally, small writes (<= dataMaxSize) must be atomically read. @@ -209,3 +243,11 @@ func BenchmarkSecretConnection(b *testing.B) { } //barSecConn.Close() race condition } + +func fingerprint(bz []byte) []byte { + if len(bz) < 40 { + return bz + } else { + return bz[:40] + } +} diff --git a/p2p/conn/wire.go b/p2p/conn/wire.go new file mode 100644 index 000000000..d73296d1b --- /dev/null +++ b/p2p/conn/wire.go @@ -0,0 +1,14 @@ +package conn + +import ( + "github.com/tendermint/go-amino" + "github.com/tendermint/go-crypto" +) + +var cdc *amino.Codec + +func init() { + cdc = amino.NewCodec() + crypto.RegisterAmino(cdc) + RegisterPacket(cdc) +} diff --git a/wire/wire.go b/wire/wire.go deleted file mode 100644 index 9d0d2c208..000000000 --- a/wire/wire.go +++ /dev/null @@ -1,60 +0,0 @@ -package wire - -import ( - "github.com/tendermint/go-wire" -) - -/* -// Expose access to a global wire codec -// TODO: maybe introduce some Context object -// containing logger, config, codec that can -// be threaded through everything to avoid this global -var cdc *wire.Codec - -func init() { - cdc = wire.NewCodec() - crypto.RegisterWire(cdc) -} -*/ - -// Just a flow through to go-wire. -// To be used later for the global codec - -func MarshalBinary(o interface{}) ([]byte, error) { - return wire.MarshalBinary(o) -} - -func UnmarshalBinary(bz []byte, ptr interface{}) error { - return wire.UnmarshalBinary(bz, ptr) -} - -func MarshalJSON(o interface{}) ([]byte, error) { - return wire.MarshalJSON(o) -} - -func UnmarshalJSON(jsonBz []byte, ptr interface{}) error { - return wire.UnmarshalJSON(jsonBz, ptr) -} - -type ConcreteType = wire.ConcreteType - -func RegisterInterface(o interface{}, ctypes ...ConcreteType) *wire.TypeInfo { - return wire.RegisterInterface(o, ctypes...) -} - -const RFC3339Millis = wire.RFC3339Millis - -/* - -func RegisterInterface(ptr interface{}, opts *wire.InterfaceOptions) { - cdc.RegisterInterface(ptr, opts) -} - -func RegisterConcrete(o interface{}, name string, opts *wire.ConcreteOptions) { - cdc.RegisterConcrete(o, name, opts) -} - -//------------------------------- - -const RFC3339Millis = wire.RFC3339Millis -*/ From ced74251e9493fe0fc82f4f89683545d735958ac Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Wed, 21 Mar 2018 02:38:27 +0100 Subject: [PATCH 002/143] maxPacketMsg -> packetMsgMax... --- p2p/conn/connection.go | 12 ++++++------ p2p/conn/secret_connection.go | 1 - 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/p2p/conn/connection.go b/p2p/conn/connection.go index 1c1a696f7..e57c633a3 100644 --- a/p2p/conn/connection.go +++ b/p2p/conn/connection.go @@ -19,8 +19,8 @@ import ( ) const ( - defaultMaxPacketMsgPayloadSize = 1024 - maxPacketMsgOverheadSize = 10 // It's actually lower but good enough + packetMsgMaxPayloadSizeDefault = 1024 + packetMsgMaxOverheadSize = 10 // It's actually lower but good enough numBatchPacketMsgs = 10 minReadBufferSize = 1024 @@ -105,7 +105,7 @@ type MConnConfig struct { RecvRate int64 `mapstructure:"recv_rate"` // Maximum payload size - MaxPacketMsgPayloadSize int `mapstructure:"max_msg_packet_payload_size"` + PacketMsgMaxPayloadSize int `mapstructure:"packet_msg_max_payload_size"` // Interval to flush writes (throttled) FlushThrottle time.Duration `mapstructure:"flush_throttle"` @@ -118,7 +118,7 @@ type MConnConfig struct { } func (cfg *MConnConfig) maxPacketMsgTotalSize() int { - return cfg.MaxPacketMsgPayloadSize + maxPacketMsgOverheadSize + return cfg.PacketMsgMaxPayloadSize + packetMsgMaxOverheadSize } // DefaultMConnConfig returns the default config. @@ -126,7 +126,7 @@ func DefaultMConnConfig() *MConnConfig { return &MConnConfig{ SendRate: defaultSendRate, RecvRate: defaultRecvRate, - MaxPacketMsgPayloadSize: defaultMaxPacketMsgPayloadSize, + PacketMsgMaxPayloadSize: packetMsgMaxPayloadSizeDefault, FlushThrottle: defaultFlushThrottle, PingInterval: defaultPingInterval, PongTimeout: defaultPongTimeout, @@ -639,7 +639,7 @@ func newChannel(conn *MConnection, desc ChannelDescriptor) *Channel { desc: desc, sendQueue: make(chan []byte, desc.SendQueueCapacity), recving: make([]byte, 0, desc.RecvBufferCapacity), - maxPacketMsgPayloadSize: conn.config.MaxPacketMsgPayloadSize, + maxPacketMsgPayloadSize: conn.config.PacketMsgMaxPayloadSize, } } diff --git a/p2p/conn/secret_connection.go b/p2p/conn/secret_connection.go index 58d2e366f..5505923f8 100644 --- a/p2p/conn/secret_connection.go +++ b/p2p/conn/secret_connection.go @@ -91,7 +91,6 @@ func MakeSecretConnection(conn io.ReadWriteCloser, locPrivKey crypto.PrivKey) (* // Share (in secret) each other's pubkey & challenge signature authSigMsg, err := shareAuthSignature(sc, locPubKey, locSignature) if err != nil { - fmt.Println(">>>", err) return nil, err } remPubKey, remSignature := authSigMsg.Key, authSigMsg.Sig From 416f03c05b3b89559fb0fe3472be8764f7837108 Mon Sep 17 00:00:00 2001 From: Zarko Milosevic Date: Mon, 5 Mar 2018 14:42:23 +0100 Subject: [PATCH 003/143] Add light client spec --- docs/specification/new-spec/light-client.md | 114 ++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 docs/specification/new-spec/light-client.md diff --git a/docs/specification/new-spec/light-client.md b/docs/specification/new-spec/light-client.md new file mode 100644 index 000000000..0ed9d36d4 --- /dev/null +++ b/docs/specification/new-spec/light-client.md @@ -0,0 +1,114 @@ +# Light client + +A light client is a process that connects to the Tendermint Full Node(s) and then tries to verify the Merkle proofs +about the blockchain application. In this document we describe mechanisms that ensures that the Tendermint light client +has the same level of security as Full Node processes (without being itself a Full Node). + +To be able to validate a Merkle proof, a light client needs to validate the blockchain header that contains the root app hash. +Validating a blockchain header in Tendermint consists in verifying that the header is committed (signed) by >2/3 of the +voting power of the corresponding validator set. As the validator set is a dynamic set (it is changing), one of the +core functionality of the light client is updating the current validator set, that is then used to verify the +blockchain header, and further the corresponding Merkle proofs. + +For the purpose of this light client specification, we assume that the Tendermint Full Node exposes the following functions over +Tendermint RPC: + +```golang +Header(height int64) (SignedHeader, error) // returns signed header for the given height +Validators(height int64) (ResultValidators, error) // returns validator set for the given height +LastHeader(valSetNumber int64) (SignedHeader, error) // returns last header signed by the validator set with the given validator set number + +type SignedHeader struct { + Header Header + Commit Commit + ValSetNumber int64 +} + +type ResultValidators struct { + BlockHeight int64 + Validators []Validator + // time the current validator set is initialised, i.e, time of the last validator change before header BlockHeight + ValSetTime int64 +} +``` + +We assume that Tendermint keeps track of the validator set changes and that each time a validator set is changed it is +being assigned the next sequence number. We can call this number the validator set sequence number. Tendermint also remembers +the Time from the header when the next validator set is initialised (starts to be in power), and we refer to this time +as validator set init time. +Furthermore, we assume that each validator set change is signed (committed) by the current validator set. More precisely, +given a block `H` that contains transactions that are modifying the current validator set, the Merkle root hash of the next +validator set (modified based on transactions from block H) will be in block `H+1` (and signed by the current validator +set), and then starting from the block `H+2`, it will be signed by the next validator set. + +Note that the real Tendermint RPC API is slightly different (for example, response messages contain more data and function +names are slightly different); we shortened (and modified) it for the purpose of this document to make the spec more +clear and simple. Furthermore, note that in case of the third function, the returned header has `ValSetNumber` equals to +`valSetNumber+1`. + + +Locally, light client manages the following state: + +```golang +valSet []Validator // current validator set (last known and verified validator set) +valSetNumber int64 // sequence number of the current validator set +valSetHash []byte // hash of the current validator set +valSetTime int64 // time when the current validator set is initialised +``` + +The light client is initialised with the trusted validator set, for example based on the known validator set hash, +validator set sequence number and the validator set init time. +The core of the light client logic is captured by the VerifyAndUpdate function that is used to 1) verify if the given header is valid, +and 2) update the validator set (when the given header is valid and it is more recent than the seen headers). + +```golang +VerifyAndUpdate(signedHeader SignedHeader): + assertThat signedHeader.valSetNumber >= valSetNumber + if isValid(signedHeader) and signedHeader.Header.Time <= valSetTime + UNBONDING_PERIOD then + setValidatorSet(signedHeader) + return true + else + updateValidatorSet(signedHeader.ValSetNumber) + return VerifyAndUpdate(signedHeader) + +isValid(signedHeader SignedHeader): + valSetOfTheHeader = Validators(signedHeader.Header.Height) + assertThat Hash(valSetOfTheHeader) == signedHeader.Header.ValSetHash + assertThat signedHeader is passing basic validation + if votingPower(signedHeader.Commit) > 2/3 * votingPower(valSetOfTheHeader) then return true + else + return false + +setValidatorSet(signedHeader SignedHeader): + nextValSet = Validators(signedHeader.Header.Height) + assertThat Hash(nextValSet) == signedHeader.Header.ValidatorsHash + valSet = nextValSet.Validators + valSetHash = signedHeader.Header.ValidatorsHash + valSetNumber = signedHeader.ValSetNumber + valSetTime = nextValSet.ValSetTime + +votingPower(commit Commit): + votingPower = 0 + for each precommit in commit.Precommits do: + if precommit.ValidatorAddress is in valSet and signature of the precommit verifies then + votingPower += valSet[precommit.ValidatorAddress].VotingPower + return votingPower + +votingPower(validatorSet []Validator): + for each validator in validatorSet do: + votingPower += validator.VotingPower + return votingPower + +updateValidatorSet(valSetNumberOfTheHeader): + while valSetNumber != valSetNumberOfTheHeader do + signedHeader = LastHeader(valSetNumber) + if isValid(signedHeader) then + setValidatorSet(signedHeader) + else return error + return +``` + +Note that in the logic above we assume that the light client will always go upward with respect to header verifications, +i.e., that it will always be used to verify more recent headers. In case a light client needs to be used to verify older +headers (go backward) the same mechanisms and similar logic can be used. In case a call to the FullNode or subsequent +checks fail, a light client need to implement some recovery strategy, for example connecting to other FullNode. From 4c2f56626ac3161d871bd4b44ec80ea0ff3b1284 Mon Sep 17 00:00:00 2001 From: Emmanuel T Odeke Date: Sat, 10 Mar 2018 21:15:55 -0800 Subject: [PATCH 004/143] lite/proxy: Validation* tests and hardening for nil dereferences Updates https://github.com/tendermint/tendermint/issues/1017 Ensure that the Validate* functions in proxy are tests and cover the case of sneakish bugs that have been encountered a few times from nil dereferences. The lite package should theoretically never panic with a nil dereference. It is meant to contain the certifiers hence it should never panic with such. Requires the following bugs to be fixed first; * https://github.com/tendermint/tendermint/issues/1298 * https://github.com/tendermint/tendermint/issues/1299 --- lite/proxy/block.go | 9 ++ lite/proxy/validate_test.go | 256 ++++++++++++++++++++++++++++++++++++ 2 files changed, 265 insertions(+) create mode 100644 lite/proxy/validate_test.go diff --git a/lite/proxy/block.go b/lite/proxy/block.go index 60cd00f0d..4cff9ee68 100644 --- a/lite/proxy/block.go +++ b/lite/proxy/block.go @@ -11,11 +11,17 @@ import ( ) func ValidateBlockMeta(meta *types.BlockMeta, check lite.Commit) error { + if meta == nil { + return errors.New("expecting a non-nil BlockMeta") + } // TODO: check the BlockID?? return ValidateHeader(meta.Header, check) } func ValidateBlock(meta *types.Block, check lite.Commit) error { + if meta == nil { + return errors.New("expecting a non-nil Block") + } err := ValidateHeader(meta.Header, check) if err != nil { return err @@ -27,6 +33,9 @@ func ValidateBlock(meta *types.Block, check lite.Commit) error { } func ValidateHeader(head *types.Header, check lite.Commit) error { + if head == nil { + return errors.New("expecting a non-nil Header") + } // make sure they are for the same height (obvious fail) if head.Height != check.Height() { return certerr.ErrHeightMismatch(head.Height, check.Height()) diff --git a/lite/proxy/validate_test.go b/lite/proxy/validate_test.go new file mode 100644 index 000000000..971a5e5b3 --- /dev/null +++ b/lite/proxy/validate_test.go @@ -0,0 +1,256 @@ +package proxy_test + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/tendermint/tendermint/lite" + "github.com/tendermint/tendermint/lite/proxy" + "github.com/tendermint/tendermint/types" +) + +var ( + deabBeefTxs = types.Txs{[]byte("DE"), []byte("AD"), []byte("BE"), []byte("EF")} + deadBeefRipEmd160Hash = deabBeefTxs.Hash() +) + +func TestValidateBlock(t *testing.T) { + tests := []struct { + block *types.Block + commit lite.Commit + wantErr string + }{ + { + block: nil, wantErr: "non-nil Block", + }, + { + block: &types.Block{}, wantErr: "nil Header", + }, + { + block: &types.Block{Header: new(types.Header)}, + }, + + // Start Header.Height mismatch test + { + block: &types.Block{Header: &types.Header{Height: 10}}, + commit: lite.Commit{Header: &types.Header{Height: 11}}, + wantErr: "don't match - 10 vs 11", + }, + + { + block: &types.Block{Header: &types.Header{Height: 11}}, + commit: lite.Commit{Header: &types.Header{Height: 11}}, + }, + // End Header.Height mismatch test + + // Start Header.Hash mismatch test + { + block: &types.Block{ + Header: &types.Header{ + Height: 11, + Time: time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC), + ValidatorsHash: []byte("Tendermint"), + }, + }, + commit: lite.Commit{Header: &types.Header{Height: 11}}, + wantErr: "Headers don't match", + }, + + { + block: &types.Block{ + Header: &types.Header{ + Height: 11, + Time: time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC), + ValidatorsHash: []byte("Tendermint"), + }, + }, + commit: lite.Commit{ + Header: &types.Header{ + Height: 11, + Time: time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC), + ValidatorsHash: []byte("Tendermint"), + }, + }, + }, + // End Header.Hash mismatch test + + // Start Header.Data hash mismatch test + { + block: &types.Block{ + Header: &types.Header{Height: 11}, + Data: &types.Data{Txs: []types.Tx{[]byte("0xDE"), []byte("AD")}}, + }, + commit: lite.Commit{ + Header: &types.Header{Height: 11}, + Commit: &types.Commit{BlockID: types.BlockID{Hash: []byte("0xDEADBEEF")}}, + }, + wantErr: "Data hash doesn't match header", + }, + { + block: &types.Block{ + Header: &types.Header{Height: 11, DataHash: deadBeefRipEmd160Hash}, + Data: &types.Data{Txs: deabBeefTxs}, + }, + commit: lite.Commit{ + Header: &types.Header{Height: 11}, + Commit: &types.Commit{BlockID: types.BlockID{Hash: []byte("DEADBEEF")}}, + }, + }, + // End Header.Data hash mismatch test + } + + assert := assert.New(t) + + for i, tt := range tests { + err := proxy.ValidateBlock(tt.block, tt.commit) + if tt.wantErr != "" { + if err == nil { + assert.FailNowf("Unexpectedly passed", "#%d", i) + } else { + assert.Contains(err.Error(), tt.wantErr, "#%d should contain the substring\n\n", i) + } + continue + } + + assert.Nil(err, "#%d: expecting a nil error", i) + } +} + +func TestValidateBlockMeta(t *testing.T) { + tests := []struct { + meta *types.BlockMeta + commit lite.Commit + wantErr string + }{ + { + meta: nil, wantErr: "non-nil BlockMeta", + }, + { + meta: &types.BlockMeta{}, wantErr: "non-nil Header", + }, + { + meta: &types.BlockMeta{Header: new(types.Header)}, + }, + + // Start Header.Height mismatch test + { + meta: &types.BlockMeta{Header: &types.Header{Height: 10}}, + commit: lite.Commit{Header: &types.Header{Height: 11}}, + wantErr: "don't match - 10 vs 11", + }, + + { + meta: &types.BlockMeta{Header: &types.Header{Height: 11}}, + commit: lite.Commit{Header: &types.Header{Height: 11}}, + }, + // End Header.Height mismatch test + + // Start Headers don't match test + { + meta: &types.BlockMeta{ + Header: &types.Header{ + Height: 11, + Time: time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC), + ValidatorsHash: []byte("Tendermint"), + }, + }, + commit: lite.Commit{Header: &types.Header{Height: 11}}, + wantErr: "Headers don't match", + }, + + { + meta: &types.BlockMeta{ + Header: &types.Header{ + Height: 11, + Time: time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC), + ValidatorsHash: []byte("Tendermint"), + }, + }, + commit: lite.Commit{ + Header: &types.Header{ + Height: 11, + Time: time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC), + ValidatorsHash: []byte("Tendermint"), + }, + }, + }, + + { + meta: &types.BlockMeta{ + Header: &types.Header{ + Height: 11, + // TODO: (@odeke-em) inquire why ValidatorsHash has to be non-blank + // for the Header to be hashed. Perhaps this is a security hole because + // an aggressor could perhaps pass in headers that don't have + // ValidatorsHash set and we won't be able to validate blocks. + ValidatorsHash: []byte("lite-test"), + // TODO: (@odeke-em) file an issue with Tendermint to get them to update + // to the latest go-wire, then no more need for this value fill to avoid + // the time zero value of less than 1970. + Time: time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC), + }, + }, + commit: lite.Commit{ + Header: &types.Header{Height: 11, DataHash: deadBeefRipEmd160Hash}, + }, + wantErr: "Headers don't match", + }, + + { + meta: &types.BlockMeta{ + Header: &types.Header{ + Height: 11, DataHash: deadBeefRipEmd160Hash, + ValidatorsHash: []byte("Tendermint"), + Time: time.Date(2017, 1, 2, 1, 1, 1, 1, time.UTC), + }, + }, + commit: lite.Commit{ + Header: &types.Header{ + Height: 11, DataHash: deadBeefRipEmd160Hash, + ValidatorsHash: []byte("Tendermint"), + Time: time.Date(2017, 1, 2, 2, 1, 1, 1, time.UTC), + }, + Commit: &types.Commit{BlockID: types.BlockID{Hash: []byte("DEADBEEF")}}, + }, + wantErr: "Headers don't match", + }, + + { + meta: &types.BlockMeta{ + Header: &types.Header{ + Height: 11, DataHash: deadBeefRipEmd160Hash, + ValidatorsHash: []byte("Tendermint"), + Time: time.Date(2017, 1, 2, 1, 1, 1, 1, time.UTC), + }, + }, + commit: lite.Commit{ + Header: &types.Header{ + Height: 11, DataHash: deadBeefRipEmd160Hash, + ValidatorsHash: []byte("Tendermint-x"), + Time: time.Date(2017, 1, 2, 1, 1, 1, 1, time.UTC), + }, + Commit: &types.Commit{BlockID: types.BlockID{Hash: []byte("DEADBEEF")}}, + }, + wantErr: "Headers don't match", + }, + // End Headers don't match test + } + + assert := assert.New(t) + + for i, tt := range tests { + err := proxy.ValidateBlockMeta(tt.meta, tt.commit) + if tt.wantErr != "" { + if err == nil { + assert.FailNowf("Unexpectedly passed", "#%d: wanted error %q", i, tt.wantErr) + } else { + assert.Contains(err.Error(), tt.wantErr, "#%d should contain the substring\n\n", i) + } + continue + } + + assert.Nil(err, "#%d: expecting a nil error", i) + } +} From 58f36bb3215c54287d6693119d0071b97052b673 Mon Sep 17 00:00:00 2001 From: Emmanuel T Odeke Date: Sat, 10 Mar 2018 21:50:15 -0800 Subject: [PATCH 005/143] Review feedback from @melekes * Fix typo on naming s/deabBeef/deadBeef/g * Use `assert.*(t,` instead of `assert.New(t);...;assert.*(` --- lite/proxy/validate_test.go | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/lite/proxy/validate_test.go b/lite/proxy/validate_test.go index 971a5e5b3..653bfb753 100644 --- a/lite/proxy/validate_test.go +++ b/lite/proxy/validate_test.go @@ -12,8 +12,8 @@ import ( ) var ( - deabBeefTxs = types.Txs{[]byte("DE"), []byte("AD"), []byte("BE"), []byte("EF")} - deadBeefRipEmd160Hash = deabBeefTxs.Hash() + deadBeefTxs = types.Txs{[]byte("DE"), []byte("AD"), []byte("BE"), []byte("EF")} + deadBeefRipEmd160Hash = deadBeefTxs.Hash() ) func TestValidateBlock(t *testing.T) { @@ -91,7 +91,7 @@ func TestValidateBlock(t *testing.T) { { block: &types.Block{ Header: &types.Header{Height: 11, DataHash: deadBeefRipEmd160Hash}, - Data: &types.Data{Txs: deabBeefTxs}, + Data: &types.Data{Txs: deadBeefTxs}, }, commit: lite.Commit{ Header: &types.Header{Height: 11}, @@ -101,20 +101,18 @@ func TestValidateBlock(t *testing.T) { // End Header.Data hash mismatch test } - assert := assert.New(t) - for i, tt := range tests { err := proxy.ValidateBlock(tt.block, tt.commit) if tt.wantErr != "" { if err == nil { - assert.FailNowf("Unexpectedly passed", "#%d", i) + assert.FailNowf(t, "Unexpectedly passed", "#%d", i) } else { - assert.Contains(err.Error(), tt.wantErr, "#%d should contain the substring\n\n", i) + assert.Contains(t, err.Error(), tt.wantErr, "#%d should contain the substring\n\n", i) } continue } - assert.Nil(err, "#%d: expecting a nil error", i) + assert.Nil(t, err, "#%d: expecting a nil error", i) } } @@ -238,19 +236,17 @@ func TestValidateBlockMeta(t *testing.T) { // End Headers don't match test } - assert := assert.New(t) - for i, tt := range tests { err := proxy.ValidateBlockMeta(tt.meta, tt.commit) if tt.wantErr != "" { if err == nil { - assert.FailNowf("Unexpectedly passed", "#%d: wanted error %q", i, tt.wantErr) + assert.FailNowf(t, "Unexpectedly passed", "#%d: wanted error %q", i, tt.wantErr) } else { - assert.Contains(err.Error(), tt.wantErr, "#%d should contain the substring\n\n", i) + assert.Contains(t, err.Error(), tt.wantErr, "#%d should contain the substring\n\n", i) } continue } - assert.Nil(err, "#%d: expecting a nil error", i) + assert.Nil(t, err, "#%d: expecting a nil error", i) } } From 8813684040212782a76e4ccf9627d7be8901f771 Mon Sep 17 00:00:00 2001 From: Emmanuel T Odeke Date: Sun, 25 Mar 2018 00:27:38 -0600 Subject: [PATCH 006/143] lite/proxy: consolidate some common test headers into a variable Addressing some feedback from @ebuchman in regards to consolidating some common test headers into a variable. I've added that for simple cases, trying to meet in the middle instead of creating helpers that obscure readibility and easy comparison of test cases. --- lite/proxy/validate_test.go | 59 ++++++++++--------------------------- 1 file changed, 15 insertions(+), 44 deletions(-) diff --git a/lite/proxy/validate_test.go b/lite/proxy/validate_test.go index 653bfb753..bd57994d7 100644 --- a/lite/proxy/validate_test.go +++ b/lite/proxy/validate_test.go @@ -12,10 +12,17 @@ import ( ) var ( - deadBeefTxs = types.Txs{[]byte("DE"), []byte("AD"), []byte("BE"), []byte("EF")} + deadBeefTxs = types.Txs{[]byte("DE"), []byte("AD"), []byte("BE"), []byte("EF")} + deadBeefRipEmd160Hash = deadBeefTxs.Hash() ) +var hdrHeight11Tendermint = &types.Header{ + Height: 11, + Time: time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC), + ValidatorsHash: []byte("Tendermint"), +} + func TestValidateBlock(t *testing.T) { tests := []struct { block *types.Block @@ -47,32 +54,14 @@ func TestValidateBlock(t *testing.T) { // Start Header.Hash mismatch test { - block: &types.Block{ - Header: &types.Header{ - Height: 11, - Time: time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC), - ValidatorsHash: []byte("Tendermint"), - }, - }, + block: &types.Block{Header: hdrHeight11Tendermint}, commit: lite.Commit{Header: &types.Header{Height: 11}}, wantErr: "Headers don't match", }, { - block: &types.Block{ - Header: &types.Header{ - Height: 11, - Time: time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC), - ValidatorsHash: []byte("Tendermint"), - }, - }, - commit: lite.Commit{ - Header: &types.Header{ - Height: 11, - Time: time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC), - ValidatorsHash: []byte("Tendermint"), - }, - }, + block: &types.Block{Header: hdrHeight11Tendermint}, + commit: lite.Commit{Header: hdrHeight11Tendermint}, }, // End Header.Hash mismatch test @@ -147,32 +136,14 @@ func TestValidateBlockMeta(t *testing.T) { // Start Headers don't match test { - meta: &types.BlockMeta{ - Header: &types.Header{ - Height: 11, - Time: time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC), - ValidatorsHash: []byte("Tendermint"), - }, - }, + meta: &types.BlockMeta{Header: hdrHeight11Tendermint}, commit: lite.Commit{Header: &types.Header{Height: 11}}, wantErr: "Headers don't match", }, { - meta: &types.BlockMeta{ - Header: &types.Header{ - Height: 11, - Time: time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC), - ValidatorsHash: []byte("Tendermint"), - }, - }, - commit: lite.Commit{ - Header: &types.Header{ - Height: 11, - Time: time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC), - ValidatorsHash: []byte("Tendermint"), - }, - }, + meta: &types.BlockMeta{Header: hdrHeight11Tendermint}, + commit: lite.Commit{Header: hdrHeight11Tendermint}, }, { @@ -208,7 +179,7 @@ func TestValidateBlockMeta(t *testing.T) { Header: &types.Header{ Height: 11, DataHash: deadBeefRipEmd160Hash, ValidatorsHash: []byte("Tendermint"), - Time: time.Date(2017, 1, 2, 2, 1, 1, 1, time.UTC), + Time: time.Date(2018, 1, 2, 1, 1, 1, 1, time.UTC), }, Commit: &types.Commit{BlockID: types.BlockID{Hash: []byte("DEADBEEF")}}, }, From 901b456151dea22c8c35aa516fb4b1809d80d586 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Mon, 26 Mar 2018 06:40:02 +0200 Subject: [PATCH 007/143] P2P now works with Amino --- Gopkg.lock | 34 +++++++-------- Gopkg.toml | 14 +++---- config/config.go | 4 +- config/toml.go | 2 +- p2p/conn/connection.go | 66 ++++++++++++++---------------- p2p/conn/connection_test.go | 40 +++++++++++++----- p2p/conn/secret_connection.go | 13 ++---- p2p/conn/secret_connection_test.go | 5 --- p2p/conn/wire.go | 3 +- p2p/key.go | 7 ++-- p2p/netaddress.go | 3 +- p2p/peer.go | 63 +++++++++++++--------------- p2p/peer_set_test.go | 2 +- p2p/peer_test.go | 10 ++--- p2p/pex/pex_reactor.go | 34 ++++++--------- p2p/pex/pex_reactor_test.go | 23 +++++------ p2p/pex/wire.go | 11 +++++ p2p/switch.go | 14 +++---- p2p/switch_test.go | 19 ++++----- p2p/test_util.go | 4 +- p2p/upnp/probe.go | 10 ++--- p2p/wire.go | 12 ++++++ 22 files changed, 193 insertions(+), 200 deletions(-) create mode 100644 p2p/pex/wire.go create mode 100644 p2p/wire.go diff --git a/Gopkg.lock b/Gopkg.lock index 9c9ffd416..7119588b1 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -229,7 +229,6 @@ revision = "169b1b37be738edb2813dab48c97a549bcf99bb5" [[projects]] - branch = "develop" name = "github.com/tendermint/abci" packages = [ "client", @@ -239,7 +238,8 @@ "server", "types" ] - revision = "4e0218467649fecf17ebc5e8161f1c888fc8ff22" + revision = "c62aed95f2ce399ec815b0cafe478af002cdc4e6" + version = "v0.10.3-dev" [[projects]] branch = "master" @@ -252,16 +252,16 @@ revision = "d8387025d2b9d158cf4efb07e7ebf814bcce2057" [[projects]] - branch = "develop" name = "github.com/tendermint/go-amino" packages = ["."] - revision = "b1f32ee20e73716d8bfe695365c0a812b2bd8ef9" + revision = "f55c3351f30e5987500020631f00e87a035ed415" + version = "0.9.3" [[projects]] - branch = "develop" name = "github.com/tendermint/go-crypto" packages = ["."] - revision = "a3800da0a15c8272cbd3c155e024bff28fe9692c" + revision = "a04f2ae5c6ba037014531241ed973116290b029e" + version = "v0.6.0" [[projects]] name = "github.com/tendermint/go-wire" @@ -273,7 +273,6 @@ version = "v0.7.3" [[projects]] - branch = "develop" name = "github.com/tendermint/tmlibs" packages = [ "autofile", @@ -289,7 +288,8 @@ "pubsub/query", "test" ] - revision = "4e5c655944c9a636eaed549e6ad8fd8011fb4d42" + revision = "e9cf47606cfcbdc28a7c16671b4a70b459e9d4cc" + version = "v0.8.0-dev" [[projects]] branch = "master" @@ -304,7 +304,7 @@ "ripemd160", "salsa20/salsa" ] - revision = "c3a3ad6d03f7a915c0f7e194b7152974bb73d287" + revision = "88942b9c40a4c9d203b82b3731787b672d6e809b" [[projects]] branch = "master" @@ -324,7 +324,7 @@ branch = "master" name = "golang.org/x/sys" packages = ["unix"] - revision = "7ceb54c8418b8f9cdf0177b511d5cbb06e9fae39" + revision = "91ee8cde435411ca3f1cd365e8f20131aed4d0a1" [[projects]] name = "golang.org/x/text" @@ -351,20 +351,16 @@ branch = "master" name = "google.golang.org/genproto" packages = ["googleapis/rpc/status"] - revision = "f8c8703595236ae70fdf8789ecb656ea0bcdcf46" + revision = "ab0870e398d5dd054b868c0db1481ab029b9a9f2" [[projects]] name = "google.golang.org/grpc" packages = [ ".", "balancer", - "balancer/base", - "balancer/roundrobin", "codes", "connectivity", "credentials", - "encoding", - "encoding/proto", "grpclb/grpc_lb_v1/messages", "grpclog", "internal", @@ -373,15 +369,13 @@ "naming", "peer", "resolver", - "resolver/dns", - "resolver/passthrough", "stats", "status", "tap", "transport" ] - revision = "8e4536a86ab602859c20df5ebfd0bd4228d08655" - version = "v1.10.0" + revision = "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + version = "v1.7.5" [[projects]] name = "gopkg.in/yaml.v2" @@ -392,6 +386,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "6da81f319b092e227b5d2c9de3b10296e9bb7287c02adb38fe547147e9e5e447" + inputs-digest = "9aace7a292e5402f97327860531e13b77f036c19843b698566ee958773404f48" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index ce1698145..aa523e119 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -26,12 +26,12 @@ [[constraint]] - branch = "master" name = "github.com/ebuchman/fail-test" + branch = "master" [[constraint]] - branch = "master" name = "github.com/fortytw2/leaktest" + branch = "master" [[constraint]] name = "github.com/go-kit/kit" @@ -54,8 +54,8 @@ version = "0.8.0" [[constraint]] - branch = "master" name = "github.com/rcrowley/go-metrics" + branch = "master" [[constraint]] name = "github.com/spf13/cobra" @@ -71,19 +71,19 @@ [[constraint]] name = "github.com/tendermint/abci" - branch = "develop" + version = "0.10.3-dev" [[constraint]] name = "github.com/tendermint/go-crypto" - branch = "develop" + version = "0.6.0" [[constraint]] name = "github.com/tendermint/go-amino" - branch = "develop" + version = "0.9.3" [[constraint]] name = "github.com/tendermint/tmlibs" - branch = "develop" + version = "0.8.0-dev" [[constraint]] name = "google.golang.org/grpc" diff --git a/config/config.go b/config/config.go index a433047f7..4b9930b61 100644 --- a/config/config.go +++ b/config/config.go @@ -270,7 +270,7 @@ type P2PConfig struct { FlushThrottleTimeout int `mapstructure:"flush_throttle_timeout"` // Maximum size of a message packet payload, in bytes - MaxMsgPacketPayloadSize int `mapstructure:"max_msg_packet_payload_size"` + MaxPacketMsgPayloadSize int `mapstructure:"max_packet_msg_payload_size"` // Rate at which packets can be sent, in bytes/second SendRate int64 `mapstructure:"send_rate"` @@ -299,7 +299,7 @@ func DefaultP2PConfig() *P2PConfig { AddrBookStrict: true, MaxNumPeers: 50, FlushThrottleTimeout: 100, - MaxMsgPacketPayloadSize: 1024, // 1 kB + MaxPacketMsgPayloadSize: 1024, // 1 kB SendRate: 512000, // 500 kB/s RecvRate: 512000, // 500 kB/s PexReactor: true, diff --git a/config/toml.go b/config/toml.go index 35e0985f9..af5be2345 100644 --- a/config/toml.go +++ b/config/toml.go @@ -142,7 +142,7 @@ flush_throttle_timeout = {{ .P2P.FlushThrottleTimeout }} max_num_peers = {{ .P2P.MaxNumPeers }} # Maximum size of a message packet payload, in bytes -max_msg_packet_payload_size = {{ .P2P.MaxMsgPacketPayloadSize }} +max_packet_msg_payload_size = {{ .P2P.MaxPacketMsgPayloadSize }} # Rate at which packets can be sent, in bytes/second send_rate = {{ .P2P.SendRate }} diff --git a/p2p/conn/connection.go b/p2p/conn/connection.go index e57c633a3..2970d960f 100644 --- a/p2p/conn/connection.go +++ b/p2p/conn/connection.go @@ -8,7 +8,6 @@ import ( "math" "net" "reflect" - "runtime/debug" "sync/atomic" "time" @@ -19,8 +18,8 @@ import ( ) const ( - packetMsgMaxPayloadSizeDefault = 1024 - packetMsgMaxOverheadSize = 10 // It's actually lower but good enough + maxPacketMsgPayloadSizeDefault = 1024 + maxPacketMsgOverheadSize = 14 numBatchPacketMsgs = 10 minReadBufferSize = 1024 @@ -57,15 +56,15 @@ The byte id and the relative priorities of each `Channel` are configured upon initialization of the connection. There are two methods for sending messages: - func (m MConnection) Send(chID byte, msg interface{}) bool {} - func (m MConnection) TrySend(chID byte, msg interface{}) bool {} + func (m MConnection) Send(chID byte, msgBytes []byte) bool {} + func (m MConnection) TrySend(chID byte, msgBytes []byte}) bool {} -`Send(chID, msg)` is a blocking call that waits until `msg` is successfully queued -for the channel with the given id byte `chID`, or until the request times out. -The message `msg` is serialized using Go-Amino. +`Send(chID, msgBytes)` is a blocking call that waits until `msg` is +successfully queued for the channel with the given id byte `chID`, or until the +request times out. The message `msg` is serialized using Go-Amino. -`TrySend(chID, msg)` is a nonblocking call that returns false if the channel's -queue is full. +`TrySend(chID, msgBytes)` is a nonblocking call that returns false if the +channel's queue is full. Inbound message bytes are handled with an onReceive callback function. */ @@ -105,7 +104,7 @@ type MConnConfig struct { RecvRate int64 `mapstructure:"recv_rate"` // Maximum payload size - PacketMsgMaxPayloadSize int `mapstructure:"packet_msg_max_payload_size"` + MaxPacketMsgPayloadSize int `mapstructure:"max_packet_msg_payload_size"` // Interval to flush writes (throttled) FlushThrottle time.Duration `mapstructure:"flush_throttle"` @@ -118,7 +117,7 @@ type MConnConfig struct { } func (cfg *MConnConfig) maxPacketMsgTotalSize() int { - return cfg.PacketMsgMaxPayloadSize + packetMsgMaxOverheadSize + return cfg.MaxPacketMsgPayloadSize + maxPacketMsgOverheadSize } // DefaultMConnConfig returns the default config. @@ -126,7 +125,7 @@ func DefaultMConnConfig() *MConnConfig { return &MConnConfig{ SendRate: defaultSendRate, RecvRate: defaultRecvRate, - PacketMsgMaxPayloadSize: packetMsgMaxPayloadSizeDefault, + MaxPacketMsgPayloadSize: maxPacketMsgPayloadSizeDefault, FlushThrottle: defaultFlushThrottle, PingInterval: defaultPingInterval, PongTimeout: defaultPongTimeout, @@ -233,8 +232,7 @@ func (c *MConnection) flush() { // Catch panics, usually caused by remote disconnects. func (c *MConnection) _recover() { if r := recover(); r != nil { - stack := debug.Stack() - err := cmn.StackError{r, stack} + err := cmn.ErrorWrap(r, "recovered panic in MConnection") c.stopForError(err) } } @@ -249,12 +247,12 @@ func (c *MConnection) stopForError(r interface{}) { } // Queues a message to be sent to channel. -func (c *MConnection) Send(chID byte, msg interface{}) bool { +func (c *MConnection) Send(chID byte, msgBytes []byte) bool { if !c.IsRunning() { return false } - c.Logger.Debug("Send", "channel", chID, "conn", c, "msg", msg) + c.Logger.Debug("Send", "channel", chID, "conn", c, "msgBytes", fmt.Sprintf("%X", msgBytes)) // Send message to channel. channel, ok := c.channelsIdx[chID] @@ -263,7 +261,7 @@ func (c *MConnection) Send(chID byte, msg interface{}) bool { return false } - success := channel.sendBytes(cdc.MustMarshalBinary(msg)) + success := channel.sendBytes(msgBytes) if success { // Wake up sendRoutine if necessary select { @@ -271,19 +269,19 @@ func (c *MConnection) Send(chID byte, msg interface{}) bool { default: } } else { - c.Logger.Error("Send failed", "channel", chID, "conn", c, "msg", msg) + c.Logger.Error("Send failed", "channel", chID, "conn", c, "msgBytes", fmt.Sprintf("%X", msgBytes)) } return success } // Queues a message to be sent to channel. // Nonblocking, returns true if successful. -func (c *MConnection) TrySend(chID byte, msg interface{}) bool { +func (c *MConnection) TrySend(chID byte, msgBytes []byte) bool { if !c.IsRunning() { return false } - c.Logger.Debug("TrySend", "channel", chID, "conn", c, "msg", msg) + c.Logger.Debug("TrySend", "channel", chID, "conn", c, "msgBytes", fmt.Sprintf("%X", msgBytes)) // Send message to channel. channel, ok := c.channelsIdx[chID] @@ -292,7 +290,7 @@ func (c *MConnection) TrySend(chID byte, msg interface{}) bool { return false } - ok = channel.trySendBytes(cdc.MustMarshalBinary(msg)) + ok = channel.trySendBytes(msgBytes) if ok { // Wake up sendRoutine if necessary select { @@ -462,18 +460,17 @@ FOR_LOOP: // Block until .recvMonitor says we can read. c.recvMonitor.Limit(c.config.maxPacketMsgTotalSize(), atomic.LoadInt64(&c.config.RecvRate), true) + // Peek into bufConnReader for debugging /* - // Peek into bufConnReader for debugging if numBytes := c.bufConnReader.Buffered(); numBytes > 0 { - log.Info("Peek connection buffer", "numBytes", numBytes, "bytes", log15.Lazy{func() []byte { - bytes, err := c.bufConnReader.Peek(cmn.MinInt(numBytes, 100)) - if err == nil { - return bytes - } else { - log.Warn("Error peeking connection buffer", "err", err) - return nil - } - }}) + bz, err := c.bufConnReader.Peek(cmn.MinInt(numBytes, 100)) + if err == nil { + // return + } else { + c.Logger.Debug("Error peeking connection buffer", "err", err) + // return nil + } + c.Logger.Info("Peek connection buffer", "numBytes", numBytes, "bz", bz) } */ @@ -639,7 +636,7 @@ func newChannel(conn *MConnection, desc ChannelDescriptor) *Channel { desc: desc, sendQueue: make(chan []byte, desc.SendQueueCapacity), recving: make([]byte, 0, desc.RecvBufferCapacity), - maxPacketMsgPayloadSize: conn.config.PacketMsgMaxPayloadSize, + maxPacketMsgPayloadSize: conn.config.MaxPacketMsgPayloadSize, } } @@ -719,7 +716,6 @@ func (ch *Channel) nextPacketMsg() PacketMsg { // Not goroutine-safe func (ch *Channel) writePacketMsgTo(w io.Writer) (n int64, err error) { var packet = ch.nextPacketMsg() - ch.Logger.Debug("Write Msg Packet", "conn", ch.conn, "packet", packet) n, err = cdc.MarshalBinaryWriter(w, packet) ch.recentlySent += n return @@ -729,7 +725,7 @@ func (ch *Channel) writePacketMsgTo(w io.Writer) (n int64, err error) { // complete. NOTE message bytes may change on next call to recvPacketMsg. // Not goroutine-safe func (ch *Channel) recvPacketMsg(packet PacketMsg) ([]byte, error) { - ch.Logger.Debug("Read Msg Packet", "conn", ch.conn, "packet", packet) + ch.Logger.Debug("Read PacketMsg", "conn", ch.conn, "packet", packet) var recvCap, recvReceived = ch.desc.RecvMessageCapacity, len(ch.recving) + len(packet.Bytes) if recvCap < recvReceived { return nil, fmt.Errorf("Received message exceeds available capacity: %v < %v", recvCap, recvReceived) diff --git a/p2p/conn/connection_test.go b/p2p/conn/connection_test.go index 193b77b26..9f03ec62b 100644 --- a/p2p/conn/connection_test.go +++ b/p2p/conn/connection_test.go @@ -1,6 +1,7 @@ package conn import ( + "bytes" "net" "testing" "time" @@ -41,7 +42,7 @@ func TestMConnectionSend(t *testing.T) { require.Nil(t, err) defer mconn.Stop() - msg := "Ant-Man" + msg := []byte("Ant-Man") assert.True(t, mconn.Send(0x01, msg)) // Note: subsequent Send/TrySend calls could pass because we are reading from // the send queue in a separate goroutine. @@ -51,7 +52,7 @@ func TestMConnectionSend(t *testing.T) { } assert.True(t, mconn.CanSend(0x01)) - msg = "Spider-Man" + msg = []byte("Spider-Man") assert.True(t, mconn.TrySend(0x01, msg)) _, err = server.Read(make([]byte, len(msg))) if err != nil { @@ -59,7 +60,7 @@ func TestMConnectionSend(t *testing.T) { } assert.False(t, mconn.CanSend(0x05), "CanSend should return false because channel is unknown") - assert.False(t, mconn.Send(0x05, "Absorbing Man"), "Send should return false because channel is unknown") + assert.False(t, mconn.Send(0x05, []byte("Absorbing Man")), "Send should return false because channel is unknown") } func TestMConnectionReceive(t *testing.T) { @@ -85,12 +86,12 @@ func TestMConnectionReceive(t *testing.T) { require.Nil(t, err) defer mconn2.Stop() - msg := "Cyclops" + msg := []byte("Cyclops") assert.True(t, mconn2.Send(0x01, msg)) select { case receivedBytes := <-receivedCh: - assert.Equal(t, []byte(msg), receivedBytes[2:]) // first 3 bytes are internal + assert.Equal(t, []byte(msg), receivedBytes) case err := <-errorsCh: t.Fatalf("Expected %s, got %+v", msg, err) case <-time.After(500 * time.Millisecond): @@ -386,7 +387,7 @@ func TestMConnectionReadErrorUnknownChannel(t *testing.T) { defer mconnClient.Stop() defer mconnServer.Stop() - msg := "Ant-Man" + msg := []byte("Ant-Man") // fail to send msg on channel unknown by client assert.False(t, mconnClient.Send(0x03, msg)) @@ -413,23 +414,40 @@ func TestMConnectionReadErrorLongMessage(t *testing.T) { // send msg thats just right var err error + var buf = new(bytes.Buffer) + // - Uvarint length of MustMarshalBinary(packet) = 1 or 2 bytes + // (as long as it's less than 16,384 bytes) + // - Prefix bytes = 4 bytes + // - ChannelID field key + byte = 2 bytes + // - EOF field key + byte = 2 bytes + // - Bytes field key = 1 bytes + // - Uvarint length of MustMarshalBinary(bytes) = 1 or 2 bytes + // - Struct terminator = 1 byte + // = up to 14 bytes overhead for the packet. var packet = PacketMsg{ ChannelID: 0x01, - Bytes: make([]byte, mconnClient.config.maxPacketMsgTotalSize()-12), EOF: 1, + Bytes: make([]byte, mconnClient.config.MaxPacketMsgPayloadSize), } - _, err = cdc.MarshalBinaryWriter(client, packet) + _, err = cdc.MarshalBinaryWriter(buf, packet) + assert.Nil(t, err) + _, err = client.Write(buf.Bytes()) assert.Nil(t, err) assert.True(t, expectSend(chOnRcv), "msg just right") + assert.False(t, expectSend(chOnErr), "msg just right") // send msg thats too long + buf = new(bytes.Buffer) packet = PacketMsg{ ChannelID: 0x01, - Bytes: make([]byte, mconnClient.config.maxPacketMsgTotalSize()-11), EOF: 1, + Bytes: make([]byte, mconnClient.config.MaxPacketMsgPayloadSize+1), } - _, err = cdc.MarshalBinaryWriter(client, packet) + _, err = cdc.MarshalBinaryWriter(buf, packet) assert.Nil(t, err) + _, err = client.Write(buf.Bytes()) + assert.NotNil(t, err) + assert.False(t, expectSend(chOnRcv), "msg too long") assert.True(t, expectSend(chOnErr), "msg too long") } @@ -458,7 +476,7 @@ func TestMConnectionTrySend(t *testing.T) { require.Nil(t, err) defer mconn.Stop() - msg := "Semicolon-Woman" + msg := []byte("Semicolon-Woman") resultCh := make(chan string, 2) assert.True(t, mconn.TrySend(0x01, msg)) server.Read(make([]byte, len(msg))) diff --git a/p2p/conn/secret_connection.go b/p2p/conn/secret_connection.go index 5505923f8..639204ec8 100644 --- a/p2p/conn/secret_connection.go +++ b/p2p/conn/secret_connection.go @@ -12,7 +12,6 @@ import ( "crypto/sha256" "encoding/binary" "errors" - "fmt" "io" "net" "time" @@ -145,8 +144,8 @@ func (sc *SecretConnection) Write(data []byte) (n int, err error) { // CONTRACT: data smaller than dataMaxSize is read atomically. func (sc *SecretConnection) Read(data []byte) (n int, err error) { if 0 < len(sc.recvBuffer) { - n_ := copy(data, sc.recvBuffer) - sc.recvBuffer = sc.recvBuffer[n_:] + n = copy(data, sc.recvBuffer) + sc.recvBuffer = sc.recvBuffer[n:] return } @@ -193,7 +192,7 @@ func genEphKeys() (ephPub, ephPriv *[32]byte) { var err error ephPub, ephPriv, err = box.GenerateKey(crand.Reader) if err != nil { - cmn.PanicCrisis("Could not generate ephemeral keypairs") + panic("Could not generate ephemeral keypairs") } return } @@ -225,9 +224,6 @@ func shareEphPubKey(conn io.ReadWriteCloser, locEphPub *[32]byte) (remEphPub *[3 if trs.FirstError() != nil { err = trs.FirstError() return - } else if trs.FirstPanic() != nil { - err = fmt.Errorf("Panic: %v", trs.FirstPanic()) - return } // Otherwise: @@ -309,9 +305,6 @@ func shareAuthSignature(sc *SecretConnection, pubKey crypto.PubKey, signature cr if trs.FirstError() != nil { err = trs.FirstError() return - } else if trs.FirstPanic() != nil { - err = fmt.Errorf("Panic: %v", trs.FirstPanic()) - return } var _recvMsg = trs.FirstValue().(authSigMessage) diff --git a/p2p/conn/secret_connection_test.go b/p2p/conn/secret_connection_test.go index 179413ddc..7d3f17b2f 100644 --- a/p2p/conn/secret_connection_test.go +++ b/p2p/conn/secret_connection_test.go @@ -73,7 +73,6 @@ func makeSecretConnPair(tb testing.TB) (fooSecConn, barSecConn *SecretConnection return nil, nil, false }, ) - require.Nil(tb, trs.FirstPanic()) require.Nil(tb, trs.FirstError()) require.True(tb, ok, "Unexpected task abortion") @@ -158,9 +157,6 @@ func TestSecretConnectionReadWrite(t *testing.T) { // If error: if trs.FirstError() != nil { return nil, trs.FirstError(), true - } else if trs.FirstPanic() != nil { - err = fmt.Errorf("Panic in task: %v", trs.FirstPanic()) - return nil, err, true } // Otherwise: @@ -173,7 +169,6 @@ func TestSecretConnectionReadWrite(t *testing.T) { genNodeRunner("foo", fooConn, fooWrites, &fooReads), genNodeRunner("bar", barConn, barWrites, &barReads), ) - require.Nil(t, trs.FirstPanic()) require.Nil(t, trs.FirstError()) require.True(t, ok, "unexpected task abortion") diff --git a/p2p/conn/wire.go b/p2p/conn/wire.go index d73296d1b..02d67f6fb 100644 --- a/p2p/conn/wire.go +++ b/p2p/conn/wire.go @@ -5,10 +5,9 @@ import ( "github.com/tendermint/go-crypto" ) -var cdc *amino.Codec +var cdc *amino.Codec = amino.NewCodec() func init() { - cdc = amino.NewCodec() crypto.RegisterAmino(cdc) RegisterPacket(cdc) } diff --git a/p2p/key.go b/p2p/key.go index ea0f0b071..18acfe862 100644 --- a/p2p/key.go +++ b/p2p/key.go @@ -3,7 +3,6 @@ package p2p import ( "bytes" "encoding/hex" - "encoding/json" "fmt" "io/ioutil" @@ -64,7 +63,7 @@ func loadNodeKey(filePath string) (*NodeKey, error) { return nil, err } nodeKey := new(NodeKey) - err = json.Unmarshal(jsonBytes, nodeKey) + err = cdc.UnmarshalJSON(jsonBytes, nodeKey) if err != nil { return nil, fmt.Errorf("Error reading NodeKey from %v: %v\n", filePath, err) } @@ -72,12 +71,12 @@ func loadNodeKey(filePath string) (*NodeKey, error) { } func genNodeKey(filePath string) (*NodeKey, error) { - privKey := crypto.GenPrivKeyEd25519().Wrap() + privKey := crypto.GenPrivKeyEd25519() nodeKey := &NodeKey{ PrivKey: privKey, } - jsonBytes, err := json.Marshal(nodeKey) + jsonBytes, err := cdc.MarshalJSON(nodeKey) if err != nil { return nil, err } diff --git a/p2p/netaddress.go b/p2p/netaddress.go index 333d16e5d..619ff3a3a 100644 --- a/p2p/netaddress.go +++ b/p2p/netaddress.go @@ -13,7 +13,6 @@ import ( "strings" "time" - "github.com/pkg/errors" cmn "github.com/tendermint/tmlibs/common" ) @@ -66,7 +65,7 @@ func NewNetAddressString(addr string) (*NetAddress, error) { idStr := spl[0] idBytes, err := hex.DecodeString(idStr) if err != nil { - return nil, errors.Wrap(err, fmt.Sprintf("Address (%s) contains invalid ID", addr)) + return nil, cmn.ErrorWrap(err, fmt.Sprintf("Address (%s) contains invalid ID", addr)) } if len(idBytes) != IDByteLength { return nil, fmt.Errorf("Address (%s) contains ID of invalid length (%d). Should be %d hex-encoded bytes", diff --git a/p2p/peer.go b/p2p/peer.go index e20271149..f1db591ce 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -5,10 +5,7 @@ import ( "net" "time" - "github.com/pkg/errors" - - crypto "github.com/tendermint/go-crypto" - wire "github.com/tendermint/go-wire" + "github.com/tendermint/go-crypto" cmn "github.com/tendermint/tmlibs/common" "github.com/tendermint/tmlibs/log" @@ -25,8 +22,8 @@ type Peer interface { NodeInfo() NodeInfo // peer's info Status() tmconn.ConnectionStatus - Send(byte, interface{}) bool - TrySend(byte, interface{}) bool + Send(byte, []byte) bool + TrySend(byte, []byte) bool Set(string, interface{}) Get(string) interface{} @@ -114,13 +111,13 @@ func newOutboundPeerConn(addr *NetAddress, config *PeerConfig, persistent bool, conn, err := dial(addr, config) if err != nil { - return pc, errors.Wrap(err, "Error creating peer") + return pc, cmn.ErrorWrap(err, "Error creating peer") } pc, err = newPeerConn(conn, config, true, persistent, ourNodePrivKey) if err != nil { if err2 := conn.Close(); err2 != nil { - return pc, errors.Wrap(err, err2.Error()) + return pc, cmn.ErrorWrap(err, err2.Error()) } return pc, err } @@ -128,7 +125,7 @@ func newOutboundPeerConn(addr *NetAddress, config *PeerConfig, persistent bool, // ensure dialed ID matches connection ID if config.AuthEnc && addr.ID != pc.ID() { if err2 := conn.Close(); err2 != nil { - return pc, errors.Wrap(err, err2.Error()) + return pc, cmn.ErrorWrap(err, err2.Error()) } return pc, ErrSwitchAuthenticationFailure{addr, pc.ID()} } @@ -157,13 +154,13 @@ func newPeerConn(rawConn net.Conn, if config.AuthEnc { // Set deadline for secret handshake if err := conn.SetDeadline(time.Now().Add(config.HandshakeTimeout * time.Second)); err != nil { - return pc, errors.Wrap(err, "Error setting deadline while encrypting connection") + return pc, cmn.ErrorWrap(err, "Error setting deadline while encrypting connection") } // Encrypt connection conn, err = tmconn.MakeSecretConnection(conn, ourNodePrivKey) if err != nil { - return pc, errors.Wrap(err, "Error creating peer") + return pc, cmn.ErrorWrap(err, "Error creating peer") } } @@ -228,9 +225,9 @@ func (p *peer) Status() tmconn.ConnectionStatus { return p.mconn.Status() } -// Send msg to the channel identified by chID byte. Returns false if the send -// queue is full after timeout, specified by MConnection. -func (p *peer) Send(chID byte, msg interface{}) bool { +// Send msg bytes to the channel identified by chID byte. Returns false if the +// send queue is full after timeout, specified by MConnection. +func (p *peer) Send(chID byte, msgBytes []byte) bool { if !p.IsRunning() { // see Switch#Broadcast, where we fetch the list of peers and loop over // them - while we're looping, one peer may be removed and stopped. @@ -238,18 +235,18 @@ func (p *peer) Send(chID byte, msg interface{}) bool { } else if !p.hasChannel(chID) { return false } - return p.mconn.Send(chID, msg) + return p.mconn.Send(chID, msgBytes) } -// TrySend msg to the channel identified by chID byte. Immediately returns +// TrySend msg bytes to the channel identified by chID byte. Immediately returns // false if the send queue is full. -func (p *peer) TrySend(chID byte, msg interface{}) bool { +func (p *peer) TrySend(chID byte, msgBytes []byte) bool { if !p.IsRunning() { return false } else if !p.hasChannel(chID) { return false } - return p.mconn.TrySend(chID, msg) + return p.mconn.TrySend(chID, msgBytes) } // Get the data for a given key. @@ -290,30 +287,26 @@ func (pc *peerConn) CloseConn() { func (pc *peerConn) HandshakeTimeout(ourNodeInfo NodeInfo, timeout time.Duration) (peerNodeInfo NodeInfo, err error) { // Set deadline for handshake so we don't block forever on conn.ReadFull if err := pc.conn.SetDeadline(time.Now().Add(timeout)); err != nil { - return peerNodeInfo, errors.Wrap(err, "Error setting deadline") + return peerNodeInfo, cmn.ErrorWrap(err, "Error setting deadline") } - var err1 error - var err2 error - cmn.Parallel( - func() { - var n int - wire.WriteBinary(&ourNodeInfo, pc.conn, &n, &err1) + var trs, _ = cmn.Parallel( + func(_ int) (val interface{}, err error, abort bool) { + _, err = cdc.MarshalBinaryWriter(pc.conn, ourNodeInfo) + return }, - func() { - var n int - wire.ReadBinary(&peerNodeInfo, pc.conn, MaxNodeInfoSize(), &n, &err2) - }) - if err1 != nil { - return peerNodeInfo, errors.Wrap(err1, "Error during handshake/write") - } - if err2 != nil { - return peerNodeInfo, errors.Wrap(err2, "Error during handshake/read") + func(_ int) (val interface{}, err error, abort bool) { + _, err = cdc.UnmarshalBinaryReader(pc.conn, &peerNodeInfo, int64(MaxNodeInfoSize())) + return + }, + ) + if err := trs.FirstError(); err != nil { + return peerNodeInfo, cmn.ErrorWrap(err, "Error during handshake") } // Remove deadline if err := pc.conn.SetDeadline(time.Time{}); err != nil { - return peerNodeInfo, errors.Wrap(err, "Error removing deadline") + return peerNodeInfo, cmn.ErrorWrap(err, "Error removing deadline") } return peerNodeInfo, nil diff --git a/p2p/peer_set_test.go b/p2p/peer_set_test.go index 174e73c9c..699b1ac46 100644 --- a/p2p/peer_set_test.go +++ b/p2p/peer_set_test.go @@ -13,7 +13,7 @@ import ( // Returns an empty kvstore peer func randPeer() *peer { - pubKey := crypto.GenPrivKeyEd25519().Wrap().PubKey() + pubKey := crypto.GenPrivKeyEd25519().PubKey() return &peer{ nodeInfo: NodeInfo{ ListenAddr: cmn.Fmt("%v.%v.%v.%v:46656", rand.Int()%256, rand.Int()%256, rand.Int()%256, rand.Int()%256), diff --git a/p2p/peer_test.go b/p2p/peer_test.go index 45e2b0457..afb34f470 100644 --- a/p2p/peer_test.go +++ b/p2p/peer_test.go @@ -20,7 +20,7 @@ func TestPeerBasic(t *testing.T) { assert, require := assert.New(t), require.New(t) // simulate remote peer - rp := &remotePeer{PrivKey: crypto.GenPrivKeyEd25519().Wrap(), Config: DefaultPeerConfig()} + rp := &remotePeer{PrivKey: crypto.GenPrivKeyEd25519(), Config: DefaultPeerConfig()} rp.Start() defer rp.Stop() @@ -47,7 +47,7 @@ func TestPeerWithoutAuthEnc(t *testing.T) { config.AuthEnc = false // simulate remote peer - rp := &remotePeer{PrivKey: crypto.GenPrivKeyEd25519().Wrap(), Config: config} + rp := &remotePeer{PrivKey: crypto.GenPrivKeyEd25519(), Config: config} rp.Start() defer rp.Stop() @@ -68,7 +68,7 @@ func TestPeerSend(t *testing.T) { config.AuthEnc = false // simulate remote peer - rp := &remotePeer{PrivKey: crypto.GenPrivKeyEd25519().Wrap(), Config: config} + rp := &remotePeer{PrivKey: crypto.GenPrivKeyEd25519(), Config: config} rp.Start() defer rp.Stop() @@ -81,7 +81,7 @@ func TestPeerSend(t *testing.T) { defer p.Stop() assert.True(p.CanSend(testCh)) - assert.True(p.Send(testCh, "Asylum")) + assert.True(p.Send(testCh, []byte("Asylum"))) } func createOutboundPeerAndPerformHandshake(addr *NetAddress, config *PeerConfig) (*peer, error) { @@ -89,7 +89,7 @@ func createOutboundPeerAndPerformHandshake(addr *NetAddress, config *PeerConfig) {ID: testCh, Priority: 1}, } reactorsByCh := map[byte]Reactor{testCh: NewTestReactor(chDescs, true)} - pk := crypto.GenPrivKeyEd25519().Wrap() + pk := crypto.GenPrivKeyEd25519() pc, err := newOutboundPeerConn(addr, config, false, pk) if err != nil { return nil, err diff --git a/p2p/pex/pex_reactor.go b/p2p/pex/pex_reactor.go index 441010aa7..43268a909 100644 --- a/p2p/pex/pex_reactor.go +++ b/p2p/pex/pex_reactor.go @@ -1,7 +1,6 @@ package pex import ( - "bytes" "fmt" "math/rand" "reflect" @@ -9,8 +8,7 @@ import ( "sync" "time" - "github.com/pkg/errors" - wire "github.com/tendermint/go-wire" + "github.com/tendermint/go-amino" cmn "github.com/tendermint/tmlibs/common" "github.com/tendermint/tendermint/p2p" @@ -165,7 +163,7 @@ func (r *PEXReactor) RemovePeer(p Peer, reason interface{}) { // Receive implements Reactor by handling incoming PEX messages. func (r *PEXReactor) Receive(chID byte, src Peer, msgBytes []byte) { - _, msg, err := DecodeMessage(msgBytes) + msg, err := DecodeMessage(msgBytes) if err != nil { r.Logger.Error("Error decoding message", "src", src, "chId", chID, "msg", msg, "err", err, "bytes", msgBytes) r.Switch.StopPeerForError(src, err) @@ -235,7 +233,7 @@ func (r *PEXReactor) RequestAddrs(p Peer) { return } r.requestsSent.Set(id, struct{}{}) - p.Send(PexChannel, struct{ PexMessage }{&pexRequestMessage{}}) + p.Send(PexChannel, cdc.MustMarshalBinary(&pexRequestMessage{})) } // ReceiveAddrs adds the given addrs to the addrbook if theres an open @@ -245,7 +243,7 @@ func (r *PEXReactor) ReceiveAddrs(addrs []*p2p.NetAddress, src Peer) error { id := string(src.ID()) if !r.requestsSent.Has(id) { - return errors.New("Received unsolicited pexAddrsMessage") + return cmn.NewError("Received unsolicited pexAddrsMessage") } r.requestsSent.Delete(id) @@ -261,7 +259,7 @@ func (r *PEXReactor) ReceiveAddrs(addrs []*p2p.NetAddress, src Peer) error { // SendAddrs sends addrs to the peer. func (r *PEXReactor) SendAddrs(p Peer, netAddrs []*p2p.NetAddress) { - p.Send(PexChannel, struct{ PexMessage }{&pexAddrsMessage{Addrs: netAddrs}}) + p.Send(PexChannel, cdc.MustMarshalBinary(&pexAddrsMessage{Addrs: netAddrs})) } // SetEnsurePeersPeriod sets period to ensure peers connected. @@ -583,27 +581,19 @@ func (r *PEXReactor) attemptDisconnects() { //----------------------------------------------------------------------------- // Messages -const ( - msgTypeRequest = byte(0x01) - msgTypeAddrs = byte(0x02) -) - // PexMessage is a primary type for PEX messages. Underneath, it could contain // either pexRequestMessage, or pexAddrsMessage messages. type PexMessage interface{} -var _ = wire.RegisterInterface( - struct{ PexMessage }{}, - wire.ConcreteType{&pexRequestMessage{}, msgTypeRequest}, - wire.ConcreteType{&pexAddrsMessage{}, msgTypeAddrs}, -) +func RegisterPexMessage(cdc *amino.Codec) { + cdc.RegisterInterface((*PexMessage)(nil), nil) + cdc.RegisterConcrete(&pexRequestMessage{}, "tendermint/p2p/PexRequestMessage", nil) + cdc.RegisterConcrete(&pexAddrsMessage{}, "tendermint/p2p/PexAddrsMessage", nil) +} // DecodeMessage implements interface registered above. -func DecodeMessage(bz []byte) (msgType byte, msg PexMessage, err error) { - msgType = bz[0] - n := new(int) - r := bytes.NewReader(bz) - msg = wire.ReadBinary(struct{ PexMessage }{}, r, maxPexMessageSize, n, &err).(struct{ PexMessage }).PexMessage +func DecodeMessage(bz []byte) (msg PexMessage, err error) { + err = cdc.UnmarshalBinary(bz, &msg) return } diff --git a/p2p/pex/pex_reactor_test.go b/p2p/pex/pex_reactor_test.go index f5d815037..00345758a 100644 --- a/p2p/pex/pex_reactor_test.go +++ b/p2p/pex/pex_reactor_test.go @@ -12,7 +12,6 @@ import ( "github.com/stretchr/testify/require" crypto "github.com/tendermint/go-crypto" - wire "github.com/tendermint/go-wire" cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/p2p/conn" @@ -114,12 +113,12 @@ func TestPEXReactorReceive(t *testing.T) { size := book.Size() addrs := []*p2p.NetAddress{peer.NodeInfo().NetAddress()} - msg := wire.BinaryBytes(struct{ PexMessage }{&pexAddrsMessage{Addrs: addrs}}) + msg := cdc.MustMarshalBinary(&pexAddrsMessage{Addrs: addrs}) r.Receive(PexChannel, peer, msg) assert.Equal(t, size+1, book.Size()) - msg = wire.BinaryBytes(struct{ PexMessage }{&pexRequestMessage{}}) - r.Receive(PexChannel, peer, msg) + msg = cdc.MustMarshalBinary(&pexRequestMessage{}) + r.Receive(PexChannel, peer, msg) // should not panic. } func TestPEXReactorRequestMessageAbuse(t *testing.T) { @@ -133,7 +132,7 @@ func TestPEXReactorRequestMessageAbuse(t *testing.T) { assert.True(t, sw.Peers().Has(peer.ID())) id := string(peer.ID()) - msg := wire.BinaryBytes(struct{ PexMessage }{&pexRequestMessage{}}) + msg := cdc.MustMarshalBinary(&pexRequestMessage{}) // first time creates the entry r.Receive(PexChannel, peer, msg) @@ -169,7 +168,7 @@ func TestPEXReactorAddrsMessageAbuse(t *testing.T) { assert.True(t, sw.Peers().Has(peer.ID())) addrs := []*p2p.NetAddress{peer.NodeInfo().NetAddress()} - msg := wire.BinaryBytes(struct{ PexMessage }{&pexAddrsMessage{Addrs: addrs}}) + msg := cdc.MustMarshalBinary(&pexAddrsMessage{Addrs: addrs}) // receive some addrs. should clear the request r.Receive(PexChannel, peer, msg) @@ -302,7 +301,7 @@ func newMockPeer() mockPeer { _, netAddr := p2p.CreateRoutableAddr() mp := mockPeer{ addr: netAddr, - pubKey: crypto.GenPrivKeyEd25519().Wrap().PubKey(), + pubKey: crypto.GenPrivKeyEd25519().PubKey(), } mp.BaseService = cmn.NewBaseService(nil, "MockPeer", mp) mp.Start() @@ -318,11 +317,11 @@ func (mp mockPeer) NodeInfo() p2p.NodeInfo { ListenAddr: mp.addr.DialString(), } } -func (mp mockPeer) Status() conn.ConnectionStatus { return conn.ConnectionStatus{} } -func (mp mockPeer) Send(byte, interface{}) bool { return false } -func (mp mockPeer) TrySend(byte, interface{}) bool { return false } -func (mp mockPeer) Set(string, interface{}) {} -func (mp mockPeer) Get(string) interface{} { return nil } +func (mp mockPeer) Status() conn.ConnectionStatus { return conn.ConnectionStatus{} } +func (mp mockPeer) Send(byte, []byte) bool { return false } +func (mp mockPeer) TrySend(byte, []byte) bool { return false } +func (mp mockPeer) Set(string, interface{}) {} +func (mp mockPeer) Get(string) interface{} { return nil } func assertPeersWithTimeout( t *testing.T, diff --git a/p2p/pex/wire.go b/p2p/pex/wire.go new file mode 100644 index 000000000..57fc93858 --- /dev/null +++ b/p2p/pex/wire.go @@ -0,0 +1,11 @@ +package pex + +import ( + "github.com/tendermint/go-amino" +) + +var cdc *amino.Codec = amino.NewCodec() + +func init() { + RegisterPexMessage(cdc) +} diff --git a/p2p/switch.go b/p2p/switch.go index 63deace29..4ad633b7e 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -8,8 +8,6 @@ import ( "sync" "time" - "github.com/pkg/errors" - cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/p2p/conn" cmn "github.com/tendermint/tmlibs/common" @@ -85,7 +83,7 @@ func NewSwitch(config *cfg.P2PConfig) *Switch { sw.peerConfig.MConfig.FlushThrottle = time.Duration(config.FlushThrottleTimeout) * time.Millisecond sw.peerConfig.MConfig.SendRate = config.SendRate sw.peerConfig.MConfig.RecvRate = config.RecvRate - sw.peerConfig.MConfig.MaxMsgPacketPayloadSize = config.MaxMsgPacketPayloadSize + sw.peerConfig.MConfig.MaxPacketMsgPayloadSize = config.MaxPacketMsgPayloadSize sw.peerConfig.AuthEnc = config.AuthEnc sw.BaseService = *cmn.NewBaseService(nil, "P2P Switch", sw) @@ -171,7 +169,7 @@ func (sw *Switch) OnStart() error { for _, reactor := range sw.reactors { err := reactor.Start() if err != nil { - return errors.Wrapf(err, "failed to start %v", reactor) + return cmn.ErrorWrap(err, "failed to start %v", reactor) } } // Start listeners @@ -206,18 +204,18 @@ func (sw *Switch) OnStop() { // Broadcast runs a go routine for each attempted send, which will block trying // to send for defaultSendTimeoutSeconds. Returns a channel which receives // success values for each attempted send (false if times out). Channel will be -// closed once msg send to all peers. +// closed once msg bytes are sent to all peers (or time out). // // NOTE: Broadcast uses goroutines, so order of broadcast may not be preserved. -func (sw *Switch) Broadcast(chID byte, msg interface{}) chan bool { +func (sw *Switch) Broadcast(chID byte, msgBytes []byte) chan bool { successChan := make(chan bool, len(sw.peers.List())) - sw.Logger.Debug("Broadcast", "channel", chID, "msg", msg) + sw.Logger.Debug("Broadcast", "channel", chID, "msgBytes", fmt.Sprintf("%X", msgBytes)) var wg sync.WaitGroup for _, peer := range sw.peers.List() { wg.Add(1) go func(peer Peer) { defer wg.Done() - success := peer.Send(chID, msg) + success := peer.Send(chID, msgBytes) successChan <- success }(peer) } diff --git a/p2p/switch_test.go b/p2p/switch_test.go index 06e8b642e..a6d519619 100644 --- a/p2p/switch_test.go +++ b/p2p/switch_test.go @@ -12,7 +12,6 @@ import ( "github.com/stretchr/testify/require" crypto "github.com/tendermint/go-crypto" - wire "github.com/tendermint/go-wire" "github.com/tendermint/tmlibs/log" cfg "github.com/tendermint/tendermint/config" @@ -125,9 +124,9 @@ func TestSwitches(t *testing.T) { } // Lets send some messages - ch0Msg := "channel zero" - ch1Msg := "channel foo" - ch2Msg := "channel bar" + ch0Msg := []byte("channel zero") + ch1Msg := []byte("channel foo") + ch2Msg := []byte("channel bar") s1.Broadcast(byte(0x00), ch0Msg) s1.Broadcast(byte(0x01), ch1Msg) @@ -138,15 +137,15 @@ func TestSwitches(t *testing.T) { assertMsgReceivedWithTimeout(t, ch2Msg, byte(0x02), s2.Reactor("bar").(*TestReactor), 10*time.Millisecond, 5*time.Second) } -func assertMsgReceivedWithTimeout(t *testing.T, msg string, channel byte, reactor *TestReactor, checkPeriod, timeout time.Duration) { +func assertMsgReceivedWithTimeout(t *testing.T, msgBytes []byte, channel byte, reactor *TestReactor, checkPeriod, timeout time.Duration) { ticker := time.NewTicker(checkPeriod) for { select { case <-ticker.C: msgs := reactor.getMsgs(channel) if len(msgs) > 0 { - if !bytes.Equal(msgs[0].Bytes, wire.BinaryBytes(msg)) { - t.Fatalf("Unexpected message bytes. Wanted: %X, Got: %X", wire.BinaryBytes(msg), msgs[0].Bytes) + if !bytes.Equal(msgs[0].Bytes, msgBytes) { + t.Fatalf("Unexpected message bytes. Wanted: %X, Got: %X", msgBytes, msgs[0].Bytes) } return } @@ -238,7 +237,7 @@ func TestSwitchStopsNonPersistentPeerOnError(t *testing.T) { defer sw.Stop() // simulate remote peer - rp := &remotePeer{PrivKey: crypto.GenPrivKeyEd25519().Wrap(), Config: DefaultPeerConfig()} + rp := &remotePeer{PrivKey: crypto.GenPrivKeyEd25519(), Config: DefaultPeerConfig()} rp.Start() defer rp.Stop() @@ -268,7 +267,7 @@ func TestSwitchReconnectsToPersistentPeer(t *testing.T) { defer sw.Stop() // simulate remote peer - rp := &remotePeer{PrivKey: crypto.GenPrivKeyEd25519().Wrap(), Config: DefaultPeerConfig()} + rp := &remotePeer{PrivKey: crypto.GenPrivKeyEd25519(), Config: DefaultPeerConfig()} rp.Start() defer rp.Stop() @@ -338,7 +337,7 @@ func BenchmarkSwitchBroadcast(b *testing.B) { // Send random message from foo channel to another for i := 0; i < b.N; i++ { chID := byte(i % 4) - successChan := s1.Broadcast(chID, "test data") + successChan := s1.Broadcast(chID, []byte("test data")) for s := range successChan { if s { numSuccess++ diff --git a/p2p/test_util.go b/p2p/test_util.go index 0d6427f43..5e4f3b8c2 100644 --- a/p2p/test_util.go +++ b/p2p/test_util.go @@ -24,7 +24,7 @@ func CreateRandomPeer(outbound bool) *peer { }, nodeInfo: NodeInfo{ ListenAddr: netAddr.DialString(), - PubKey: crypto.GenPrivKeyEd25519().Wrap().PubKey(), + PubKey: crypto.GenPrivKeyEd25519().PubKey(), }, mconn: &conn.MConnection{}, } @@ -131,7 +131,7 @@ func MakeSwitch(cfg *cfg.P2PConfig, i int, network, version string, initSwitch f // new switch, add reactors // TODO: let the config be passed in? nodeKey := &NodeKey{ - PrivKey: crypto.GenPrivKeyEd25519().Wrap(), + PrivKey: crypto.GenPrivKeyEd25519(), } sw := NewSwitch(cfg) sw.SetLogger(log.TestingLogger()) diff --git a/p2p/upnp/probe.go b/p2p/upnp/probe.go index d2338b95e..55479415f 100644 --- a/p2p/upnp/probe.go +++ b/p2p/upnp/probe.go @@ -5,8 +5,6 @@ import ( "net" "time" - "github.com/pkg/errors" - cmn "github.com/tendermint/tmlibs/common" "github.com/tendermint/tmlibs/log" ) @@ -19,26 +17,26 @@ type UPNPCapabilities struct { func makeUPNPListener(intPort int, extPort int, logger log.Logger) (NAT, net.Listener, net.IP, error) { nat, err := Discover() if err != nil { - return nil, nil, nil, errors.Errorf("NAT upnp could not be discovered: %v", err) + return nil, nil, nil, fmt.Errorf("NAT upnp could not be discovered: %v", err) } logger.Info(cmn.Fmt("ourIP: %v", nat.(*upnpNAT).ourIP)) ext, err := nat.GetExternalAddress() if err != nil { - return nat, nil, nil, errors.Errorf("External address error: %v", err) + return nat, nil, nil, fmt.Errorf("External address error: %v", err) } logger.Info(cmn.Fmt("External address: %v", ext)) port, err := nat.AddPortMapping("tcp", extPort, intPort, "Tendermint UPnP Probe", 0) if err != nil { - return nat, nil, ext, errors.Errorf("Port mapping error: %v", err) + return nat, nil, ext, fmt.Errorf("Port mapping error: %v", err) } logger.Info(cmn.Fmt("Port mapping mapped: %v", port)) // also run the listener, open for all remote addresses. listener, err := net.Listen("tcp", fmt.Sprintf(":%v", intPort)) if err != nil { - return nat, nil, ext, errors.Errorf("Error establishing listener: %v", err) + return nat, nil, ext, fmt.Errorf("Error establishing listener: %v", err) } return nat, listener, ext, nil } diff --git a/p2p/wire.go b/p2p/wire.go new file mode 100644 index 000000000..a90ac851c --- /dev/null +++ b/p2p/wire.go @@ -0,0 +1,12 @@ +package p2p + +import ( + "github.com/tendermint/go-amino" + "github.com/tendermint/go-crypto" +) + +var cdc = amino.NewCodec() + +func init() { + crypto.RegisterAmino(cdc) +} From 4085c7249659888615d86ab4df3c4f8816f94473 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Tue, 27 Mar 2018 12:32:42 +0200 Subject: [PATCH 008/143] sort /tx_search results by height by default Refs #1366 --- state/txindex/kv/kv.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/state/txindex/kv/kv.go b/state/txindex/kv/kv.go index 24e982725..510074b50 100644 --- a/state/txindex/kv/kv.go +++ b/state/txindex/kv/kv.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/hex" "fmt" + "sort" "strconv" "strings" "time" @@ -202,6 +203,11 @@ func (txi *TxIndex) Search(q *query.Query) ([]*types.TxResult, error) { i++ } + // sort by height by default + sort.Slice(results, func(i, j int) bool { + return results[i].Height < results[j].Height + }) + return results, nil } From 2b63f57b4c4506ff4bc799dcc4d346bb2b18ef05 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Tue, 27 Mar 2018 12:33:15 +0200 Subject: [PATCH 009/143] fix tx_indexer's matchRange before we're using IteratePrefix, which is wrong because we want full range, not just "account.number=1". --- state/txindex/kv/kv.go | 132 +++++++++++++++++++++++------------- state/txindex/kv/kv_test.go | 37 ++++++++-- 2 files changed, 116 insertions(+), 53 deletions(-) diff --git a/state/txindex/kv/kv.go b/state/txindex/kv/kv.go index 510074b50..1f544e58b 100644 --- a/state/txindex/kv/kv.go +++ b/state/txindex/kv/kv.go @@ -171,10 +171,10 @@ func (txi *TxIndex) Search(q *query.Query) ([]*types.TxResult, error) { for _, r := range ranges { if !hashesInitialized { - hashes = txi.matchRange(r, startKeyForRange(r, height)) + hashes = txi.matchRange(r, []byte(r.key)) hashesInitialized = true } else { - hashes = intersect(hashes, txi.matchRange(r, startKeyForRange(r, height))) + hashes = intersect(hashes, txi.matchRange(r, []byte(r.key))) } } } @@ -242,6 +242,52 @@ type queryRange struct { includeUpperBound bool } +func (r queryRange) lowerBoundValue() interface{} { + if r.lowerBound == nil { + return nil + } + + if r.includeLowerBound { + return r.lowerBound + } else { + switch t := r.lowerBound.(type) { + case int64: + return t + 1 + case time.Time: + return t.Unix() + 1 + default: + panic("not implemented") + } + } +} + +func (r queryRange) AnyBound() interface{} { + if r.lowerBound != nil { + return r.lowerBound + } else { + return r.upperBound + } +} + +func (r queryRange) upperBoundValue() interface{} { + if r.upperBound == nil { + return nil + } + + if r.includeUpperBound { + return r.upperBound + } else { + switch t := r.upperBound.(type) { + case int64: + return t - 1 + case time.Time: + return t.Unix() - 1 + default: + panic("not implemented") + } + } +} + func lookForRanges(conditions []query.Condition) (ranges queryRanges, indexes []int) { ranges = make(queryRanges) for i, c := range conditions { @@ -305,34 +351,49 @@ func (txi *TxIndex) match(c query.Condition, startKey []byte) (hashes [][]byte) return } -func (txi *TxIndex) matchRange(r queryRange, startKey []byte) (hashes [][]byte) { - it := dbm.IteratePrefix(txi.store, startKey) +func (txi *TxIndex) matchRange(r queryRange, prefix []byte) (hashes [][]byte) { + // create a map to prevent duplicates + hashesMap := make(map[string][]byte) + + lowerBound := r.lowerBoundValue() + upperBound := r.upperBoundValue() + + it := dbm.IteratePrefix(txi.store, prefix) defer it.Close() LOOP: for ; it.Valid(); it.Next() { if !isTagKey(it.Key()) { continue } - if r.upperBound != nil { - // no other way to stop iterator other than checking for upperBound - switch (r.upperBound).(type) { - case int64: - v, err := strconv.ParseInt(extractValueFromKey(it.Key()), 10, 64) - if err == nil && v == r.upperBound { - if r.includeUpperBound { - hashes = append(hashes, it.Value()) - } - break LOOP - } - // XXX: passing time in a ABCI Tags is not yet implemented - // case time.Time: - // v := strconv.ParseInt(extractValueFromKey(it.Key()), 10, 64) - // if v == r.upperBound { - // break - // } + switch r.AnyBound().(type) { + case int64: + v, err := strconv.ParseInt(extractValueFromKey(it.Key()), 10, 64) + if err != nil { + continue LOOP } + include := true + if lowerBound != nil && v < lowerBound.(int64) { + include = false + } + if upperBound != nil && v > upperBound.(int64) { + include = false + } + if include { + hashesMap[fmt.Sprintf("%X", it.Value())] = it.Value() + } + // XXX: passing time in a ABCI Tags is not yet implemented + // case time.Time: + // v := strconv.ParseInt(extractValueFromKey(it.Key()), 10, 64) + // if v == r.upperBound { + // break + // } } - hashes = append(hashes, it.Value()) + } + hashes = make([][]byte, len(hashesMap)) + i := 0 + for _, h := range hashesMap { + hashes[i] = h + i++ } return } @@ -350,33 +411,6 @@ func startKey(c query.Condition, height int64) []byte { return []byte(key) } -func startKeyForRange(r queryRange, height int64) []byte { - if r.lowerBound == nil { - return []byte(r.key) - } - - var lowerBound interface{} - if r.includeLowerBound { - lowerBound = r.lowerBound - } else { - switch t := r.lowerBound.(type) { - case int64: - lowerBound = t + 1 - case time.Time: - lowerBound = t.Unix() + 1 - default: - panic("not implemented") - } - } - var key string - if height > 0 { - key = fmt.Sprintf("%s/%v/%d", r.key, lowerBound, height) - } else { - key = fmt.Sprintf("%s/%v", r.key, lowerBound) - } - return []byte(key) -} - func isTagKey(key []byte) bool { return strings.Count(string(key), tagKeySeparator) == 3 } diff --git a/state/txindex/kv/kv_test.go b/state/txindex/kv/kv_test.go index 7ffc36e50..810da23bd 100644 --- a/state/txindex/kv/kv_test.go +++ b/state/txindex/kv/kv_test.go @@ -20,7 +20,7 @@ func TestTxIndex(t *testing.T) { indexer := NewTxIndex(db.NewMemDB()) tx := types.Tx("HELLO WORLD") - txResult := &types.TxResult{1, 0, tx, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeTypeOK, Log: "", Tags: []cmn.KVPair{}, Fee: cmn.KI64Pair{Key: []uint8{}, Value: 0}}} + txResult := &types.TxResult{Height: 1, Index: 0, Tx: tx, Result: abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeTypeOK, Log: "", Tags: []cmn.KVPair{}, Fee: cmn.KI64Pair{Key: []uint8{}, Value: 0}}} hash := tx.Hash() batch := txindex.NewBatch(1) @@ -35,7 +35,7 @@ func TestTxIndex(t *testing.T) { assert.Equal(t, txResult, loadedTxResult) tx2 := types.Tx("BYE BYE WORLD") - txResult2 := &types.TxResult{1, 0, tx2, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeTypeOK, Log: "", Tags: []cmn.KVPair{}, Fee: cmn.KI64Pair{Key: []uint8{}, Value: 0}}} + txResult2 := &types.TxResult{Height: 1, Index: 0, Tx: tx2, Result: abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeTypeOK, Log: "", Tags: []cmn.KVPair{}, Fee: cmn.KI64Pair{Key: []uint8{}, Value: 0}}} hash2 := tx2.Hash() err = indexer.Index(txResult2) @@ -122,6 +122,35 @@ func TestTxSearchOneTxWithMultipleSameTagsButDifferentValues(t *testing.T) { assert.Equal(t, []*types.TxResult{txResult}, results) } +func TestTxSearchMultipleTxs(t *testing.T) { + allowedTags := []string{"account.number"} + indexer := NewTxIndex(db.NewMemDB(), IndexTags(allowedTags)) + + // indexed first, but bigger height (to test the order of transactions) + txResult := txResultWithTags([]cmn.KVPair{ + {Key: []byte("account.number"), Value: []byte("1")}, + }) + txResult.Tx = types.Tx("Bob's account") + txResult.Height = 2 + err := indexer.Index(txResult) + require.NoError(t, err) + + // indexed second, but smaller height (to test the order of transactions) + txResult2 := txResultWithTags([]cmn.KVPair{ + {Key: []byte("account.number"), Value: []byte("2")}, + }) + txResult2.Tx = types.Tx("Alice's account") + txResult2.Height = 1 + err = indexer.Index(txResult2) + require.NoError(t, err) + + results, err := indexer.Search(query.MustParse("account.number >= 1")) + assert.NoError(t, err) + + require.Len(t, results, 2) + assert.Equal(t, []*types.TxResult{txResult2, txResult}, results) +} + func TestIndexAllTags(t *testing.T) { indexer := NewTxIndex(db.NewMemDB(), IndexAllTags()) @@ -146,12 +175,12 @@ func TestIndexAllTags(t *testing.T) { func txResultWithTags(tags []cmn.KVPair) *types.TxResult { tx := types.Tx("HELLO WORLD") - return &types.TxResult{1, 0, tx, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeTypeOK, Log: "", Tags: tags, Fee: cmn.KI64Pair{Key: []uint8{}, Value: 0}}} + return &types.TxResult{Height: 1, Index: 0, Tx: tx, Result: abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeTypeOK, Log: "", Tags: tags, Fee: cmn.KI64Pair{Key: []uint8{}, Value: 0}}} } func benchmarkTxIndex(txsCount int, b *testing.B) { tx := types.Tx("HELLO WORLD") - txResult := &types.TxResult{1, 0, tx, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeTypeOK, Log: "", Tags: []cmn.KVPair{}, Fee: cmn.KI64Pair{Key: []uint8{}, Value: 0}}} + txResult := &types.TxResult{Height: 1, Index: 0, Tx: tx, Result: abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeTypeOK, Log: "", Tags: []cmn.KVPair{}, Fee: cmn.KI64Pair{Key: []uint8{}, Value: 0}}} dir, err := ioutil.TempDir("", "tx_index_db") if err != nil { From 2d857c4b1bf1cd09bf6631aae6945984c392eac8 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 28 Mar 2018 15:44:58 +0200 Subject: [PATCH 010/143] add hash field to ResultTx (/tx and /tx_search endpoints) (#1374) Refs #1367 --- CHANGELOG.md | 5 +++++ rpc/client/rpc_test.go | 22 +++++++++++----------- rpc/core/tx.go | 10 ++++++++-- rpc/core/types/responses.go | 1 + 4 files changed, 25 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aea9b5450..47b39b23a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,11 @@ BUG FIXES: - Graceful handling/recovery for apps that have non-determinism or fail to halt - Graceful handling/recovery for violations of safety, or liveness +## 0.17.2 (TBD) + +IMPROVEMENTS: +- [rpc] `/tx` and `/tx_search` responses now include the transaction hash + ## 0.17.1 (March 27th, 2018) BUG FIXES: diff --git a/rpc/client/rpc_test.go b/rpc/client/rpc_test.go index f23ba6160..c5c5822f9 100644 --- a/rpc/client/rpc_test.go +++ b/rpc/client/rpc_test.go @@ -253,13 +253,11 @@ func TestBroadcastTxCommit(t *testing.T) { } func TestTx(t *testing.T) { - assert, require := assert.New(t), require.New(t) - // first we broadcast a tx c := getHTTPClient() _, _, tx := MakeTxKV() bres, err := c.BroadcastTxCommit(tx) - require.Nil(err, "%+v", err) + require.Nil(t, err, "%+v", err) txHeight := bres.Height txHash := bres.Hash @@ -289,18 +287,19 @@ func TestTx(t *testing.T) { ptx, err := c.Tx(tc.hash, tc.prove) if !tc.valid { - require.NotNil(err) + require.NotNil(t, err) } else { - require.Nil(err, "%+v", err) - assert.EqualValues(txHeight, ptx.Height) - assert.EqualValues(tx, ptx.Tx) - assert.Zero(ptx.Index) - assert.True(ptx.TxResult.IsOK()) + require.Nil(t, err, "%+v", err) + assert.EqualValues(t, txHeight, ptx.Height) + assert.EqualValues(t, tx, ptx.Tx) + assert.Zero(t, ptx.Index) + assert.True(t, ptx.TxResult.IsOK()) + assert.EqualValues(t, txHash, ptx.Hash) // time to verify the proof proof := ptx.Proof - if tc.prove && assert.EqualValues(tx, proof.Data) { - assert.True(proof.Proof.Verify(proof.Index, proof.Total, txHash, proof.RootHash)) + if tc.prove && assert.EqualValues(t, tx, proof.Data) { + assert.True(t, proof.Proof.Verify(proof.Index, proof.Total, txHash, proof.RootHash)) } } } @@ -333,6 +332,7 @@ func TestTxSearch(t *testing.T) { assert.EqualValues(t, tx, ptx.Tx) assert.Zero(t, ptx.Index) assert.True(t, ptx.TxResult.IsOK()) + assert.EqualValues(t, txHash, ptx.Hash) // time to verify the proof proof := ptx.Proof diff --git a/rpc/core/tx.go b/rpc/core/tx.go index f592326b1..18120ded1 100644 --- a/rpc/core/tx.go +++ b/rpc/core/tx.go @@ -44,7 +44,8 @@ import ( // "code": 0 // }, // "index": 0, -// "height": 52 +// "height": 52, +// "hash": "2B8EC32BA2579B3B8606E42C06DE2F7AFA2556EF" // }, // "id": "", // "jsonrpc": "2.0" @@ -67,6 +68,7 @@ import ( // - `tx_result`: the `abci.Result` object // - `index`: `int` - index of the transaction // - `height`: `int` - height of the block where this transaction was in +// - `hash`: `[]byte` - hash of the transaction func Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { // if index is disabled, return error @@ -93,6 +95,7 @@ func Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { } return &ctypes.ResultTx{ + Hash: hash, Height: height, Index: uint32(index), TxResult: r.Result, @@ -137,7 +140,8 @@ func Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { // "tx": "mvZHHa7HhZ4aRT0xMDA=", // "tx_result": {}, // "index": 31, -// "height": 12 +// "height": 12, +// "hash": "2B8EC32BA2579B3B8606E42C06DE2F7AFA2556EF" // } // ], // "id": "", @@ -161,6 +165,7 @@ func Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { // - `tx_result`: the `abci.Result` object // - `index`: `int` - index of the transaction // - `height`: `int` - height of the block where this transaction was in +// - `hash`: `[]byte` - hash of the transaction func TxSearch(query string, prove bool) ([]*ctypes.ResultTx, error) { // if index is disabled, return error if _, ok := txIndexer.(*null.TxIndex); ok { @@ -191,6 +196,7 @@ func TxSearch(query string, prove bool) ([]*ctypes.ResultTx, error) { } apiResults[i] = &ctypes.ResultTx{ + Hash: r.Tx.Hash(), Height: height, Index: index, TxResult: r.Result, diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index 7567fe650..5b49e1af6 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -123,6 +123,7 @@ type ResultBroadcastTxCommit struct { } type ResultTx struct { + Hash cmn.HexBytes `json:"hash"` Height int64 `json:"height"` Index uint32 `json:"index"` TxResult abci.ResponseDeliverTx `json:"tx_result"` From 2fa7af46148253e6697a0708a1f76c43d268a357 Mon Sep 17 00:00:00 2001 From: Vladislav Dmitriyev Date: Wed, 28 Mar 2018 16:59:09 +0300 Subject: [PATCH 011/143] [lite] fixed listen address (#1384) --- cmd/tendermint/commands/lite.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/tendermint/commands/lite.go b/cmd/tendermint/commands/lite.go index 5ea20a785..6866e4d6c 100644 --- a/cmd/tendermint/commands/lite.go +++ b/cmd/tendermint/commands/lite.go @@ -34,14 +34,14 @@ var ( ) func init() { - LiteCmd.Flags().StringVar(&listenAddr, "laddr", ":8888", "Serve the proxy on the given port") + LiteCmd.Flags().StringVar(&listenAddr, "laddr", "tcp://localhost:8888", "Serve the proxy on the given address") LiteCmd.Flags().StringVar(&nodeAddr, "node", "tcp://localhost:46657", "Connect to a Tendermint node at this address") LiteCmd.Flags().StringVar(&chainID, "chain-id", "tendermint", "Specify the Tendermint chain ID") LiteCmd.Flags().StringVar(&home, "home-dir", ".tendermint-lite", "Specify the home directory") } func ensureAddrHasSchemeOrDefaultToTCP(addr string) (string, error) { - u, err := url.Parse(nodeAddr) + u, err := url.Parse(addr) if err != nil { return "", err } From 49986b05bc2344d38b15c1db82a2c5c31ab89ca3 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Tue, 27 Mar 2018 16:11:55 +0200 Subject: [PATCH 012/143] update tmlibs Refs #1376 --- Gopkg.lock | 6 +++--- Gopkg.toml | 6 ++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 894d98ead..09dab7a5b 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -268,6 +268,7 @@ version = "v0.7.3" [[projects]] + branch = "develop" name = "github.com/tendermint/tmlibs" packages = [ "autofile", @@ -283,8 +284,7 @@ "pubsub/query", "test" ] - revision = "24da7009c3d8c019b40ba4287495749e3160caca" - version = "v0.7.1" + revision = "0f2811441f4cf44b414df16ceae3c0931c74662e" [[projects]] branch = "master" @@ -381,6 +381,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "4dca5dbd2d280d093d7c8fc423606ab86d6ad1b241b076a7716c2093b5a09231" + inputs-digest = "88f227f33e785669706f12e131610a86fd3c00f5dfd6f22c62d46e0688049726" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 098d5ea9b..8de0de9c2 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -82,9 +82,11 @@ source = "github.com/tendermint/go-amino" version = "~0.7.3" -[[constraint]] +[[override]] +# [[constraint]] name = "github.com/tendermint/tmlibs" - version = "~0.7.1" + # version = "~0.7.1" + branch = "develop" [[constraint]] name = "google.golang.org/grpc" From 22949e6dfd39180be97ca7f17d8df0c370ea884f Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Tue, 27 Mar 2018 17:10:21 +0200 Subject: [PATCH 013/143] new tmlibs Parallel implementation --- p2p/conn/connection.go | 4 +- p2p/conn/secret_connection.go | 87 ++++++++++++++++++------------ p2p/conn/secret_connection_test.go | 76 +++++++++++++++++--------- p2p/peer.go | 24 ++++----- 4 files changed, 117 insertions(+), 74 deletions(-) diff --git a/p2p/conn/connection.go b/p2p/conn/connection.go index 9acaf6175..e4ae4df80 100644 --- a/p2p/conn/connection.go +++ b/p2p/conn/connection.go @@ -7,7 +7,6 @@ import ( "io" "math" "net" - "runtime/debug" "sync/atomic" "time" @@ -230,8 +229,7 @@ func (c *MConnection) flush() { // Catch panics, usually caused by remote disconnects. func (c *MConnection) _recover() { if r := recover(); r != nil { - stack := debug.Stack() - err := cmn.StackError{r, stack} + err := cmn.ErrorWrap(r, "recovered from panic") c.stopForError(err) } } diff --git a/p2p/conn/secret_connection.go b/p2p/conn/secret_connection.go index aa6db05bb..3495853e0 100644 --- a/p2p/conn/secret_connection.go +++ b/p2p/conn/secret_connection.go @@ -20,8 +20,8 @@ import ( "golang.org/x/crypto/nacl/secretbox" "golang.org/x/crypto/ripemd160" - "github.com/tendermint/go-crypto" - "github.com/tendermint/go-wire" + crypto "github.com/tendermint/go-crypto" + wire "github.com/tendermint/go-wire" cmn "github.com/tendermint/tmlibs/common" ) @@ -200,26 +200,36 @@ func genEphKeys() (ephPub, ephPriv *[32]byte) { } func shareEphPubKey(conn io.ReadWriteCloser, locEphPub *[32]byte) (remEphPub *[32]byte, err error) { - var err1, err2 error - - cmn.Parallel( - func() { - _, err1 = conn.Write(locEphPub[:]) + // Send our pubkey and receive theirs in tandem. + var trs, _ = cmn.Parallel( + func(_ int) (val interface{}, err error, abort bool) { + var _, err1 = conn.Write(locEphPub[:]) + if err1 != nil { + return nil, err1, true // abort + } else { + return nil, nil, false + } }, - func() { - remEphPub = new([32]byte) - _, err2 = io.ReadFull(conn, remEphPub[:]) + func(_ int) (val interface{}, err error, abort bool) { + var _remEphPub [32]byte + var _, err2 = io.ReadFull(conn, _remEphPub[:]) + if err2 != nil { + return nil, err2, true // abort + } else { + return _remEphPub, nil, false + } }, ) - if err1 != nil { - return nil, err1 - } - if err2 != nil { - return nil, err2 + // If error: + if trs.FirstError() != nil { + err = trs.FirstError() + return } - return remEphPub, nil + // Otherwise: + var _remEphPub = trs.FirstValue().([32]byte) + return &_remEphPub, nil } func computeSharedSecret(remPubKey, locPrivKey *[32]byte) (shrSecret *[32]byte) { @@ -268,33 +278,42 @@ type authSigMessage struct { Sig crypto.Signature } -func shareAuthSignature(sc *SecretConnection, pubKey crypto.PubKey, signature crypto.Signature) (*authSigMessage, error) { - var recvMsg authSigMessage - var err1, err2 error - - cmn.Parallel( - func() { +func shareAuthSignature(sc *SecretConnection, pubKey crypto.PubKey, signature crypto.Signature) (recvMsg *authSigMessage, err error) { + // Send our info and receive theirs in tandem. + var trs, _ = cmn.Parallel( + func(_ int) (val interface{}, err error, abort bool) { msgBytes := wire.BinaryBytes(authSigMessage{pubKey.Wrap(), signature.Wrap()}) - _, err1 = sc.Write(msgBytes) + var _, err1 = sc.Write(msgBytes) + if err1 != nil { + return nil, err1, true // abort + } else { + return nil, nil, false + } }, - func() { + func(_ int) (val interface{}, err error, abort bool) { readBuffer := make([]byte, authSigMsgSize) - _, err2 = io.ReadFull(sc, readBuffer) + var _, err2 = io.ReadFull(sc, readBuffer) if err2 != nil { - return + return nil, err2, true // abort } n := int(0) // not used. - recvMsg = wire.ReadBinary(authSigMessage{}, bytes.NewBuffer(readBuffer), authSigMsgSize, &n, &err2).(authSigMessage) - }) + var _recvMsg = wire.ReadBinary(authSigMessage{}, bytes.NewBuffer(readBuffer), authSigMsgSize, &n, &err2).(authSigMessage) + if err2 != nil { + return nil, err2, true // abort + } else { + return _recvMsg, nil, false + } + }, + ) - if err1 != nil { - return nil, err1 - } - if err2 != nil { - return nil, err2 + // If error: + if trs.FirstError() != nil { + err = trs.FirstError() + return } - return &recvMsg, nil + var _recvMsg = trs.FirstValue().(authSigMessage) + return &_recvMsg, nil } //-------------------------------------------------------------------------------- diff --git a/p2p/conn/secret_connection_test.go b/p2p/conn/secret_connection_test.go index c81b3b285..54c453a7c 100644 --- a/p2p/conn/secret_connection_test.go +++ b/p2p/conn/secret_connection_test.go @@ -1,9 +1,12 @@ package conn import ( + "fmt" "io" "testing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" crypto "github.com/tendermint/go-crypto" cmn "github.com/tendermint/tmlibs/common" ) @@ -36,33 +39,41 @@ func makeSecretConnPair(tb testing.TB) (fooSecConn, barSecConn *SecretConnection barPrvKey := crypto.GenPrivKeyEd25519().Wrap() barPubKey := barPrvKey.PubKey() - cmn.Parallel( - func() { - var err error + var trs, ok = cmn.Parallel( + func(_ int) (val interface{}, err error, abort bool) { fooSecConn, err = MakeSecretConnection(fooConn, fooPrvKey) if err != nil { tb.Errorf("Failed to establish SecretConnection for foo: %v", err) - return + return nil, err, true } remotePubBytes := fooSecConn.RemotePubKey() if !remotePubBytes.Equals(barPubKey) { - tb.Errorf("Unexpected fooSecConn.RemotePubKey. Expected %v, got %v", + err = fmt.Errorf("Unexpected fooSecConn.RemotePubKey. Expected %v, got %v", barPubKey, fooSecConn.RemotePubKey()) + tb.Error(err) + return nil, err, false } + return nil, nil, false }, - func() { - var err error + func(_ int) (val interface{}, err error, abort bool) { barSecConn, err = MakeSecretConnection(barConn, barPrvKey) if barSecConn == nil { tb.Errorf("Failed to establish SecretConnection for bar: %v", err) - return + return nil, err, true } remotePubBytes := barSecConn.RemotePubKey() if !remotePubBytes.Equals(fooPubKey) { - tb.Errorf("Unexpected barSecConn.RemotePubKey. Expected %v, got %v", + err = fmt.Errorf("Unexpected barSecConn.RemotePubKey. Expected %v, got %v", fooPubKey, barSecConn.RemotePubKey()) + tb.Error(err) + return nil, nil, false } - }) + return nil, nil, false + }, + ) + + require.Nil(tb, trs.FirstError()) + require.True(tb, ok, "Unexpected task abortion") return } @@ -89,59 +100,76 @@ func TestSecretConnectionReadWrite(t *testing.T) { } // A helper that will run with (fooConn, fooWrites, fooReads) and vice versa - genNodeRunner := func(nodeConn kvstoreConn, nodeWrites []string, nodeReads *[]string) func() { - return func() { + genNodeRunner := func(nodeConn kvstoreConn, nodeWrites []string, nodeReads *[]string) cmn.Task { + return func(_ int) (interface{}, error, bool) { // Node handskae nodePrvKey := crypto.GenPrivKeyEd25519().Wrap() nodeSecretConn, err := MakeSecretConnection(nodeConn, nodePrvKey) if err != nil { t.Errorf("Failed to establish SecretConnection for node: %v", err) - return + return nil, err, true } - // In parallel, handle reads and writes - cmn.Parallel( - func() { + // In parallel, handle some reads and writes. + var trs, ok = cmn.Parallel( + func(_ int) (interface{}, error, bool) { // Node writes for _, nodeWrite := range nodeWrites { n, err := nodeSecretConn.Write([]byte(nodeWrite)) if err != nil { t.Errorf("Failed to write to nodeSecretConn: %v", err) - return + return nil, err, true } if n != len(nodeWrite) { - t.Errorf("Failed to write all bytes. Expected %v, wrote %v", len(nodeWrite), n) - return + err = fmt.Errorf("Failed to write all bytes. Expected %v, wrote %v", len(nodeWrite), n) + t.Error(err) + return nil, err, true } } if err := nodeConn.PipeWriter.Close(); err != nil { t.Error(err) + return nil, err, true } + return nil, nil, false }, - func() { + func(_ int) (interface{}, error, bool) { // Node reads readBuffer := make([]byte, dataMaxSize) for { n, err := nodeSecretConn.Read(readBuffer) if err == io.EOF { - return + return nil, nil, false } else if err != nil { t.Errorf("Failed to read from nodeSecretConn: %v", err) - return + return nil, err, true } *nodeReads = append(*nodeReads, string(readBuffer[:n])) } if err := nodeConn.PipeReader.Close(); err != nil { t.Error(err) + return nil, err, true } - }) + return nil, nil, false + }, + ) + assert.True(t, ok, "Unexpected task abortion") + + // If error: + if trs.FirstError() != nil { + return nil, trs.FirstError(), true + } + + // Otherwise: + return nil, nil, false } } // Run foo & bar in parallel - cmn.Parallel( + var trs, ok = cmn.Parallel( genNodeRunner(fooConn, fooWrites, &fooReads), genNodeRunner(barConn, barWrites, &barReads), ) + require.Nil(t, trs.FirstError()) + require.True(t, ok, "unexpected task abortion") // A helper to ensure that the writes and reads match. // Additionally, small writes (<= dataMaxSize) must be atomically read. diff --git a/p2p/peer.go b/p2p/peer.go index 4af6eeaae..0fa7ca034 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -293,22 +293,20 @@ func (pc *peerConn) HandshakeTimeout(ourNodeInfo NodeInfo, timeout time.Duration return peerNodeInfo, errors.Wrap(err, "Error setting deadline") } - var err1 error - var err2 error - cmn.Parallel( - func() { + var trs, _ = cmn.Parallel( + func(_ int) (val interface{}, err error, abort bool) { var n int - wire.WriteBinary(&ourNodeInfo, pc.conn, &n, &err1) + wire.WriteBinary(&ourNodeInfo, pc.conn, &n, &err) + return }, - func() { + func(_ int) (val interface{}, err error, abort bool) { var n int - wire.ReadBinary(&peerNodeInfo, pc.conn, MaxNodeInfoSize(), &n, &err2) - }) - if err1 != nil { - return peerNodeInfo, errors.Wrap(err1, "Error during handshake/write") - } - if err2 != nil { - return peerNodeInfo, errors.Wrap(err2, "Error during handshake/read") + wire.ReadBinary(&peerNodeInfo, pc.conn, MaxNodeInfoSize(), &n, &err) + return + }, + ) + if err := trs.FirstError(); err != nil { + return peerNodeInfo, errors.Wrap(err, "Error during handshake") } // Remove deadline From ead9daf1ba95a1d275845a5ab205eea99252e983 Mon Sep 17 00:00:00 2001 From: Tomoya Ishizaki Date: Thu, 29 Mar 2018 02:40:47 +0900 Subject: [PATCH 014/143] Fix code style (#1362) * cfg: Uniform style for method args and var names --- config/config.go | 150 +++++++++++++++++++++++------------------------ 1 file changed, 75 insertions(+), 75 deletions(-) diff --git a/config/config.go b/config/config.go index 6b3f87b57..0844fbd51 100644 --- a/config/config.go +++ b/config/config.go @@ -137,10 +137,6 @@ type BaseConfig struct { DBPath string `mapstructure:"db_dir"` } -func (c BaseConfig) ChainID() string { - return c.chainID -} - // DefaultBaseConfig returns a default base configuration for a Tendermint node func DefaultBaseConfig() BaseConfig { return BaseConfig{ @@ -161,12 +157,16 @@ func DefaultBaseConfig() BaseConfig { // TestBaseConfig returns a base configuration for testing a Tendermint node func TestBaseConfig() BaseConfig { - conf := DefaultBaseConfig() - conf.chainID = "tendermint_test" - conf.ProxyApp = "kvstore" - conf.FastSync = false - conf.DBBackend = "memdb" - return conf + cfg := DefaultBaseConfig() + cfg.chainID = "tendermint_test" + cfg.ProxyApp = "kvstore" + cfg.FastSync = false + cfg.DBBackend = "memdb" + return cfg +} + +func (b BaseConfig) ChainID() string { + return b.chainID } // GenesisFile returns the full path to the genesis.json file @@ -229,11 +229,11 @@ func DefaultRPCConfig() *RPCConfig { // TestRPCConfig returns a configuration for testing the RPC server func TestRPCConfig() *RPCConfig { - conf := DefaultRPCConfig() - conf.ListenAddress = "tcp://0.0.0.0:36657" - conf.GRPCListenAddress = "tcp://0.0.0.0:36658" - conf.Unsafe = true - return conf + cfg := DefaultRPCConfig() + cfg.ListenAddress = "tcp://0.0.0.0:36657" + cfg.GRPCListenAddress = "tcp://0.0.0.0:36658" + cfg.Unsafe = true + return cfg } //----------------------------------------------------------------------------- @@ -313,11 +313,11 @@ func DefaultP2PConfig() *P2PConfig { // TestP2PConfig returns a configuration for testing the peer-to-peer layer func TestP2PConfig() *P2PConfig { - conf := DefaultP2PConfig() - conf.ListenAddress = "tcp://0.0.0.0:36656" - conf.SkipUPNP = true - conf.FlushThrottleTimeout = 10 - return conf + cfg := DefaultP2PConfig() + cfg.ListenAddress = "tcp://0.0.0.0:36656" + cfg.SkipUPNP = true + cfg.FlushThrottleTimeout = 10 + return cfg } // AddrBookFile returns the full path to the address book @@ -351,9 +351,9 @@ func DefaultMempoolConfig() *MempoolConfig { // TestMempoolConfig returns a configuration for testing the Tendermint mempool func TestMempoolConfig() *MempoolConfig { - config := DefaultMempoolConfig() - config.CacheSize = 1000 - return config + cfg := DefaultMempoolConfig() + cfg.CacheSize = 1000 + return cfg } // WalDir returns the full path to the mempool's write-ahead log @@ -397,46 +397,6 @@ type ConsensusConfig struct { PeerQueryMaj23SleepDuration int `mapstructure:"peer_query_maj23_sleep_duration"` } -// WaitForTxs returns true if the consensus should wait for transactions before entering the propose step -func (cfg *ConsensusConfig) WaitForTxs() bool { - return !cfg.CreateEmptyBlocks || cfg.CreateEmptyBlocksInterval > 0 -} - -// EmptyBlocks returns the amount of time to wait before proposing an empty block or starting the propose timer if there are no txs available -func (cfg *ConsensusConfig) EmptyBlocksInterval() time.Duration { - return time.Duration(cfg.CreateEmptyBlocksInterval) * time.Second -} - -// Propose returns the amount of time to wait for a proposal -func (cfg *ConsensusConfig) Propose(round int) time.Duration { - return time.Duration(cfg.TimeoutPropose+cfg.TimeoutProposeDelta*round) * time.Millisecond -} - -// Prevote returns the amount of time to wait for straggler votes after receiving any +2/3 prevotes -func (cfg *ConsensusConfig) Prevote(round int) time.Duration { - return time.Duration(cfg.TimeoutPrevote+cfg.TimeoutPrevoteDelta*round) * time.Millisecond -} - -// Precommit returns the amount of time to wait for straggler votes after receiving any +2/3 precommits -func (cfg *ConsensusConfig) Precommit(round int) time.Duration { - return time.Duration(cfg.TimeoutPrecommit+cfg.TimeoutPrecommitDelta*round) * time.Millisecond -} - -// Commit returns the amount of time to wait for straggler votes after receiving +2/3 precommits for a single block (ie. a commit). -func (cfg *ConsensusConfig) Commit(t time.Time) time.Time { - return t.Add(time.Duration(cfg.TimeoutCommit) * time.Millisecond) -} - -// PeerGossipSleep returns the amount of time to sleep if there is nothing to send from the ConsensusReactor -func (cfg *ConsensusConfig) PeerGossipSleep() time.Duration { - return time.Duration(cfg.PeerGossipSleepDuration) * time.Millisecond -} - -// PeerQueryMaj23Sleep returns the amount of time to sleep after each VoteSetMaj23Message is sent in the ConsensusReactor -func (cfg *ConsensusConfig) PeerQueryMaj23Sleep() time.Duration { - return time.Duration(cfg.PeerQueryMaj23SleepDuration) * time.Millisecond -} - // DefaultConsensusConfig returns a default configuration for the consensus service func DefaultConsensusConfig() *ConsensusConfig { return &ConsensusConfig{ @@ -461,18 +421,58 @@ func DefaultConsensusConfig() *ConsensusConfig { // TestConsensusConfig returns a configuration for testing the consensus service func TestConsensusConfig() *ConsensusConfig { - config := DefaultConsensusConfig() - config.TimeoutPropose = 100 - config.TimeoutProposeDelta = 1 - config.TimeoutPrevote = 10 - config.TimeoutPrevoteDelta = 1 - config.TimeoutPrecommit = 10 - config.TimeoutPrecommitDelta = 1 - config.TimeoutCommit = 10 - config.SkipTimeoutCommit = true - config.PeerGossipSleepDuration = 5 - config.PeerQueryMaj23SleepDuration = 250 - return config + cfg := DefaultConsensusConfig() + cfg.TimeoutPropose = 100 + cfg.TimeoutProposeDelta = 1 + cfg.TimeoutPrevote = 10 + cfg.TimeoutPrevoteDelta = 1 + cfg.TimeoutPrecommit = 10 + cfg.TimeoutPrecommitDelta = 1 + cfg.TimeoutCommit = 10 + cfg.SkipTimeoutCommit = true + cfg.PeerGossipSleepDuration = 5 + cfg.PeerQueryMaj23SleepDuration = 250 + return cfg +} + +// WaitForTxs returns true if the consensus should wait for transactions before entering the propose step +func (c *ConsensusConfig) WaitForTxs() bool { + return !c.CreateEmptyBlocks || c.CreateEmptyBlocksInterval > 0 +} + +// EmptyBlocks returns the amount of time to wait before proposing an empty block or starting the propose timer if there are no txs available +func (c *ConsensusConfig) EmptyBlocksInterval() time.Duration { + return time.Duration(c.CreateEmptyBlocksInterval) * time.Second +} + +// Propose returns the amount of time to wait for a proposal +func (c *ConsensusConfig) Propose(round int) time.Duration { + return time.Duration(c.TimeoutPropose+c.TimeoutProposeDelta*round) * time.Millisecond +} + +// Prevote returns the amount of time to wait for straggler votes after receiving any +2/3 prevotes +func (c *ConsensusConfig) Prevote(round int) time.Duration { + return time.Duration(c.TimeoutPrevote+c.TimeoutPrevoteDelta*round) * time.Millisecond +} + +// Precommit returns the amount of time to wait for straggler votes after receiving any +2/3 precommits +func (c *ConsensusConfig) Precommit(round int) time.Duration { + return time.Duration(c.TimeoutPrecommit+c.TimeoutPrecommitDelta*round) * time.Millisecond +} + +// Commit returns the amount of time to wait for straggler votes after receiving +2/3 precommits for a single block (ie. a commit). +func (c *ConsensusConfig) Commit(t time.Time) time.Time { + return t.Add(time.Duration(c.TimeoutCommit) * time.Millisecond) +} + +// PeerGossipSleep returns the amount of time to sleep if there is nothing to send from the ConsensusReactor +func (c *ConsensusConfig) PeerGossipSleep() time.Duration { + return time.Duration(c.PeerGossipSleepDuration) * time.Millisecond +} + +// PeerQueryMaj23Sleep returns the amount of time to sleep after each VoteSetMaj23Message is sent in the ConsensusReactor +func (c *ConsensusConfig) PeerQueryMaj23Sleep() time.Duration { + return time.Duration(c.PeerQueryMaj23SleepDuration) * time.Millisecond } // WalFile returns the full path to the write-ahead log file From 575a46d9d4340a4c96fd498c8e28eb5fd5c7ee1f Mon Sep 17 00:00:00 2001 From: Alex Hernandez Date: Thu, 29 Mar 2018 17:28:29 +0800 Subject: [PATCH 015/143] fix typo on block header (#1387) --- types/block.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/types/block.go b/types/block.go index 970ca36f4..31443b1a2 100644 --- a/types/block.go +++ b/types/block.go @@ -215,7 +215,7 @@ func (h *Header) StringIndented(indent string) string { %s Data: %v %s Validators: %v %s App: %v -%s Conensus: %v +%s Consensus: %v %s Results: %v %s Evidence: %v %s}#%v`, From 34974e393281b76570f0d781a988178122a85ac2 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Sat, 31 Mar 2018 00:18:43 +0200 Subject: [PATCH 016/143] Make types use Amino; Refactor PrivValidator* to FilePV/SocketPV --- docs/architecture/adr-008-priv-validator.md | 145 +----- types/block.go | 39 +- types/block_test.go | 3 +- types/canonical_json.go | 62 ++- types/evidence.go | 98 ++-- types/evidence_test.go | 24 +- types/genesis.go | 18 +- types/genesis_test.go | 37 +- types/heartbeat.go | 8 +- types/heartbeat_test.go | 9 +- types/params.go | 21 +- types/priv_validator.go | 483 ++------------------ types/priv_validator/json.go | 197 -------- types/priv_validator/priv_validator.go | 345 ++++++++++++++ types/priv_validator/priv_validator_test.go | 151 +++--- types/priv_validator/sign_info.go | 238 ---------- types/priv_validator/socket.go | 212 ++++----- types/priv_validator/socket_tcp.go | 2 +- types/priv_validator/socket_tcp_test.go | 2 +- types/priv_validator/socket_test.go | 70 ++- types/priv_validator/unencrypted.go | 66 --- types/priv_validator/upgrade.go | 59 --- types/priv_validator/upgrade_pv/main.go | 21 - types/priv_validator/wire.go | 13 + types/priv_validator_test.go | 255 ----------- types/proposal.go | 6 +- types/proposal_test.go | 30 +- types/results.go | 6 +- types/results_test.go | 14 +- types/signable.go | 5 - types/test_util.go | 6 +- types/tx.go | 13 +- types/tx_test.go | 18 +- types/validator.go | 11 +- types/validator_set.go | 21 +- types/validator_set_test.go | 7 +- types/vote.go | 6 +- types/vote_set_test.go | 2 +- types/vote_test.go | 20 +- types/wire.go | 12 + 40 files changed, 856 insertions(+), 1899 deletions(-) delete mode 100644 types/priv_validator/json.go create mode 100644 types/priv_validator/priv_validator.go delete mode 100644 types/priv_validator/sign_info.go delete mode 100644 types/priv_validator/unencrypted.go delete mode 100644 types/priv_validator/upgrade.go delete mode 100644 types/priv_validator/upgrade_pv/main.go create mode 100644 types/priv_validator/wire.go delete mode 100644 types/priv_validator_test.go create mode 100644 types/wire.go diff --git a/docs/architecture/adr-008-priv-validator.md b/docs/architecture/adr-008-priv-validator.md index 0861e9415..4c1d87bed 100644 --- a/docs/architecture/adr-008-priv-validator.md +++ b/docs/architecture/adr-008-priv-validator.md @@ -1,128 +1,29 @@ -# ADR 008: PrivValidator +# ADR 008: SocketPV -## Context +Tendermint node's should support only two in-process PrivValidator +implementations: -The current PrivValidator is monolithic and isn't easily reuseable by alternative signers. +- FilePV uses an unencrypted private key in a "priv_validator.json" file - no + configuration required (just `tendermint init`). +- SocketPV uses a socket to send signing requests to another process - user is + responsible for starting that process themselves. -For instance, see https://github.com/tendermint/tendermint/issues/673 +The SocketPV address can be provided via flags at the command line - doing so +will cause Tendermint to ignore any "priv_validator.json" file and to listen on +the given address for incoming connections from an external priv_validator +process. It will halt any operation until at least one external process +succesfully connected. -The goal is to have a clean PrivValidator interface like: +The external priv_validator process will dial the address to connect to +Tendermint, and then Tendermint will send requests on the ensuing connection to +sign votes and proposals. Thus the external process initiates the connection, +but the Tendermint process makes all requests. In a later stage we're going to +support multiple validators for fault tolerance. To prevent double signing they +need to be synced, which is deferred to an external solution (see #1185). -``` -type PrivValidator interface { - Address() data.Bytes - PubKey() crypto.PubKey - - SignVote(chainID string, vote *types.Vote) error - SignProposal(chainID string, proposal *types.Proposal) error - SignHeartbeat(chainID string, heartbeat *types.Heartbeat) error -} -``` - -It should also be easy to re-use the LastSignedInfo logic to avoid double signing. - -## Decision - -Tendermint node's should support only two in-process PrivValidator implementations: - -- PrivValidatorUnencrypted uses an unencrypted private key in a "priv_validator.json" file - no configuration required (just `tendermint init`). -- PrivValidatorSocket uses a socket to send signing requests to another process - user is responsible for starting that process themselves. - -The PrivValidatorSocket address can be provided via flags at the command line - -doing so will cause Tendermint to ignore any "priv_validator.json" file and to listen -on the given address for incoming connections from an external priv_validator process. -It will halt any operation until at least one external process succesfully -connected. - -The external priv_validator process will dial the address to connect to Tendermint, -and then Tendermint will send requests on the ensuing connection to sign votes and proposals. -Thus the external process initiates the connection, but the Tendermint process makes all requests. -In a later stage we're going to support multiple validators for fault -tolerance. To prevent double signing they need to be synced, which is deferred -to an external solution (see #1185). - -In addition, Tendermint will provide implementations that can be run in that external process. -These include: - -- PrivValidatorEncrypted uses an encrypted private key persisted to disk - user must enter password to decrypt key when process is started. -- PrivValidatorLedger uses a Ledger Nano S to handle all signing. - -What follows are descriptions of useful types - -### Signer - -``` -type Signer interface { - Sign(msg []byte) (crypto.Signature, error) -} -``` - -Signer signs a message. It can also return an error. - -### ValidatorID - - -ValidatorID is just the Address and PubKey - -``` -type ValidatorID struct { - Address data.Bytes `json:"address"` - PubKey crypto.PubKey `json:"pub_key"` -} -``` - -### LastSignedInfo - -LastSignedInfo tracks the last thing we signed: - -``` -type LastSignedInfo struct { - Height int64 `json:"height"` - Round int `json:"round"` - Step int8 `json:"step"` - Signature crypto.Signature `json:"signature,omitempty"` // so we dont lose signatures - SignBytes data.Bytes `json:"signbytes,omitempty"` // so we dont lose signatures -} -``` - -It exposes methods for signing votes and proposals using a `Signer`. - -This allows it to easily be reused by developers implemented their own PrivValidator. - -### PrivValidatorUnencrypted - -``` -type PrivValidatorUnencrypted struct { - ID types.ValidatorID `json:"id"` - PrivKey PrivKey `json:"priv_key"` - LastSignedInfo *LastSignedInfo `json:"last_signed_info"` -} -``` - -Has the same structure as currently, but broken up into sub structs. - -Note the LastSignedInfo is mutated in place every time we sign. - -### PrivValidatorJSON - -The "priv_validator.json" file supports only the PrivValidatorUnencrypted type. - -It unmarshals into PrivValidatorJSON, which is used as the default PrivValidator type. -It wraps the PrivValidatorUnencrypted and persists it to disk after every signature. - -## Status - -Accepted. - -## Consequences - -### Positive - -- Cleaner separation of components enabling re-use. - -### Negative - -- More files - led to creation of new directory. - -### Neutral +In addition, Tendermint will provide implementations that can be run in that +external process. These include: +- FilePV will encrypt the private key, and the user must enter password to + decrypt key when process is started. +- LedgerPV uses a Ledger Nano S to handle all signing. diff --git a/types/block.go b/types/block.go index 53fc6a811..9d8bb42ad 100644 --- a/types/block.go +++ b/types/block.go @@ -7,7 +7,6 @@ import ( "strings" "time" - wire "github.com/tendermint/tendermint/wire" cmn "github.com/tendermint/tmlibs/common" "github.com/tendermint/tmlibs/merkle" "golang.org/x/crypto/ripemd160" @@ -95,7 +94,7 @@ func (b *Block) Hash() cmn.HexBytes { // MakePartSet returns a PartSet containing parts of a serialized block. // This is the form in which the block is gossipped to peers. func (b *Block) MakePartSet(partSize int) *PartSet { - bz, err := wire.MarshalBinary(b) + bz, err := cdc.MarshalBinary(b) if err != nil { panic(err) } @@ -183,19 +182,19 @@ func (h *Header) Hash() cmn.HexBytes { return nil } return merkle.SimpleHashFromMap(map[string]merkle.Hasher{ - "ChainID": wireHasher(h.ChainID), - "Height": wireHasher(h.Height), - "Time": wireHasher(h.Time), - "NumTxs": wireHasher(h.NumTxs), - "TotalTxs": wireHasher(h.TotalTxs), - "LastBlockID": wireHasher(h.LastBlockID), - "LastCommit": wireHasher(h.LastCommitHash), - "Data": wireHasher(h.DataHash), - "Validators": wireHasher(h.ValidatorsHash), - "App": wireHasher(h.AppHash), - "Consensus": wireHasher(h.ConsensusHash), - "Results": wireHasher(h.LastResultsHash), - "Evidence": wireHasher(h.EvidenceHash), + "ChainID": aminoHasher(h.ChainID), + "Height": aminoHasher(h.Height), + "Time": aminoHasher(h.Time), + "NumTxs": aminoHasher(h.NumTxs), + "TotalTxs": aminoHasher(h.TotalTxs), + "LastBlockID": aminoHasher(h.LastBlockID), + "LastCommit": aminoHasher(h.LastCommitHash), + "Data": aminoHasher(h.DataHash), + "Validators": aminoHasher(h.ValidatorsHash), + "App": aminoHasher(h.AppHash), + "Consensus": aminoHasher(h.ConsensusHash), + "Results": aminoHasher(h.LastResultsHash), + "Evidence": aminoHasher(h.EvidenceHash), }) } @@ -364,7 +363,7 @@ func (commit *Commit) Hash() cmn.HexBytes { if commit.hash == nil { bs := make([]merkle.Hasher, len(commit.Precommits)) for i, precommit := range commit.Precommits { - bs[i] = wireHasher(precommit) + bs[i] = aminoHasher(precommit) } commit.hash = merkle.SimpleHashFromHashers(bs) } @@ -499,7 +498,7 @@ func (blockID BlockID) Equals(other BlockID) bool { // Key returns a machine-readable string representation of the BlockID func (blockID BlockID) Key() string { - bz, err := wire.MarshalBinary(blockID.PartsHeader) + bz, err := cdc.MarshalBinary(blockID.PartsHeader) if err != nil { panic(err) } @@ -519,7 +518,7 @@ type hasher struct { func (h hasher) Hash() []byte { hasher := ripemd160.New() - bz, err := wire.MarshalBinary(h.item) + bz, err := cdc.MarshalBinaryBare(h.item) if err != nil { panic(err) } @@ -531,11 +530,11 @@ func (h hasher) Hash() []byte { } -func tmHash(item interface{}) []byte { +func aminoHash(item interface{}) []byte { h := hasher{item} return h.Hash() } -func wireHasher(item interface{}) merkle.Hasher { +func aminoHasher(item interface{}) merkle.Hasher { return hasher{item} } diff --git a/types/block_test.go b/types/block_test.go index 1fcfa469b..d50980b3d 100644 --- a/types/block_test.go +++ b/types/block_test.go @@ -13,8 +13,7 @@ func TestValidateBlock(t *testing.T) { lastID := makeBlockIDRandom() h := int64(3) - voteSet, _, vals := randVoteSet(h-1, 1, VoteTypePrecommit, - 10, 1) + voteSet, _, vals := randVoteSet(h-1, 1, VoteTypePrecommit, 10, 1) commit, err := MakeCommit(lastID, h-1, 1, voteSet, vals) require.NoError(t, err) diff --git a/types/canonical_json.go b/types/canonical_json.go index 4eeeb2064..95ade9c67 100644 --- a/types/canonical_json.go +++ b/types/canonical_json.go @@ -3,14 +3,14 @@ package types import ( "time" - wire "github.com/tendermint/tendermint/wire" + "github.com/tendermint/go-amino" cmn "github.com/tendermint/tmlibs/common" ) -// canonical json is wire's json for structs with fields in alphabetical order +// Canonical json is amino's json for structs with fields in alphabetical order // TimeFormat is used for generating the sigs -const TimeFormat = wire.RFC3339Millis +const TimeFormat = amino.RFC3339Millis type CanonicalJSONBlockID struct { Hash cmn.HexBytes `json:"hash,omitempty"` @@ -18,11 +18,13 @@ type CanonicalJSONBlockID struct { } type CanonicalJSONPartSetHeader struct { - Hash cmn.HexBytes `json:"hash"` - Total int `json:"total"` + Hash cmn.HexBytes `json:"hash,omitempty"` + Total int `json:"total,omitempty"` } type CanonicalJSONProposal struct { + ChainID string `json:"@chain_id"` + Type string `json:"@type"` BlockPartsHeader CanonicalJSONPartSetHeader `json:"block_parts_header"` Height int64 `json:"height"` POLBlockID CanonicalJSONBlockID `json:"pol_block_id"` @@ -32,14 +34,18 @@ type CanonicalJSONProposal struct { } type CanonicalJSONVote struct { + ChainID string `json:"@chain_id"` + Type string `json:"@type"` BlockID CanonicalJSONBlockID `json:"block_id"` Height int64 `json:"height"` Round int `json:"round"` Timestamp string `json:"timestamp"` - Type byte `json:"type"` + VoteType byte `json:"type"` } type CanonicalJSONHeartbeat struct { + ChainID string `json:"@chain_id"` + Type string `json:"@type"` Height int64 `json:"height"` Round int `json:"round"` Sequence int `json:"sequence"` @@ -47,24 +53,6 @@ type CanonicalJSONHeartbeat struct { ValidatorIndex int `json:"validator_index"` } -//------------------------------------ -// Messages including a "chain id" can only be applied to one chain, hence "Once" - -type CanonicalJSONOnceProposal struct { - ChainID string `json:"chain_id"` - Proposal CanonicalJSONProposal `json:"proposal"` -} - -type CanonicalJSONOnceVote struct { - ChainID string `json:"chain_id"` - Vote CanonicalJSONVote `json:"vote"` -} - -type CanonicalJSONOnceHeartbeat struct { - ChainID string `json:"chain_id"` - Heartbeat CanonicalJSONHeartbeat `json:"heartbeat"` -} - //----------------------------------- // Canonicalize the structs @@ -82,8 +70,10 @@ func CanonicalPartSetHeader(psh PartSetHeader) CanonicalJSONPartSetHeader { } } -func CanonicalProposal(proposal *Proposal) CanonicalJSONProposal { +func CanonicalProposal(chainID string, proposal *Proposal) CanonicalJSONProposal { return CanonicalJSONProposal{ + ChainID: chainID, + Type: "proposal", BlockPartsHeader: CanonicalPartSetHeader(proposal.BlockPartsHeader), Height: proposal.Height, Timestamp: CanonicalTime(proposal.Timestamp), @@ -93,28 +83,32 @@ func CanonicalProposal(proposal *Proposal) CanonicalJSONProposal { } } -func CanonicalVote(vote *Vote) CanonicalJSONVote { +func CanonicalVote(chainID string, vote *Vote) CanonicalJSONVote { return CanonicalJSONVote{ + ChainID: chainID, + Type: "vote", BlockID: CanonicalBlockID(vote.BlockID), Height: vote.Height, Round: vote.Round, Timestamp: CanonicalTime(vote.Timestamp), - Type: vote.Type, + VoteType: vote.Type, } } -func CanonicalHeartbeat(heartbeat *Heartbeat) CanonicalJSONHeartbeat { +func CanonicalHeartbeat(chainID string, heartbeat *Heartbeat) CanonicalJSONHeartbeat { return CanonicalJSONHeartbeat{ - heartbeat.Height, - heartbeat.Round, - heartbeat.Sequence, - heartbeat.ValidatorAddress, - heartbeat.ValidatorIndex, + ChainID: chainID, + Type: "heartbeat", + Height: heartbeat.Height, + Round: heartbeat.Round, + Sequence: heartbeat.Sequence, + ValidatorAddress: heartbeat.ValidatorAddress, + ValidatorIndex: heartbeat.ValidatorIndex, } } func CanonicalTime(t time.Time) string { - // note that sending time over wire resets it to + // Note that sending time over amino resets it to // local time, we need to force UTC here, so the // signatures match return t.UTC().Format(TimeFormat) diff --git a/types/evidence.go b/types/evidence.go index 9e1f6af0e..658c40227 100644 --- a/types/evidence.go +++ b/types/evidence.go @@ -4,8 +4,8 @@ import ( "bytes" "fmt" + "github.com/tendermint/go-amino" "github.com/tendermint/go-crypto" - wire "github.com/tendermint/tendermint/wire" "github.com/tendermint/tmlibs/merkle" ) @@ -38,56 +38,11 @@ type Evidence interface { String() string } -//------------------------------------------- - -// EvidenceList is a list of Evidence. Evidences is not a word. -type EvidenceList []Evidence - -// Hash returns the simple merkle root hash of the EvidenceList. -func (evl EvidenceList) Hash() []byte { - // Recursive impl. - // Copied from tmlibs/merkle to avoid allocations - switch len(evl) { - case 0: - return nil - case 1: - return evl[0].Hash() - default: - left := EvidenceList(evl[:(len(evl)+1)/2]).Hash() - right := EvidenceList(evl[(len(evl)+1)/2:]).Hash() - return merkle.SimpleHashFromTwoHashes(left, right) - } +func RegisterEvidence(cdc *amino.Codec) { + cdc.RegisterInterface((*Evidence)(nil), nil) + cdc.RegisterConcrete(&DuplicateVoteEvidence{}, "tendermint/Evidence", nil) } -func (evl EvidenceList) String() string { - s := "" - for _, e := range evl { - s += fmt.Sprintf("%s\t\t", e) - } - return s -} - -// Has returns true if the evidence is in the EvidenceList. -func (evl EvidenceList) Has(evidence Evidence) bool { - for _, ev := range evl { - if ev.Equal(evidence) { - return true - } - } - return false -} - -//------------------------------------------- - -const ( - evidenceTypeDuplicateVote = byte(0x01) -) - -var _ = wire.RegisterInterface( - struct{ Evidence }{}, - wire.ConcreteType{&DuplicateVoteEvidence{}, evidenceTypeDuplicateVote}, -) - //------------------------------------------- // DuplicateVoteEvidence contains evidence a validator signed two conflicting votes. @@ -120,7 +75,7 @@ func (dve *DuplicateVoteEvidence) Index() int { // Hash returns the hash of the evidence. func (dve *DuplicateVoteEvidence) Hash() []byte { - return wireHasher(dve).Hash() + return aminoHasher(dve).Hash() } // Verify returns an error if the two votes aren't conflicting. @@ -165,8 +120,8 @@ func (dve *DuplicateVoteEvidence) Equal(ev Evidence) bool { } // just check their hashes - dveHash := wireHasher(dve).Hash() - evHash := wireHasher(ev).Hash() + dveHash := aminoHasher(dve).Hash() + evHash := aminoHasher(ev).Hash() return bytes.Equal(dveHash, evHash) } @@ -216,3 +171,42 @@ func (e MockBadEvidence) Equal(ev Evidence) bool { func (e MockBadEvidence) String() string { return fmt.Sprintf("BadEvidence: %d/%s/%d", e.Height_, e.Address_, e.Index_) } + +//------------------------------------------- + +// EvidenceList is a list of Evidence. Evidences is not a word. +type EvidenceList []Evidence + +// Hash returns the simple merkle root hash of the EvidenceList. +func (evl EvidenceList) Hash() []byte { + // Recursive impl. + // Copied from tmlibs/merkle to avoid allocations + switch len(evl) { + case 0: + return nil + case 1: + return evl[0].Hash() + default: + left := EvidenceList(evl[:(len(evl)+1)/2]).Hash() + right := EvidenceList(evl[(len(evl)+1)/2:]).Hash() + return merkle.SimpleHashFromTwoHashes(left, right) + } +} + +func (evl EvidenceList) String() string { + s := "" + for _, e := range evl { + s += fmt.Sprintf("%s\t\t", e) + } + return s +} + +// Has returns true if the evidence is in the EvidenceList. +func (evl EvidenceList) Has(evidence Evidence) bool { + for _, ev := range evl { + if ev.Equal(evidence) { + return true + } + } + return false +} diff --git a/types/evidence_test.go b/types/evidence_test.go index 84811514a..f2b1f91bb 100644 --- a/types/evidence_test.go +++ b/types/evidence_test.go @@ -4,7 +4,6 @@ import ( "testing" "github.com/stretchr/testify/assert" - cmn "github.com/tendermint/tmlibs/common" ) type voteData struct { @@ -13,25 +12,25 @@ type voteData struct { valid bool } -func makeVote(val *PrivValidatorFS, chainID string, valIndex int, height int64, round, step int, blockID BlockID) *Vote { +func makeVote(val PrivValidator, chainID string, valIndex int, height int64, round, step int, blockID BlockID) *Vote { v := &Vote{ - ValidatorAddress: val.PubKey.Address(), + ValidatorAddress: val.GetAddress(), ValidatorIndex: valIndex, Height: height, Round: round, Type: byte(step), BlockID: blockID, } - sig := val.PrivKey.Sign(v.SignBytes(chainID)) - v.Signature = sig + err := val.SignVote(chainID, v) + if err != nil { + panic(err) + } return v - } func TestEvidence(t *testing.T) { - _, tmpFilePath := cmn.Tempfile("priv_validator_") - val := GenPrivValidatorFS(tmpFilePath) - val2 := GenPrivValidatorFS(tmpFilePath) + val := NewMockPV() + val2 := NewMockPV() blockID := makeBlockID("blockhash", 1000, "partshash") blockID2 := makeBlockID("blockhash2", 1000, "partshash") blockID3 := makeBlockID("blockhash", 10000, "partshash") @@ -41,7 +40,10 @@ func TestEvidence(t *testing.T) { vote1 := makeVote(val, chainID, 0, 10, 2, 1, blockID) badVote := makeVote(val, chainID, 0, 10, 2, 1, blockID) - badVote.Signature = val2.PrivKey.Sign(badVote.SignBytes(chainID)) + err := val2.SignVote(chainID, badVote) + if err != nil { + panic(err) + } cases := []voteData{ {vote1, makeVote(val, chainID, 0, 10, 2, 1, blockID2), true}, // different block ids @@ -59,7 +61,7 @@ func TestEvidence(t *testing.T) { for _, c := range cases { ev := &DuplicateVoteEvidence{ - PubKey: val.PubKey, + PubKey: val.GetPubKey(), VoteA: c.vote1, VoteB: c.vote2, } diff --git a/types/genesis.go b/types/genesis.go index ef2d16791..36d2a46f1 100644 --- a/types/genesis.go +++ b/types/genesis.go @@ -5,9 +5,7 @@ import ( "io/ioutil" "time" - "github.com/pkg/errors" - - crypto "github.com/tendermint/go-crypto" + "github.com/tendermint/go-crypto" cmn "github.com/tendermint/tmlibs/common" ) @@ -33,7 +31,7 @@ type GenesisDoc struct { // SaveAs is a utility method for saving GenensisDoc as a JSON file. func (genDoc *GenesisDoc) SaveAs(file string) error { - genDocBytes, err := json.Marshal(genDoc) + genDocBytes, err := cdc.MarshalJSON(genDoc) if err != nil { return err } @@ -55,7 +53,7 @@ func (genDoc *GenesisDoc) ValidatorHash() []byte { func (genDoc *GenesisDoc) ValidateAndComplete() error { if genDoc.ChainID == "" { - return errors.Errorf("Genesis doc must include non-empty chain_id") + return cmn.NewError("Genesis doc must include non-empty chain_id") } if genDoc.ConsensusParams == nil { @@ -67,12 +65,12 @@ func (genDoc *GenesisDoc) ValidateAndComplete() error { } if len(genDoc.Validators) == 0 { - return errors.Errorf("The genesis file must have at least one validator") + return cmn.NewError("The genesis file must have at least one validator") } for _, v := range genDoc.Validators { if v.Power == 0 { - return errors.Errorf("The genesis file cannot contain validators with no voting power: %v", v) + return cmn.NewError("The genesis file cannot contain validators with no voting power: %v", v) } } @@ -89,7 +87,7 @@ func (genDoc *GenesisDoc) ValidateAndComplete() error { // GenesisDocFromJSON unmarshalls JSON data into a GenesisDoc. func GenesisDocFromJSON(jsonBlob []byte) (*GenesisDoc, error) { genDoc := GenesisDoc{} - err := json.Unmarshal(jsonBlob, &genDoc) + err := cdc.UnmarshalJSON(jsonBlob, &genDoc) if err != nil { return nil, err } @@ -105,11 +103,11 @@ func GenesisDocFromJSON(jsonBlob []byte) (*GenesisDoc, error) { func GenesisDocFromFile(genDocFile string) (*GenesisDoc, error) { jsonBlob, err := ioutil.ReadFile(genDocFile) if err != nil { - return nil, errors.Wrap(err, "Couldn't read GenesisDoc file") + return nil, cmn.ErrorWrap(err, "Couldn't read GenesisDoc file") } genDoc, err := GenesisDocFromJSON(jsonBlob) if err != nil { - return nil, errors.Wrap(err, cmn.Fmt("Error reading GenesisDoc at %v", genDocFile)) + return nil, cmn.ErrorWrap(err, cmn.Fmt("Error reading GenesisDoc at %v", genDocFile)) } return genDoc, nil } diff --git a/types/genesis_test.go b/types/genesis_test.go index aa713289f..17ebf1cfc 100644 --- a/types/genesis_test.go +++ b/types/genesis_test.go @@ -1,35 +1,34 @@ package types import ( - "encoding/json" - "testing" - "github.com/stretchr/testify/assert" - - crypto "github.com/tendermint/go-crypto" + "github.com/tendermint/go-crypto" + "testing" ) -func TestGenesis(t *testing.T) { +func TestGenesisBad(t *testing.T) { // test some bad ones from raw json testCases := [][]byte{ - []byte{}, // empty - []byte{1, 1, 1, 1, 1}, // junk - []byte(`{}`), // empty - []byte(`{"chain_id": "mychain"}`), // missing validators - []byte(`{"chain_id": "mychain", "validators": []`), // missing validators - []byte(`{"chain_id": "mychain", "validators": [{}]`), // missing validators - []byte(`{"validators":[{"pub_key": - {"type":"ed25519","data":"961EAB8752E51A03618502F55C2B6E09C38C65635C64CCF3173ED452CF86C957"}, - "power":10,"name":""}]}`), // missing chain_id + []byte{}, // empty + []byte{1, 1, 1, 1, 1}, // junk + []byte(`{}`), // empty + []byte(`{"chain_id":"mychain"}`), // missing validators + []byte(`{"chain_id":"mychain","validators":[]}`), // missing validators + []byte(`{"chain_id":"mychain","validators":[{}]}`), // missing validators + []byte(`{"chain_id":"mychain","validators":null}`), // missing validators + []byte(`{"chain_id":"mychain"}`), // missing validators + []byte(`{"validators":[{"pub_key":{"type":"AC26791624DE60","value":"AT/+aaL1eB0477Mud9JMm8Sh8BIvOYlPGC9KkIUmFaE="},"power":10,"name":""}]}`), // missing chain_id } for _, testCase := range testCases { _, err := GenesisDocFromJSON(testCase) assert.Error(t, err, "expected error for empty genDoc json") } +} +func TestGenesisGood(t *testing.T) { // test a good one by raw json - genDocBytes := []byte(`{"genesis_time":"0001-01-01T00:00:00Z","chain_id":"test-chain-QDKdJr","consensus_params":null,"validators":[{"pub_key":{"type":"ed25519","data":"961EAB8752E51A03618502F55C2B6E09C38C65635C64CCF3173ED452CF86C957"},"power":10,"name":""}],"app_hash":"","app_state":{"account_owner": "Bob"}}`) + genDocBytes := []byte(`{"genesis_time":"0001-01-01T00:00:00Z","chain_id":"test-chain-QDKdJr","consensus_params":null,"validators":[{"pub_key":{"type":"AC26791624DE60","value":"AT/+aaL1eB0477Mud9JMm8Sh8BIvOYlPGC9KkIUmFaE="},"power":10,"name":""}],"app_hash":"","app_state":{"account_owner": "Bob"}}`) _, err := GenesisDocFromJSON(genDocBytes) assert.NoError(t, err, "expected no error for good genDoc json") @@ -38,7 +37,7 @@ func TestGenesis(t *testing.T) { ChainID: "abc", Validators: []GenesisValidator{{crypto.GenPrivKeyEd25519().PubKey(), 10, "myval"}}, } - genDocBytes, err = json.Marshal(baseGenDoc) + genDocBytes, err = cdc.MarshalJSON(baseGenDoc) assert.NoError(t, err, "error marshalling genDoc") // test base gendoc and check consensus params were filled @@ -47,14 +46,14 @@ func TestGenesis(t *testing.T) { assert.NotNil(t, genDoc.ConsensusParams, "expected consensus params to be filled in") // create json with consensus params filled - genDocBytes, err = json.Marshal(genDoc) + genDocBytes, err = cdc.MarshalJSON(genDoc) assert.NoError(t, err, "error marshalling genDoc") genDoc, err = GenesisDocFromJSON(genDocBytes) assert.NoError(t, err, "expected no error for valid genDoc json") // test with invalid consensus params genDoc.ConsensusParams.BlockSize.MaxBytes = 0 - genDocBytes, err = json.Marshal(genDoc) + genDocBytes, err = cdc.MarshalJSON(genDoc) assert.NoError(t, err, "error marshalling genDoc") genDoc, err = GenesisDocFromJSON(genDocBytes) assert.Error(t, err, "expected error for genDoc json with block size of 0") diff --git a/types/heartbeat.go b/types/heartbeat.go index fc5f8ad7f..097dd22db 100644 --- a/types/heartbeat.go +++ b/types/heartbeat.go @@ -4,7 +4,6 @@ import ( "fmt" "github.com/tendermint/go-crypto" - "github.com/tendermint/tendermint/wire" cmn "github.com/tendermint/tmlibs/common" ) @@ -14,7 +13,7 @@ import ( // json field tags because we always want the JSON // representation to be in its canonical form. type Heartbeat struct { - ValidatorAddress Address `json:"validator_address"` + ValidatorAddress Address `json:"validator_address"` ValidatorIndex int `json:"validator_index"` Height int64 `json:"height"` Round int `json:"round"` @@ -25,10 +24,7 @@ type Heartbeat struct { // SignBytes returns the Heartbeat bytes for signing. // It panics if the Heartbeat is nil. func (heartbeat *Heartbeat) SignBytes(chainID string) []byte { - bz, err := wire.MarshalJSON(CanonicalJSONOnceHeartbeat{ - chainID, - CanonicalHeartbeat(heartbeat), - }) + bz, err := cdc.MarshalJSON(CanonicalHeartbeat(chainID, heartbeat)) if err != nil { panic(err) } diff --git a/types/heartbeat_test.go b/types/heartbeat_test.go index 206636166..3c4536028 100644 --- a/types/heartbeat_test.go +++ b/types/heartbeat_test.go @@ -25,22 +25,23 @@ func TestHeartbeatString(t *testing.T) { require.Contains(t, nilHb.String(), "nil", "expecting a string and no panic") hb := &Heartbeat{ValidatorIndex: 1, Height: 11, Round: 2} - require.Equal(t, hb.String(), "Heartbeat{1:000000000000 11/02 (0) {}}") + require.Equal(t, hb.String(), "Heartbeat{1:000000000000 11/02 (0) }") var key crypto.PrivKeyEd25519 hb.Signature = key.Sign([]byte("Tendermint")) - require.Equal(t, hb.String(), "Heartbeat{1:000000000000 11/02 (0) {/FF41E371B9BF.../}}") + require.Equal(t, hb.String(), "Heartbeat{1:000000000000 11/02 (0) /FF41E371B9BF.../}") } func TestHeartbeatWriteSignBytes(t *testing.T) { hb := &Heartbeat{ValidatorIndex: 1, Height: 10, Round: 1} bz := hb.SignBytes("0xdeadbeef") - require.Equal(t, string(bz), `{"chain_id":"0xdeadbeef","heartbeat":{"height":10,"round":1,"sequence":0,"validator_address":"","validator_index":1}}`) + // XXX HMMMMMMM + require.Equal(t, string(bz), `{"@chain_id":"0xdeadbeef","@type":"heartbeat","height":10,"round":1,"sequence":0,"validator_address":"","validator_index":1}`) plainHb := &Heartbeat{} bz = plainHb.SignBytes("0xdeadbeef") - require.Equal(t, string(bz), `{"chain_id":"0xdeadbeef","heartbeat":{"height":0,"round":0,"sequence":0,"validator_address":"","validator_index":0}}`) + require.Equal(t, string(bz), `{"@chain_id":"0xdeadbeef","@type":"heartbeat","height":0,"round":0,"sequence":0,"validator_address":"","validator_index":0}`) require.Panics(t, func() { var nilHb *Heartbeat diff --git a/types/params.go b/types/params.go index 0e8ac577f..0626663e5 100644 --- a/types/params.go +++ b/types/params.go @@ -1,9 +1,8 @@ package types import ( - "github.com/pkg/errors" - abci "github.com/tendermint/abci/types" + cmn "github.com/tendermint/tmlibs/common" "github.com/tendermint/tmlibs/merkle" ) @@ -89,15 +88,15 @@ func DefaultEvidenceParams() EvidenceParams { func (params *ConsensusParams) Validate() error { // ensure some values are greater than 0 if params.BlockSize.MaxBytes <= 0 { - return errors.Errorf("BlockSize.MaxBytes must be greater than 0. Got %d", params.BlockSize.MaxBytes) + return cmn.NewError("BlockSize.MaxBytes must be greater than 0. Got %d", params.BlockSize.MaxBytes) } if params.BlockGossip.BlockPartSizeBytes <= 0 { - return errors.Errorf("BlockGossip.BlockPartSizeBytes must be greater than 0. Got %d", params.BlockGossip.BlockPartSizeBytes) + return cmn.NewError("BlockGossip.BlockPartSizeBytes must be greater than 0. Got %d", params.BlockGossip.BlockPartSizeBytes) } // ensure blocks aren't too big if params.BlockSize.MaxBytes > maxBlockSizeBytes { - return errors.Errorf("BlockSize.MaxBytes is too big. %d > %d", + return cmn.NewError("BlockSize.MaxBytes is too big. %d > %d", params.BlockSize.MaxBytes, maxBlockSizeBytes) } return nil @@ -107,12 +106,12 @@ func (params *ConsensusParams) Validate() error { // in the block header func (params *ConsensusParams) Hash() []byte { return merkle.SimpleHashFromMap(map[string]merkle.Hasher{ - "block_gossip_part_size_bytes": wireHasher(params.BlockGossip.BlockPartSizeBytes), - "block_size_max_bytes": wireHasher(params.BlockSize.MaxBytes), - "block_size_max_gas": wireHasher(params.BlockSize.MaxGas), - "block_size_max_txs": wireHasher(params.BlockSize.MaxTxs), - "tx_size_max_bytes": wireHasher(params.TxSize.MaxBytes), - "tx_size_max_gas": wireHasher(params.TxSize.MaxGas), + "block_gossip_part_size_bytes": aminoHasher(params.BlockGossip.BlockPartSizeBytes), + "block_size_max_bytes": aminoHasher(params.BlockSize.MaxBytes), + "block_size_max_gas": aminoHasher(params.BlockSize.MaxGas), + "block_size_max_txs": aminoHasher(params.BlockSize.MaxTxs), + "tx_size_max_bytes": aminoHasher(params.TxSize.MaxBytes), + "tx_size_max_gas": aminoHasher(params.TxSize.MaxGas), }) } diff --git a/types/priv_validator.go b/types/priv_validator.go index daa456bc0..06a024215 100644 --- a/types/priv_validator.go +++ b/types/priv_validator.go @@ -2,88 +2,11 @@ package types import ( "bytes" - "encoding/json" - "errors" "fmt" - "io/ioutil" - "sync" - "time" - crypto "github.com/tendermint/go-crypto" - cmn "github.com/tendermint/tmlibs/common" + "github.com/tendermint/go-crypto" ) -// TODO: type ? -const ( - stepNone int8 = 0 // Used to distinguish the initial state - stepPropose int8 = 1 - stepPrevote int8 = 2 - stepPrecommit int8 = 3 -) - -func voteToStep(vote *Vote) int8 { - switch vote.Type { - case VoteTypePrevote: - return stepPrevote - case VoteTypePrecommit: - return stepPrecommit - default: - cmn.PanicSanity("Unknown vote type") - return 0 - } -} - -//-------------------------------------------------------------- -// PrivValidator is being upgraded! See types/priv_validator/ - -// ValidatorID contains the identity of the validator. -type ValidatorID struct { - Address cmn.HexBytes `json:"address"` - PubKey crypto.PubKey `json:"pub_key"` -} - -// PrivValidator defines the functionality of a local Tendermint validator -// that signs votes, proposals, and heartbeats, and never double signs. -type PrivValidator2 interface { - Address() (Address, error) // redundant since .PubKey().Address() - PubKey() (crypto.PubKey, error) - - SignVote(chainID string, vote *Vote) error - SignProposal(chainID string, proposal *Proposal) error - SignHeartbeat(chainID string, heartbeat *Heartbeat) error -} - -type TestSigner interface { - Address() cmn.HexBytes - PubKey() crypto.PubKey - Sign([]byte) (crypto.Signature, error) -} - -func GenSigner() TestSigner { - return &DefaultTestSigner{ - crypto.GenPrivKeyEd25519().Wrap(), - } -} - -type DefaultTestSigner struct { - crypto.PrivKey -} - -func (ds *DefaultTestSigner) Address() cmn.HexBytes { - return ds.PubKey().Address() -} - -func (ds *DefaultTestSigner) PubKey() crypto.PubKey { - return ds.PrivKey.PubKey() -} - -func (ds *DefaultTestSigner) Sign(msg []byte) (crypto.Signature, error) { - return ds.PrivKey.Sign(msg), nil -} - -//-------------------------------------------------------------- -// TODO: Deprecate! - // PrivValidator defines the functionality of a local Tendermint validator // that signs votes, proposals, and heartbeats, and never double signs. type PrivValidator interface { @@ -95,313 +18,10 @@ type PrivValidator interface { SignHeartbeat(chainID string, heartbeat *Heartbeat) error } -// PrivValidatorFS implements PrivValidator using data persisted to disk -// to prevent double signing. The Signer itself can be mutated to use -// something besides the default, for instance a hardware signer. -// NOTE: the directory containing the privVal.filePath must already exist. -type PrivValidatorFS struct { - Address Address `json:"address"` - PubKey crypto.PubKey `json:"pub_key"` - LastHeight int64 `json:"last_height"` - LastRound int `json:"last_round"` - LastStep int8 `json:"last_step"` - LastSignature crypto.Signature `json:"last_signature,omitempty"` // so we dont lose signatures - LastSignBytes cmn.HexBytes `json:"last_signbytes,omitempty"` // so we dont lose signatures +//---------------------------------------- +// Misc. - // PrivKey should be empty if a Signer other than the default is being used. - PrivKey crypto.PrivKey `json:"priv_key"` - Signer `json:"-"` - - // For persistence. - // Overloaded for testing. - filePath string - mtx sync.Mutex -} - -// Signer is an interface that defines how to sign messages. -// It is the caller's duty to verify the msg before calling Sign, -// eg. to avoid double signing. -// Currently, the only callers are SignVote, SignProposal, and SignHeartbeat. -type Signer interface { - Sign(msg []byte) (crypto.Signature, error) -} - -// DefaultSigner implements Signer. -// It uses a standard, unencrypted crypto.PrivKey. -type DefaultSigner struct { - PrivKey crypto.PrivKey `json:"priv_key"` -} - -// NewDefaultSigner returns an instance of DefaultSigner. -func NewDefaultSigner(priv crypto.PrivKey) *DefaultSigner { - return &DefaultSigner{ - PrivKey: priv, - } -} - -// Sign implements Signer. It signs the byte slice with a private key. -func (ds *DefaultSigner) Sign(msg []byte) (crypto.Signature, error) { - return ds.PrivKey.Sign(msg), nil -} - -// GetAddress returns the address of the validator. -// Implements PrivValidator. -func (pv *PrivValidatorFS) GetAddress() Address { - return pv.Address -} - -// GetPubKey returns the public key of the validator. -// Implements PrivValidator. -func (pv *PrivValidatorFS) GetPubKey() crypto.PubKey { - return pv.PubKey -} - -// GenPrivValidatorFS generates a new validator with randomly generated private key -// and sets the filePath, but does not call Save(). -func GenPrivValidatorFS(filePath string) *PrivValidatorFS { - privKey := crypto.GenPrivKeyEd25519().Wrap() - return &PrivValidatorFS{ - Address: privKey.PubKey().Address(), - PubKey: privKey.PubKey(), - PrivKey: privKey, - LastStep: stepNone, - Signer: NewDefaultSigner(privKey), - filePath: filePath, - } -} - -// LoadPrivValidatorFS loads a PrivValidatorFS from the filePath. -func LoadPrivValidatorFS(filePath string) *PrivValidatorFS { - return LoadPrivValidatorFSWithSigner(filePath, func(privVal PrivValidator) Signer { - return NewDefaultSigner(privVal.(*PrivValidatorFS).PrivKey) - }) -} - -// LoadOrGenPrivValidatorFS loads a PrivValidatorFS from the given filePath -// or else generates a new one and saves it to the filePath. -func LoadOrGenPrivValidatorFS(filePath string) *PrivValidatorFS { - var privVal *PrivValidatorFS - if cmn.FileExists(filePath) { - privVal = LoadPrivValidatorFS(filePath) - } else { - privVal = GenPrivValidatorFS(filePath) - privVal.Save() - } - return privVal -} - -// LoadPrivValidatorWithSigner loads a PrivValidatorFS with a custom -// signer object. The PrivValidatorFS handles double signing prevention by persisting -// data to the filePath, while the Signer handles the signing. -// If the filePath does not exist, the PrivValidatorFS must be created manually and saved. -func LoadPrivValidatorFSWithSigner(filePath string, signerFunc func(PrivValidator) Signer) *PrivValidatorFS { - privValJSONBytes, err := ioutil.ReadFile(filePath) - if err != nil { - cmn.Exit(err.Error()) - } - privVal := &PrivValidatorFS{} - err = json.Unmarshal(privValJSONBytes, &privVal) - if err != nil { - cmn.Exit(cmn.Fmt("Error reading PrivValidator from %v: %v\n", filePath, err)) - } - - privVal.filePath = filePath - privVal.Signer = signerFunc(privVal) - return privVal -} - -// Save persists the PrivValidatorFS to disk. -func (privVal *PrivValidatorFS) Save() { - privVal.mtx.Lock() - defer privVal.mtx.Unlock() - privVal.save() -} - -func (privVal *PrivValidatorFS) save() { - outFile := privVal.filePath - if outFile == "" { - panic("Cannot save PrivValidator: filePath not set") - } - jsonBytes, err := json.Marshal(privVal) - if err != nil { - panic(err) - } - err = cmn.WriteFileAtomic(outFile, jsonBytes, 0600) - if err != nil { - panic(err) - } -} - -// Reset resets all fields in the PrivValidatorFS. -// NOTE: Unsafe! -func (privVal *PrivValidatorFS) Reset() { - var sig crypto.Signature - privVal.LastHeight = 0 - privVal.LastRound = 0 - privVal.LastStep = 0 - privVal.LastSignature = sig - privVal.LastSignBytes = nil - privVal.Save() -} - -// SignVote signs a canonical representation of the vote, along with the -// chainID. Implements PrivValidator. -func (privVal *PrivValidatorFS) SignVote(chainID string, vote *Vote) error { - privVal.mtx.Lock() - defer privVal.mtx.Unlock() - if err := privVal.signVote(chainID, vote); err != nil { - return errors.New(cmn.Fmt("Error signing vote: %v", err)) - } - return nil -} - -// SignProposal signs a canonical representation of the proposal, along with -// the chainID. Implements PrivValidator. -func (privVal *PrivValidatorFS) SignProposal(chainID string, proposal *Proposal) error { - privVal.mtx.Lock() - defer privVal.mtx.Unlock() - if err := privVal.signProposal(chainID, proposal); err != nil { - return fmt.Errorf("Error signing proposal: %v", err) - } - return nil -} - -// returns error if HRS regression or no LastSignBytes. returns true if HRS is unchanged -func (privVal *PrivValidatorFS) checkHRS(height int64, round int, step int8) (bool, error) { - if privVal.LastHeight > height { - return false, errors.New("Height regression") - } - - if privVal.LastHeight == height { - if privVal.LastRound > round { - return false, errors.New("Round regression") - } - - if privVal.LastRound == round { - if privVal.LastStep > step { - return false, errors.New("Step regression") - } else if privVal.LastStep == step { - if privVal.LastSignBytes != nil { - if privVal.LastSignature.Empty() { - panic("privVal: LastSignature is nil but LastSignBytes is not!") - } - return true, nil - } - return false, errors.New("No LastSignature found") - } - } - } - return false, nil -} - -// signVote checks if the vote is good to sign and sets the vote signature. -// It may need to set the timestamp as well if the vote is otherwise the same as -// a previously signed vote (ie. we crashed after signing but before the vote hit the WAL). -func (privVal *PrivValidatorFS) signVote(chainID string, vote *Vote) error { - height, round, step := vote.Height, vote.Round, voteToStep(vote) - signBytes := vote.SignBytes(chainID) - - sameHRS, err := privVal.checkHRS(height, round, step) - if err != nil { - return err - } - - // We might crash before writing to the wal, - // causing us to try to re-sign for the same HRS. - // If signbytes are the same, use the last signature. - // If they only differ by timestamp, use last timestamp and signature - // Otherwise, return error - if sameHRS { - if bytes.Equal(signBytes, privVal.LastSignBytes) { - vote.Signature = privVal.LastSignature - } else if timestamp, ok := checkVotesOnlyDifferByTimestamp(privVal.LastSignBytes, signBytes); ok { - vote.Timestamp = timestamp - vote.Signature = privVal.LastSignature - } else { - err = fmt.Errorf("Conflicting data") - } - return err - } - - // It passed the checks. Sign the vote - sig, err := privVal.Sign(signBytes) - if err != nil { - return err - } - privVal.saveSigned(height, round, step, signBytes, sig) - vote.Signature = sig - return nil -} - -// signProposal checks if the proposal is good to sign and sets the proposal signature. -// It may need to set the timestamp as well if the proposal is otherwise the same as -// a previously signed proposal ie. we crashed after signing but before the proposal hit the WAL). -func (privVal *PrivValidatorFS) signProposal(chainID string, proposal *Proposal) error { - height, round, step := proposal.Height, proposal.Round, stepPropose - signBytes := proposal.SignBytes(chainID) - - sameHRS, err := privVal.checkHRS(height, round, step) - if err != nil { - return err - } - - // We might crash before writing to the wal, - // causing us to try to re-sign for the same HRS. - // If signbytes are the same, use the last signature. - // If they only differ by timestamp, use last timestamp and signature - // Otherwise, return error - if sameHRS { - if bytes.Equal(signBytes, privVal.LastSignBytes) { - proposal.Signature = privVal.LastSignature - } else if timestamp, ok := checkProposalsOnlyDifferByTimestamp(privVal.LastSignBytes, signBytes); ok { - proposal.Timestamp = timestamp - proposal.Signature = privVal.LastSignature - } else { - err = fmt.Errorf("Conflicting data") - } - return err - } - - // It passed the checks. Sign the proposal - sig, err := privVal.Sign(signBytes) - if err != nil { - return err - } - privVal.saveSigned(height, round, step, signBytes, sig) - proposal.Signature = sig - return nil -} - -// Persist height/round/step and signature -func (privVal *PrivValidatorFS) saveSigned(height int64, round int, step int8, - signBytes []byte, sig crypto.Signature) { - - privVal.LastHeight = height - privVal.LastRound = round - privVal.LastStep = step - privVal.LastSignature = sig - privVal.LastSignBytes = signBytes - privVal.save() -} - -// SignHeartbeat signs a canonical representation of the heartbeat, along with the chainID. -// Implements PrivValidator. -func (privVal *PrivValidatorFS) SignHeartbeat(chainID string, heartbeat *Heartbeat) error { - privVal.mtx.Lock() - defer privVal.mtx.Unlock() - var err error - heartbeat.Signature, err = privVal.Sign(heartbeat.SignBytes(chainID)) - return err -} - -// String returns a string representation of the PrivValidatorFS. -func (privVal *PrivValidatorFS) String() string { - return fmt.Sprintf("PrivValidator{%v LH:%v, LR:%v, LS:%v}", privVal.GetAddress(), privVal.LastHeight, privVal.LastRound, privVal.LastStep) -} - -//------------------------------------- - -type PrivValidatorsByAddress []*PrivValidatorFS +type PrivValidatorsByAddress []PrivValidator func (pvs PrivValidatorsByAddress) Len() int { return len(pvs) @@ -417,56 +37,53 @@ func (pvs PrivValidatorsByAddress) Swap(i, j int) { pvs[j] = it } -//------------------------------------- +//---------------------------------------- +// MockPV -// returns the timestamp from the lastSignBytes. -// returns true if the only difference in the votes is their timestamp. -func checkVotesOnlyDifferByTimestamp(lastSignBytes, newSignBytes []byte) (time.Time, bool) { - var lastVote, newVote CanonicalJSONOnceVote - if err := json.Unmarshal(lastSignBytes, &lastVote); err != nil { - panic(fmt.Sprintf("LastSignBytes cannot be unmarshalled into vote: %v", err)) - } - if err := json.Unmarshal(newSignBytes, &newVote); err != nil { - panic(fmt.Sprintf("signBytes cannot be unmarshalled into vote: %v", err)) - } - - lastTime, err := time.Parse(TimeFormat, lastVote.Vote.Timestamp) - if err != nil { - panic(err) - } - - // set the times to the same value and check equality - now := CanonicalTime(time.Now()) - lastVote.Vote.Timestamp = now - newVote.Vote.Timestamp = now - lastVoteBytes, _ := json.Marshal(lastVote) - newVoteBytes, _ := json.Marshal(newVote) - - return lastTime, bytes.Equal(newVoteBytes, lastVoteBytes) +// MockPV implements PrivValidator without any safety or persistence. +// Only use it for testing. +type MockPV struct { + privKey crypto.PrivKey } -// returns the timestamp from the lastSignBytes. -// returns true if the only difference in the proposals is their timestamp -func checkProposalsOnlyDifferByTimestamp(lastSignBytes, newSignBytes []byte) (time.Time, bool) { - var lastProposal, newProposal CanonicalJSONOnceProposal - if err := json.Unmarshal(lastSignBytes, &lastProposal); err != nil { - panic(fmt.Sprintf("LastSignBytes cannot be unmarshalled into proposal: %v", err)) - } - if err := json.Unmarshal(newSignBytes, &newProposal); err != nil { - panic(fmt.Sprintf("signBytes cannot be unmarshalled into proposal: %v", err)) - } - - lastTime, err := time.Parse(TimeFormat, lastProposal.Proposal.Timestamp) - if err != nil { - panic(err) - } - - // set the times to the same value and check equality - now := CanonicalTime(time.Now()) - lastProposal.Proposal.Timestamp = now - newProposal.Proposal.Timestamp = now - lastProposalBytes, _ := json.Marshal(lastProposal) - newProposalBytes, _ := json.Marshal(newProposal) - - return lastTime, bytes.Equal(newProposalBytes, lastProposalBytes) +func NewMockPV() *MockPV { + return &MockPV{crypto.GenPrivKeyEd25519()} +} + +// Implements PrivValidator. +func (pv *MockPV) GetAddress() Address { + return pv.privKey.PubKey().Address() +} + +// Implements PrivValidator. +func (pv *MockPV) GetPubKey() crypto.PubKey { + return pv.privKey.PubKey() +} + +// Implements PrivValidator. +func (pv *MockPV) SignVote(chainID string, vote *Vote) error { + signBytes := vote.SignBytes(chainID) + sig := pv.privKey.Sign(signBytes) + vote.Signature = sig + return nil +} + +// Implements PrivValidator. +func (pv *MockPV) SignProposal(chainID string, proposal *Proposal) error { + signBytes := proposal.SignBytes(chainID) + sig := pv.privKey.Sign(signBytes) + proposal.Signature = sig + return nil +} + +// signHeartbeat signs the heartbeat without any checking. +func (pv *MockPV) SignHeartbeat(chainID string, heartbeat *Heartbeat) error { + sig := pv.privKey.Sign(heartbeat.SignBytes(chainID)) + heartbeat.Signature = sig + return nil +} + +// String returns a string representation of the MockPV. +func (pv *MockPV) String() string { + return fmt.Sprintf("MockPV{%v}", pv.GetAddress()) } diff --git a/types/priv_validator/json.go b/types/priv_validator/json.go deleted file mode 100644 index 5c0849ebd..000000000 --- a/types/priv_validator/json.go +++ /dev/null @@ -1,197 +0,0 @@ -package types - -import ( - "bytes" - "encoding/json" - "fmt" - "io/ioutil" - - crypto "github.com/tendermint/go-crypto" - "github.com/tendermint/tendermint/types" - cmn "github.com/tendermint/tmlibs/common" -) - -// PrivValidator aliases types.PrivValidator -type PrivValidator = types.PrivValidator2 - -//----------------------------------------------------- - -// PrivKey implements Signer -type PrivKey crypto.PrivKey - -// Sign - Implements Signer -func (pk PrivKey) Sign(msg []byte) (crypto.Signature, error) { - return crypto.PrivKey(pk).Sign(msg), nil -} - -// MarshalJSON satisfies json.Marshaler. -func (pk PrivKey) MarshalJSON() ([]byte, error) { - return crypto.PrivKey(pk).MarshalJSON() -} - -// UnmarshalJSON satisfies json.Unmarshaler. -func (pk *PrivKey) UnmarshalJSON(b []byte) error { - cpk := new(crypto.PrivKey) - if err := cpk.UnmarshalJSON(b); err != nil { - return err - } - *pk = (PrivKey)(*cpk) - return nil -} - -//----------------------------------------------------- - -var _ types.PrivValidator2 = (*PrivValidatorJSON)(nil) - -// PrivValidatorJSON wraps PrivValidatorUnencrypted -// and persists it to disk after every SignVote and SignProposal. -type PrivValidatorJSON struct { - *PrivValidatorUnencrypted - - filePath string -} - -// SignVote implements PrivValidator. It persists to disk. -func (pvj *PrivValidatorJSON) SignVote(chainID string, vote *types.Vote) error { - err := pvj.PrivValidatorUnencrypted.SignVote(chainID, vote) - if err != nil { - return err - } - pvj.Save() - return nil -} - -// SignProposal implements PrivValidator. It persists to disk. -func (pvj *PrivValidatorJSON) SignProposal(chainID string, proposal *types.Proposal) error { - err := pvj.PrivValidatorUnencrypted.SignProposal(chainID, proposal) - if err != nil { - return err - } - pvj.Save() - return nil -} - -//------------------------------------------------------- - -// String returns a string representation of the PrivValidatorJSON. -func (pvj *PrivValidatorJSON) String() string { - addr, err := pvj.Address() - if err != nil { - panic(err) - } - - return fmt.Sprintf("PrivValidator{%v %v}", addr, pvj.PrivValidatorUnencrypted.String()) -} - -// Save persists the PrivValidatorJSON to disk. -func (pvj *PrivValidatorJSON) Save() { - pvj.save() -} - -func (pvj *PrivValidatorJSON) save() { - if pvj.filePath == "" { - panic("Cannot save PrivValidator: filePath not set") - } - jsonBytes, err := json.Marshal(pvj) - if err != nil { - // ; BOOM!!! - panic(err) - } - err = cmn.WriteFileAtomic(pvj.filePath, jsonBytes, 0600) - if err != nil { - // ; BOOM!!! - panic(err) - } -} - -// Reset resets the PrivValidatorUnencrypted. Panics if the Signer is the wrong type. -// NOTE: Unsafe! -func (pvj *PrivValidatorJSON) Reset() { - pvj.PrivValidatorUnencrypted.LastSignedInfo.Reset() - pvj.Save() -} - -//---------------------------------------------------------------- - -// GenPrivValidatorJSON generates a new validator with randomly generated private key -// and the given filePath. It does not persist to file. -func GenPrivValidatorJSON(filePath string) *PrivValidatorJSON { - privKey := crypto.GenPrivKeyEd25519().Wrap() - return &PrivValidatorJSON{ - PrivValidatorUnencrypted: NewPrivValidatorUnencrypted(privKey), - filePath: filePath, - } -} - -// LoadPrivValidatorJSON loads a PrivValidatorJSON from the filePath. -func LoadPrivValidatorJSON(filePath string) *PrivValidatorJSON { - pvJSONBytes, err := ioutil.ReadFile(filePath) - if err != nil { - cmn.Exit(err.Error()) - } - pvj := PrivValidatorJSON{} - err = json.Unmarshal(pvJSONBytes, &pvj) - if err != nil { - cmn.Exit(cmn.Fmt("Error reading PrivValidatorJSON from %v: %v\n", filePath, err)) - } - - // enable persistence - pvj.filePath = filePath - return &pvj -} - -// LoadOrGenPrivValidatorJSON loads a PrivValidatorJSON from the given filePath -// or else generates a new one and saves it to the filePath. -func LoadOrGenPrivValidatorJSON(filePath string) *PrivValidatorJSON { - var pvj *PrivValidatorJSON - if cmn.FileExists(filePath) { - pvj = LoadPrivValidatorJSON(filePath) - } else { - pvj = GenPrivValidatorJSON(filePath) - pvj.Save() - } - return pvj -} - -//-------------------------------------------------------------- - -// NewTestPrivValidator returns a PrivValidatorJSON with a tempfile -// for the file path. -func NewTestPrivValidator(signer types.TestSigner) *PrivValidatorJSON { - _, tempFilePath := cmn.Tempfile("priv_validator_") - pv := &PrivValidatorJSON{ - PrivValidatorUnencrypted: NewPrivValidatorUnencrypted(signer.(*types.DefaultTestSigner).PrivKey), - filePath: tempFilePath, - } - return pv -} - -//------------------------------------------------------ - -// PrivValidatorsByAddress is a list of PrivValidatorJSON ordered by their -// addresses. -type PrivValidatorsByAddress []*PrivValidatorJSON - -func (pvs PrivValidatorsByAddress) Len() int { - return len(pvs) -} - -func (pvs PrivValidatorsByAddress) Less(i, j int) bool { - iaddr, err := pvs[j].Address() - if err != nil { - panic(err) - } - - jaddr, err := pvs[i].Address() - if err != nil { - panic(err) - } - - return bytes.Compare(iaddr, jaddr) == -1 -} - -func (pvs PrivValidatorsByAddress) Swap(i, j int) { - it := pvs[i] - pvs[i] = pvs[j] - pvs[j] = it -} diff --git a/types/priv_validator/priv_validator.go b/types/priv_validator/priv_validator.go new file mode 100644 index 000000000..baff28f64 --- /dev/null +++ b/types/priv_validator/priv_validator.go @@ -0,0 +1,345 @@ +package privval + +import ( + "bytes" + "errors" + "fmt" + "io/ioutil" + "sync" + "time" + + "github.com/tendermint/go-crypto" + "github.com/tendermint/tendermint/types" + cmn "github.com/tendermint/tmlibs/common" +) + +// TODO: type ? +const ( + stepNone int8 = 0 // Used to distinguish the initial state + stepPropose int8 = 1 + stepPrevote int8 = 2 + stepPrecommit int8 = 3 +) + +func voteToStep(vote *types.Vote) int8 { + switch vote.Type { + case types.VoteTypePrevote: + return stepPrevote + case types.VoteTypePrecommit: + return stepPrecommit + default: + cmn.PanicSanity("Unknown vote type") + return 0 + } +} + +// FilePV implements PrivValidator using data persisted to disk +// to prevent double signing. +// NOTE: the directory containing the pv.filePath must already exist. +type FilePV struct { + Address types.Address `json:"address"` + PubKey crypto.PubKey `json:"pub_key"` + LastHeight int64 `json:"last_height"` + LastRound int `json:"last_round"` + LastStep int8 `json:"last_step"` + LastSignature crypto.Signature `json:"last_signature,omitempty"` // so we dont lose signatures XXX Why would we lose signatures? + LastSignBytes cmn.HexBytes `json:"last_signbytes,omitempty"` // so we dont lose signatures XXX Why would we lose signatures? + PrivKey crypto.PrivKey `json:"priv_key"` + + // For persistence. + // Overloaded for testing. + filePath string + mtx sync.Mutex +} + +// GetAddress returns the address of the validator. +// Implements PrivValidator. +func (pv *FilePV) GetAddress() types.Address { + return pv.Address +} + +// GetPubKey returns the public key of the validator. +// Implements PrivValidator. +func (pv *FilePV) GetPubKey() crypto.PubKey { + return pv.PubKey +} + +// GenFilePV generates a new validator with randomly generated private key +// and sets the filePath, but does not call Save(). +func GenFilePV(filePath string) *FilePV { + privKey := crypto.GenPrivKeyEd25519() + return &FilePV{ + Address: privKey.PubKey().Address(), + PubKey: privKey.PubKey(), + PrivKey: privKey, + LastStep: stepNone, + filePath: filePath, + } +} + +// LoadFilePV loads a FilePV from the filePath. The FilePV handles double +// signing prevention by persisting data to the filePath. If the filePath does +// not exist, the FilePV must be created manually and saved. +func LoadFilePV(filePath string) *FilePV { + pvJSONBytes, err := ioutil.ReadFile(filePath) + if err != nil { + cmn.Exit(err.Error()) + } + pv := &FilePV{} + err = cdc.UnmarshalJSON(pvJSONBytes, &pv) + if err != nil { + cmn.Exit(cmn.Fmt("Error reading PrivValidator from %v: %v\n", filePath, err)) + } + + pv.filePath = filePath + return pv +} + +// LoadOrGenFilePV loads a FilePV from the given filePath +// or else generates a new one and saves it to the filePath. +func LoadOrGenFilePV(filePath string) *FilePV { + var pv *FilePV + if cmn.FileExists(filePath) { + pv = LoadFilePV(filePath) + } else { + pv = GenFilePV(filePath) + pv.Save() + } + return pv +} + +// Save persists the FilePV to disk. +func (pv *FilePV) Save() { + pv.mtx.Lock() + defer pv.mtx.Unlock() + pv.save() +} + +func (pv *FilePV) save() { + outFile := pv.filePath + if outFile == "" { + panic("Cannot save PrivValidator: filePath not set") + } + jsonBytes, err := cdc.MarshalJSON(pv) + if err != nil { + panic(err) + } + err = cmn.WriteFileAtomic(outFile, jsonBytes, 0600) + if err != nil { + panic(err) + } +} + +// Reset resets all fields in the FilePV. +// NOTE: Unsafe! +func (pv *FilePV) Reset() { + var sig crypto.Signature + pv.LastHeight = 0 + pv.LastRound = 0 + pv.LastStep = 0 + pv.LastSignature = sig + pv.LastSignBytes = nil + pv.Save() +} + +// SignVote signs a canonical representation of the vote, along with the +// chainID. Implements PrivValidator. +func (pv *FilePV) SignVote(chainID string, vote *types.Vote) error { + pv.mtx.Lock() + defer pv.mtx.Unlock() + if err := pv.signVote(chainID, vote); err != nil { + return errors.New(cmn.Fmt("Error signing vote: %v", err)) + } + return nil +} + +// SignProposal signs a canonical representation of the proposal, along with +// the chainID. Implements PrivValidator. +func (pv *FilePV) SignProposal(chainID string, proposal *types.Proposal) error { + pv.mtx.Lock() + defer pv.mtx.Unlock() + if err := pv.signProposal(chainID, proposal); err != nil { + return fmt.Errorf("Error signing proposal: %v", err) + } + return nil +} + +// returns error if HRS regression or no LastSignBytes. returns true if HRS is unchanged +func (pv *FilePV) checkHRS(height int64, round int, step int8) (bool, error) { + if pv.LastHeight > height { + return false, errors.New("Height regression") + } + + if pv.LastHeight == height { + if pv.LastRound > round { + return false, errors.New("Round regression") + } + + if pv.LastRound == round { + if pv.LastStep > step { + return false, errors.New("Step regression") + } else if pv.LastStep == step { + if pv.LastSignBytes != nil { + if pv.LastSignature == nil { + panic("pv: LastSignature is nil but LastSignBytes is not!") + } + return true, nil + } + return false, errors.New("No LastSignature found") + } + } + } + return false, nil +} + +// signVote checks if the vote is good to sign and sets the vote signature. +// It may need to set the timestamp as well if the vote is otherwise the same as +// a previously signed vote (ie. we crashed after signing but before the vote hit the WAL). +func (pv *FilePV) signVote(chainID string, vote *types.Vote) error { + height, round, step := vote.Height, vote.Round, voteToStep(vote) + signBytes := vote.SignBytes(chainID) + + sameHRS, err := pv.checkHRS(height, round, step) + if err != nil { + return err + } + + // We might crash before writing to the wal, + // causing us to try to re-sign for the same HRS. + // If signbytes are the same, use the last signature. + // If they only differ by timestamp, use last timestamp and signature + // Otherwise, return error + if sameHRS { + if bytes.Equal(signBytes, pv.LastSignBytes) { + vote.Signature = pv.LastSignature + } else if timestamp, ok := checkVotesOnlyDifferByTimestamp(pv.LastSignBytes, signBytes); ok { + vote.Timestamp = timestamp + vote.Signature = pv.LastSignature + } else { + err = fmt.Errorf("Conflicting data") + } + return err + } + + // It passed the checks. Sign the vote + sig := pv.PrivKey.Sign(signBytes) + pv.saveSigned(height, round, step, signBytes, sig) + vote.Signature = sig + return nil +} + +// signProposal checks if the proposal is good to sign and sets the proposal signature. +// It may need to set the timestamp as well if the proposal is otherwise the same as +// a previously signed proposal ie. we crashed after signing but before the proposal hit the WAL). +func (pv *FilePV) signProposal(chainID string, proposal *types.Proposal) error { + height, round, step := proposal.Height, proposal.Round, stepPropose + signBytes := proposal.SignBytes(chainID) + + sameHRS, err := pv.checkHRS(height, round, step) + if err != nil { + return err + } + + // We might crash before writing to the wal, + // causing us to try to re-sign for the same HRS. + // If signbytes are the same, use the last signature. + // If they only differ by timestamp, use last timestamp and signature + // Otherwise, return error + if sameHRS { + if bytes.Equal(signBytes, pv.LastSignBytes) { + proposal.Signature = pv.LastSignature + } else if timestamp, ok := checkProposalsOnlyDifferByTimestamp(pv.LastSignBytes, signBytes); ok { + proposal.Timestamp = timestamp + proposal.Signature = pv.LastSignature + } else { + err = fmt.Errorf("Conflicting data") + } + return err + } + + // It passed the checks. Sign the proposal + sig := pv.PrivKey.Sign(signBytes) + pv.saveSigned(height, round, step, signBytes, sig) + proposal.Signature = sig + return nil +} + +// Persist height/round/step and signature +func (pv *FilePV) saveSigned(height int64, round int, step int8, + signBytes []byte, sig crypto.Signature) { + + pv.LastHeight = height + pv.LastRound = round + pv.LastStep = step + pv.LastSignature = sig + pv.LastSignBytes = signBytes + pv.save() +} + +// SignHeartbeat signs a canonical representation of the heartbeat, along with the chainID. +// Implements PrivValidator. +func (pv *FilePV) SignHeartbeat(chainID string, heartbeat *types.Heartbeat) error { + pv.mtx.Lock() + defer pv.mtx.Unlock() + heartbeat.Signature = pv.PrivKey.Sign(heartbeat.SignBytes(chainID)) + return nil +} + +// String returns a string representation of the FilePV. +func (pv *FilePV) String() string { + return fmt.Sprintf("PrivValidator{%v LH:%v, LR:%v, LS:%v}", pv.GetAddress(), pv.LastHeight, pv.LastRound, pv.LastStep) +} + +//------------------------------------- + +// returns the timestamp from the lastSignBytes. +// returns true if the only difference in the votes is their timestamp. +func checkVotesOnlyDifferByTimestamp(lastSignBytes, newSignBytes []byte) (time.Time, bool) { + var lastVote, newVote types.CanonicalJSONVote + if err := cdc.UnmarshalJSON(lastSignBytes, &lastVote); err != nil { + panic(fmt.Sprintf("LastSignBytes cannot be unmarshalled into vote: %v", err)) + } + if err := cdc.UnmarshalJSON(newSignBytes, &newVote); err != nil { + panic(fmt.Sprintf("signBytes cannot be unmarshalled into vote: %v", err)) + } + + lastTime, err := time.Parse(types.TimeFormat, lastVote.Timestamp) + if err != nil { + panic(err) + } + + // set the times to the same value and check equality + now := types.CanonicalTime(time.Now()) + lastVote.Timestamp = now + newVote.Timestamp = now + lastVoteBytes, _ := cdc.MarshalJSON(lastVote) + newVoteBytes, _ := cdc.MarshalJSON(newVote) + + return lastTime, bytes.Equal(newVoteBytes, lastVoteBytes) +} + +// returns the timestamp from the lastSignBytes. +// returns true if the only difference in the proposals is their timestamp +func checkProposalsOnlyDifferByTimestamp(lastSignBytes, newSignBytes []byte) (time.Time, bool) { + var lastProposal, newProposal types.CanonicalJSONProposal + if err := cdc.UnmarshalJSON(lastSignBytes, &lastProposal); err != nil { + panic(fmt.Sprintf("LastSignBytes cannot be unmarshalled into proposal: %v", err)) + } + if err := cdc.UnmarshalJSON(newSignBytes, &newProposal); err != nil { + panic(fmt.Sprintf("signBytes cannot be unmarshalled into proposal: %v", err)) + } + + lastTime, err := time.Parse(types.TimeFormat, lastProposal.Timestamp) + if err != nil { + panic(err) + } + + // set the times to the same value and check equality + now := types.CanonicalTime(time.Now()) + lastProposal.Timestamp = now + newProposal.Timestamp = now + lastProposalBytes, _ := cdc.MarshalJSON(lastProposal) + newProposalBytes, _ := cdc.MarshalJSON(newProposal) + + return lastTime, bytes.Equal(newProposalBytes, lastProposalBytes) +} diff --git a/types/priv_validator/priv_validator_test.go b/types/priv_validator/priv_validator_test.go index 120a0c86e..c7212d594 100644 --- a/types/priv_validator/priv_validator_test.go +++ b/types/priv_validator/priv_validator_test.go @@ -1,8 +1,7 @@ -package types +package privval import ( - "encoding/hex" - "encoding/json" + "encoding/base64" "fmt" "os" "testing" @@ -10,113 +9,89 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - crypto "github.com/tendermint/go-crypto" - cmn "github.com/tendermint/tmlibs/common" - + "github.com/tendermint/go-crypto" "github.com/tendermint/tendermint/types" + cmn "github.com/tendermint/tmlibs/common" ) func TestGenLoadValidator(t *testing.T) { - assert, require := assert.New(t), require.New(t) + assert := assert.New(t) _, tempFilePath := cmn.Tempfile("priv_validator_") - privVal := GenPrivValidatorJSON(tempFilePath) + privVal := GenFilePV(tempFilePath) height := int64(100) - privVal.LastSignedInfo.Height = height + privVal.LastHeight = height privVal.Save() - addr, err := privVal.Address() - require.Nil(err) + addr := privVal.GetAddress() - privVal = LoadPrivValidatorJSON(tempFilePath) - pAddr, err := privVal.Address() - require.Nil(err) - - assert.Equal(addr, pAddr, "expected privval addr to be the same") - assert.Equal(height, privVal.LastSignedInfo.Height, "expected privval.LastHeight to have been saved") + privVal = LoadFilePV(tempFilePath) + assert.Equal(addr, privVal.GetAddress(), "expected privval addr to be the same") + assert.Equal(height, privVal.LastHeight, "expected privval.LastHeight to have been saved") } func TestLoadOrGenValidator(t *testing.T) { - assert, require := assert.New(t), require.New(t) + assert := assert.New(t) _, tempFilePath := cmn.Tempfile("priv_validator_") if err := os.Remove(tempFilePath); err != nil { t.Error(err) } - privVal := LoadOrGenPrivValidatorJSON(tempFilePath) - addr, err := privVal.Address() - require.Nil(err) - - privVal = LoadOrGenPrivValidatorJSON(tempFilePath) - pAddr, err := privVal.Address() - require.Nil(err) - - assert.Equal(addr, pAddr, "expected privval addr to be the same") + privVal := LoadOrGenFilePV(tempFilePath) + addr := privVal.GetAddress() + privVal = LoadOrGenFilePV(tempFilePath) + assert.Equal(addr, privVal.GetAddress(), "expected privval addr to be the same") } func TestUnmarshalValidator(t *testing.T) { assert, require := assert.New(t), require.New(t) // create some fixed values - addrStr := "D028C9981F7A87F3093672BF0D5B0E2A1B3ED456" - pubStr := "3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8" - privStr := "27F82582AEFAE7AB151CFB01C48BB6C1A0DA78F9BDDA979A9F70A84D074EB07D3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8" - addrBytes, _ := hex.DecodeString(addrStr) - pubBytes, _ := hex.DecodeString(pubStr) - privBytes, _ := hex.DecodeString(privStr) - - // prepend type byte - pubKey, err := crypto.PubKeyFromBytes(append([]byte{1}, pubBytes...)) - require.Nil(err, "%+v", err) - privKey, err := crypto.PrivKeyFromBytes(append([]byte{1}, privBytes...)) - require.Nil(err, "%+v", err) + privKey := crypto.GenPrivKeyEd25519() + pubKey := privKey.PubKey() + addr := pubKey.Address() + pubArray := [32]byte(pubKey.(crypto.PubKeyEd25519)) + pubBytes := pubArray[:] + privArray := [64]byte(privKey) + privBytes := privArray[:] + pubB64 := base64.StdEncoding.EncodeToString(pubBytes) + privB64 := base64.StdEncoding.EncodeToString(privBytes) serialized := fmt.Sprintf(`{ - "id": { - "address": "%s", - "pub_key": { - "type": "ed25519", - "data": "%s" - } - }, - "priv_key": { - "type": "ed25519", - "data": "%s" - }, - "last_signed_info": { - "height": 0, - "round": 0, - "step": 0, - "signature": null - } -}`, addrStr, pubStr, privStr) + "address": "%s", + "pub_key": { + "type": "AC26791624DE60", + "value": "%s" + }, + "last_height": 0, + "last_round": 0, + "last_step": 0, + "priv_key": { + "type": "954568A3288910", + "value": "%s" + } +}`, addr, pubB64, privB64) - val := PrivValidatorJSON{} - err = json.Unmarshal([]byte(serialized), &val) + val := FilePV{} + err := cdc.UnmarshalJSON([]byte(serialized), &val) require.Nil(err, "%+v", err) // make sure the values match - vAddr, err := val.Address() - require.Nil(err) - - pKey, err := val.PubKey() - require.Nil(err) - - assert.EqualValues(addrBytes, vAddr) - assert.EqualValues(pubKey, pKey) + assert.EqualValues(addr, val.GetAddress()) + assert.EqualValues(pubKey, val.GetPubKey()) assert.EqualValues(privKey, val.PrivKey) // export it and make sure it is the same - out, err := json.Marshal(val) + out, err := cdc.MarshalJSON(val) require.Nil(err, "%+v", err) assert.JSONEq(serialized, string(out)) } func TestSignVote(t *testing.T) { - assert, require := assert.New(t), require.New(t) + assert := assert.New(t) _, tempFilePath := cmn.Tempfile("priv_validator_") - privVal := GenPrivValidatorJSON(tempFilePath) + privVal := GenFilePV(tempFilePath) block1 := types.BlockID{[]byte{1, 2, 3}, types.PartSetHeader{}} block2 := types.BlockID{[]byte{3, 2, 1}, types.PartSetHeader{}} @@ -124,11 +99,8 @@ func TestSignVote(t *testing.T) { voteType := types.VoteTypePrevote // sign a vote for first time - addr, err := privVal.Address() - require.Nil(err) - - vote := newVote(addr, 0, height, round, voteType, block1) - err = privVal.SignVote("mychainid", vote) + vote := newVote(privVal.Address, 0, height, round, voteType, block1) + err := privVal.SignVote("mychainid", vote) assert.NoError(err, "expected no error signing vote") // try to sign the same vote again; should be fine @@ -137,10 +109,10 @@ func TestSignVote(t *testing.T) { // now try some bad votes cases := []*types.Vote{ - newVote(addr, 0, height, round-1, voteType, block1), // round regression - newVote(addr, 0, height-1, round, voteType, block1), // height regression - newVote(addr, 0, height-2, round+4, voteType, block1), // height regression and different round - newVote(addr, 0, height, round, voteType, block2), // different block + newVote(privVal.Address, 0, height, round-1, voteType, block1), // round regression + newVote(privVal.Address, 0, height-1, round, voteType, block1), // height regression + newVote(privVal.Address, 0, height-2, round+4, voteType, block1), // height regression and different round + newVote(privVal.Address, 0, height, round, voteType, block2), // different block } for _, c := range cases { @@ -160,7 +132,7 @@ func TestSignProposal(t *testing.T) { assert := assert.New(t) _, tempFilePath := cmn.Tempfile("priv_validator_") - privVal := GenPrivValidatorJSON(tempFilePath) + privVal := GenFilePV(tempFilePath) block1 := types.PartSetHeader{5, []byte{1, 2, 3}} block2 := types.PartSetHeader{10, []byte{3, 2, 1}} @@ -197,10 +169,8 @@ func TestSignProposal(t *testing.T) { } func TestDifferByTimestamp(t *testing.T) { - require := require.New(t) - _, tempFilePath := cmn.Tempfile("priv_validator_") - privVal := GenPrivValidatorJSON(tempFilePath) + privVal := GenFilePV(tempFilePath) block1 := types.PartSetHeader{5, []byte{1, 2, 3}} height, round := int64(10), 1 @@ -217,7 +187,8 @@ func TestDifferByTimestamp(t *testing.T) { // manipulate the timestamp. should get changed back proposal.Timestamp = proposal.Timestamp.Add(time.Millisecond) - proposal.Signature = crypto.Signature{} + var emptySig crypto.Signature + proposal.Signature = emptySig err = privVal.SignProposal("mychainid", proposal) assert.NoError(t, err, "expected no error on signing same proposal") @@ -228,13 +199,10 @@ func TestDifferByTimestamp(t *testing.T) { // test vote { - addr, err := privVal.Address() - require.Nil(err) - voteType := types.VoteTypePrevote blockID := types.BlockID{[]byte{1, 2, 3}, types.PartSetHeader{}} - vote := newVote(addr, 0, height, round, voteType, blockID) - err = privVal.SignVote("mychainid", vote) + vote := newVote(privVal.Address, 0, height, round, voteType, blockID) + err := privVal.SignVote("mychainid", vote) assert.NoError(t, err, "expected no error signing vote") signBytes := vote.SignBytes(chainID) @@ -243,7 +211,8 @@ func TestDifferByTimestamp(t *testing.T) { // manipulate the timestamp. should get changed back vote.Timestamp = vote.Timestamp.Add(time.Millisecond) - vote.Signature = crypto.Signature{} + var emptySig crypto.Signature + vote.Signature = emptySig err = privVal.SignVote("mychainid", vote) assert.NoError(t, err, "expected no error on signing same vote") @@ -253,7 +222,7 @@ func TestDifferByTimestamp(t *testing.T) { } } -func newVote(addr cmn.HexBytes, idx int, height int64, round int, typ byte, blockID types.BlockID) *types.Vote { +func newVote(addr types.Address, idx int, height int64, round int, typ byte, blockID types.BlockID) *types.Vote { return &types.Vote{ ValidatorAddress: addr, ValidatorIndex: idx, diff --git a/types/priv_validator/sign_info.go b/types/priv_validator/sign_info.go deleted file mode 100644 index 746131a96..000000000 --- a/types/priv_validator/sign_info.go +++ /dev/null @@ -1,238 +0,0 @@ -package types - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "time" - - crypto "github.com/tendermint/go-crypto" - "github.com/tendermint/tendermint/types" - cmn "github.com/tendermint/tmlibs/common" -) - -// TODO: type ? -const ( - stepNone int8 = 0 // Used to distinguish the initial state - stepPropose int8 = 1 - stepPrevote int8 = 2 - stepPrecommit int8 = 3 -) - -func voteToStep(vote *types.Vote) int8 { - switch vote.Type { - case types.VoteTypePrevote: - return stepPrevote - case types.VoteTypePrecommit: - return stepPrecommit - default: - panic("Unknown vote type") - } -} - -//------------------------------------- - -// LastSignedInfo contains information about the latest -// data signed by a validator to help prevent double signing. -type LastSignedInfo struct { - Height int64 `json:"height"` - Round int `json:"round"` - Step int8 `json:"step"` - Signature crypto.Signature `json:"signature,omitempty"` // so we dont lose signatures - SignBytes cmn.HexBytes `json:"signbytes,omitempty"` // so we dont lose signatures -} - -func NewLastSignedInfo() *LastSignedInfo { - return &LastSignedInfo{ - Step: stepNone, - } -} - -func (info *LastSignedInfo) String() string { - return fmt.Sprintf("LH:%v, LR:%v, LS:%v", info.Height, info.Round, info.Step) -} - -// Verify returns an error if there is a height/round/step regression -// or if the HRS matches but there are no LastSignBytes. -// It returns true if HRS matches exactly and the LastSignature exists. -// It panics if the HRS matches, the LastSignBytes are not empty, but the LastSignature is empty. -func (info LastSignedInfo) Verify(height int64, round int, step int8) (bool, error) { - if info.Height > height { - return false, errors.New("Height regression") - } - - if info.Height == height { - if info.Round > round { - return false, errors.New("Round regression") - } - - if info.Round == round { - if info.Step > step { - return false, errors.New("Step regression") - } else if info.Step == step { - if info.SignBytes != nil { - if info.Signature.Empty() { - panic("info: LastSignature is nil but LastSignBytes is not!") - } - return true, nil - } - return false, errors.New("No LastSignature found") - } - } - } - return false, nil -} - -// Set height/round/step and signature on the info -func (info *LastSignedInfo) Set(height int64, round int, step int8, - signBytes []byte, sig crypto.Signature) { - - info.Height = height - info.Round = round - info.Step = step - info.Signature = sig - info.SignBytes = signBytes -} - -// Reset resets all the values. -// XXX: Unsafe. -func (info *LastSignedInfo) Reset() { - info.Height = 0 - info.Round = 0 - info.Step = 0 - info.Signature = crypto.Signature{} - info.SignBytes = nil -} - -// SignVote checks the height/round/step (HRS) are greater than the latest state of the LastSignedInfo. -// If so, it signs the vote, updates the LastSignedInfo, and sets the signature on the vote. -// If the HRS are equal and the only thing changed is the timestamp, it sets the vote.Timestamp to the previous -// value and the Signature to the LastSignedInfo.Signature. -// Else it returns an error. -func (lsi *LastSignedInfo) SignVote(signer types.Signer, chainID string, vote *types.Vote) error { - height, round, step := vote.Height, vote.Round, voteToStep(vote) - signBytes := vote.SignBytes(chainID) - - sameHRS, err := lsi.Verify(height, round, step) - if err != nil { - return err - } - - // We might crash before writing to the wal, - // causing us to try to re-sign for the same HRS. - // If signbytes are the same, use the last signature. - // If they only differ by timestamp, use last timestamp and signature - // Otherwise, return error - if sameHRS { - if bytes.Equal(signBytes, lsi.SignBytes) { - vote.Signature = lsi.Signature - } else if timestamp, ok := checkVotesOnlyDifferByTimestamp(lsi.SignBytes, signBytes); ok { - vote.Timestamp = timestamp - vote.Signature = lsi.Signature - } else { - err = fmt.Errorf("Conflicting data") - } - return err - } - sig, err := signer.Sign(signBytes) - if err != nil { - return err - } - lsi.Set(height, round, step, signBytes, sig) - vote.Signature = sig - return nil -} - -// SignProposal checks if the height/round/step (HRS) are greater than the latest state of the LastSignedInfo. -// If so, it signs the proposal, updates the LastSignedInfo, and sets the signature on the proposal. -// If the HRS are equal and the only thing changed is the timestamp, it sets the timestamp to the previous -// value and the Signature to the LastSignedInfo.Signature. -// Else it returns an error. -func (lsi *LastSignedInfo) SignProposal(signer types.Signer, chainID string, proposal *types.Proposal) error { - height, round, step := proposal.Height, proposal.Round, stepPropose - signBytes := proposal.SignBytes(chainID) - - sameHRS, err := lsi.Verify(height, round, step) - if err != nil { - return err - } - - // We might crash before writing to the wal, - // causing us to try to re-sign for the same HRS. - // If signbytes are the same, use the last signature. - // If they only differ by timestamp, use last timestamp and signature - // Otherwise, return error - if sameHRS { - if bytes.Equal(signBytes, lsi.SignBytes) { - proposal.Signature = lsi.Signature - } else if timestamp, ok := checkProposalsOnlyDifferByTimestamp(lsi.SignBytes, signBytes); ok { - proposal.Timestamp = timestamp - proposal.Signature = lsi.Signature - } else { - err = fmt.Errorf("Conflicting data") - } - return err - } - sig, err := signer.Sign(signBytes) - if err != nil { - return err - } - lsi.Set(height, round, step, signBytes, sig) - proposal.Signature = sig - return nil -} - -//------------------------------------- - -// returns the timestamp from the lastSignBytes. -// returns true if the only difference in the votes is their timestamp. -func checkVotesOnlyDifferByTimestamp(lastSignBytes, newSignBytes []byte) (time.Time, bool) { - var lastVote, newVote types.CanonicalJSONOnceVote - if err := json.Unmarshal(lastSignBytes, &lastVote); err != nil { - panic(fmt.Sprintf("LastSignBytes cannot be unmarshalled into vote: %v", err)) - } - if err := json.Unmarshal(newSignBytes, &newVote); err != nil { - panic(fmt.Sprintf("signBytes cannot be unmarshalled into vote: %v", err)) - } - - lastTime, err := time.Parse(types.TimeFormat, lastVote.Vote.Timestamp) - if err != nil { - panic(err) - } - - // set the times to the same value and check equality - now := types.CanonicalTime(time.Now()) - lastVote.Vote.Timestamp = now - newVote.Vote.Timestamp = now - lastVoteBytes, _ := json.Marshal(lastVote) - newVoteBytes, _ := json.Marshal(newVote) - - return lastTime, bytes.Equal(newVoteBytes, lastVoteBytes) -} - -// returns the timestamp from the lastSignBytes. -// returns true if the only difference in the proposals is their timestamp -func checkProposalsOnlyDifferByTimestamp(lastSignBytes, newSignBytes []byte) (time.Time, bool) { - var lastProposal, newProposal types.CanonicalJSONOnceProposal - if err := json.Unmarshal(lastSignBytes, &lastProposal); err != nil { - panic(fmt.Sprintf("LastSignBytes cannot be unmarshalled into proposal: %v", err)) - } - if err := json.Unmarshal(newSignBytes, &newProposal); err != nil { - panic(fmt.Sprintf("signBytes cannot be unmarshalled into proposal: %v", err)) - } - - lastTime, err := time.Parse(types.TimeFormat, lastProposal.Proposal.Timestamp) - if err != nil { - panic(err) - } - - // set the times to the same value and check equality - now := types.CanonicalTime(time.Now()) - lastProposal.Proposal.Timestamp = now - newProposal.Proposal.Timestamp = now - lastProposalBytes, _ := json.Marshal(lastProposal) - newProposalBytes, _ := json.Marshal(newProposal) - - return lastTime, bytes.Equal(newProposalBytes, lastProposalBytes) -} diff --git a/types/priv_validator/socket.go b/types/priv_validator/socket.go index 26cab72b9..9f59a8152 100644 --- a/types/priv_validator/socket.go +++ b/types/priv_validator/socket.go @@ -1,14 +1,14 @@ -package types +package privval import ( + "errors" "fmt" "io" "net" "time" - "github.com/pkg/errors" - crypto "github.com/tendermint/go-crypto" - wire "github.com/tendermint/go-wire" + "github.com/tendermint/go-amino" + "github.com/tendermint/go-crypto" cmn "github.com/tendermint/tmlibs/common" "github.com/tendermint/tmlibs/log" @@ -37,36 +37,36 @@ var ( connHeartbeat = time.Second * defaultConnHeartBeatSeconds ) -// SocketClientOption sets an optional parameter on the SocketClient. -type SocketClientOption func(*SocketClient) +// SocketPVOption sets an optional parameter on the SocketPV. +type SocketPVOption func(*SocketPV) -// SocketClientAcceptDeadline sets the deadline for the SocketClient listener. +// SocketPVAcceptDeadline sets the deadline for the SocketPV listener. // A zero time value disables the deadline. -func SocketClientAcceptDeadline(deadline time.Duration) SocketClientOption { - return func(sc *SocketClient) { sc.acceptDeadline = deadline } +func SocketPVAcceptDeadline(deadline time.Duration) SocketPVOption { + return func(sc *SocketPV) { sc.acceptDeadline = deadline } } -// SocketClientConnDeadline sets the read and write deadline for connections +// SocketPVConnDeadline sets the read and write deadline for connections // from external signing processes. -func SocketClientConnDeadline(deadline time.Duration) SocketClientOption { - return func(sc *SocketClient) { sc.connDeadline = deadline } +func SocketPVConnDeadline(deadline time.Duration) SocketPVOption { + return func(sc *SocketPV) { sc.connDeadline = deadline } } -// SocketClientHeartbeat sets the period on which to check the liveness of the +// SocketPVHeartbeat sets the period on which to check the liveness of the // connected Signer connections. -func SocketClientHeartbeat(period time.Duration) SocketClientOption { - return func(sc *SocketClient) { sc.connHeartbeat = period } +func SocketPVHeartbeat(period time.Duration) SocketPVOption { + return func(sc *SocketPV) { sc.connHeartbeat = period } } -// SocketClientConnWait sets the timeout duration before connection of external +// SocketPVConnWait sets the timeout duration before connection of external // signing processes are considered to be unsuccessful. -func SocketClientConnWait(timeout time.Duration) SocketClientOption { - return func(sc *SocketClient) { sc.connWaitTimeout = timeout } +func SocketPVConnWait(timeout time.Duration) SocketPVOption { + return func(sc *SocketPV) { sc.connWaitTimeout = timeout } } -// SocketClient implements PrivValidator, it uses a socket to request signatures +// SocketPV implements PrivValidator, it uses a socket to request signatures // from an external process. -type SocketClient struct { +type SocketPV struct { cmn.BaseService addr string @@ -80,16 +80,16 @@ type SocketClient struct { listener net.Listener } -// Check that SocketClient implements PrivValidator2. -var _ types.PrivValidator2 = (*SocketClient)(nil) +// Check that SocketPV implements PrivValidator. +var _ types.PrivValidator = (*SocketPV)(nil) -// NewSocketClient returns an instance of SocketClient. -func NewSocketClient( +// NewSocketPV returns an instance of SocketPV. +func NewSocketPV( logger log.Logger, socketAddr string, privKey crypto.PrivKeyEd25519, -) *SocketClient { - sc := &SocketClient{ +) *SocketPV { + sc := &SocketPV{ addr: socketAddr, acceptDeadline: acceptDeadline, connDeadline: connDeadline, @@ -98,15 +98,14 @@ func NewSocketClient( privKey: privKey, } - sc.BaseService = *cmn.NewBaseService(logger, "SocketClient", sc) + sc.BaseService = *cmn.NewBaseService(logger, "SocketPV", sc) return sc } // GetAddress implements PrivValidator. -// TODO(xla): Remove when PrivValidator2 replaced PrivValidator. -func (sc *SocketClient) GetAddress() types.Address { - addr, err := sc.Address() +func (sc *SocketPV) GetAddress() types.Address { + addr, err := sc.getAddress() if err != nil { panic(err) } @@ -115,8 +114,8 @@ func (sc *SocketClient) GetAddress() types.Address { } // Address is an alias for PubKey().Address(). -func (sc *SocketClient) Address() (cmn.HexBytes, error) { - p, err := sc.PubKey() +func (sc *SocketPV) getAddress() (cmn.HexBytes, error) { + p, err := sc.getPubKey() if err != nil { return nil, err } @@ -125,9 +124,8 @@ func (sc *SocketClient) Address() (cmn.HexBytes, error) { } // GetPubKey implements PrivValidator. -// TODO(xla): Remove when PrivValidator2 replaced PrivValidator. -func (sc *SocketClient) GetPubKey() crypto.PubKey { - pubKey, err := sc.PubKey() +func (sc *SocketPV) GetPubKey() crypto.PubKey { + pubKey, err := sc.getPubKey() if err != nil { panic(err) } @@ -135,23 +133,22 @@ func (sc *SocketClient) GetPubKey() crypto.PubKey { return pubKey } -// PubKey implements PrivValidator2. -func (sc *SocketClient) PubKey() (crypto.PubKey, error) { +func (sc *SocketPV) getPubKey() (crypto.PubKey, error) { err := writeMsg(sc.conn, &PubKeyMsg{}) if err != nil { - return crypto.PubKey{}, err + return nil, err } res, err := readMsg(sc.conn) if err != nil { - return crypto.PubKey{}, err + return nil, err } return res.(*PubKeyMsg).PubKey, nil } -// SignVote implements PrivValidator2. -func (sc *SocketClient) SignVote(chainID string, vote *types.Vote) error { +// SignVote implements PrivValidator. +func (sc *SocketPV) SignVote(chainID string, vote *types.Vote) error { err := writeMsg(sc.conn, &SignVoteMsg{Vote: vote}) if err != nil { return err @@ -167,8 +164,8 @@ func (sc *SocketClient) SignVote(chainID string, vote *types.Vote) error { return nil } -// SignProposal implements PrivValidator2. -func (sc *SocketClient) SignProposal( +// SignProposal implements PrivValidator. +func (sc *SocketPV) SignProposal( chainID string, proposal *types.Proposal, ) error { @@ -187,8 +184,8 @@ func (sc *SocketClient) SignProposal( return nil } -// SignHeartbeat implements PrivValidator2. -func (sc *SocketClient) SignHeartbeat( +// SignHeartbeat implements PrivValidator. +func (sc *SocketPV) SignHeartbeat( chainID string, heartbeat *types.Heartbeat, ) error { @@ -208,21 +205,22 @@ func (sc *SocketClient) SignHeartbeat( } // OnStart implements cmn.Service. -func (sc *SocketClient) OnStart() error { +func (sc *SocketPV) OnStart() error { if err := sc.listen(); err != nil { + err = cmn.ErrorWrap(err, "failed to listen") sc.Logger.Error( "OnStart", - "err", errors.Wrap(err, "failed to listen"), + "err", err, ) - return err } conn, err := sc.waitConnection() if err != nil { + err = cmn.ErrorWrap(err, "failed to accept connection") sc.Logger.Error( "OnStart", - "err", errors.Wrap(err, "failed to accept connection"), + "err", err, ) return err @@ -234,27 +232,29 @@ func (sc *SocketClient) OnStart() error { } // OnStop implements cmn.Service. -func (sc *SocketClient) OnStop() { +func (sc *SocketPV) OnStop() { if sc.conn != nil { if err := sc.conn.Close(); err != nil { + err = cmn.ErrorWrap(err, "failed to close connection") sc.Logger.Error( "OnStop", - "err", errors.Wrap(err, "failed to close connection"), + "err", err, ) } } if sc.listener != nil { if err := sc.listener.Close(); err != nil { + err = cmn.ErrorWrap(err, "failed to close listener") sc.Logger.Error( "OnStop", - "err", errors.Wrap(err, "failed to close listener"), + "err", err, ) } } } -func (sc *SocketClient) acceptConnection() (net.Conn, error) { +func (sc *SocketPV) acceptConnection() (net.Conn, error) { conn, err := sc.listener.Accept() if err != nil { if !sc.IsRunning() { @@ -264,7 +264,7 @@ func (sc *SocketClient) acceptConnection() (net.Conn, error) { } - conn, err = p2pconn.MakeSecretConnection(conn, sc.privKey.Wrap()) + conn, err = p2pconn.MakeSecretConnection(conn, sc.privKey) if err != nil { return nil, err } @@ -272,7 +272,7 @@ func (sc *SocketClient) acceptConnection() (net.Conn, error) { return conn, nil } -func (sc *SocketClient) listen() error { +func (sc *SocketPV) listen() error { ln, err := net.Listen(cmn.ProtocolAndAddress(sc.addr)) if err != nil { return err @@ -290,7 +290,7 @@ func (sc *SocketClient) listen() error { // waitConnection uses the configured wait timeout to error if no external // process connects in the time period. -func (sc *SocketClient) waitConnection() (net.Conn, error) { +func (sc *SocketPV) waitConnection() (net.Conn, error) { var ( connc = make(chan net.Conn, 1) errc = make(chan error, 1) @@ -311,7 +311,7 @@ func (sc *SocketClient) waitConnection() (net.Conn, error) { return conn, nil case err := <-errc: if _, ok := err.(timeoutError); ok { - return nil, errors.Wrap(ErrConnWaitTimeout, err.Error()) + return nil, cmn.ErrorWrap(ErrConnWaitTimeout, err.Error()) } return nil, err case <-time.After(sc.connWaitTimeout): @@ -344,7 +344,7 @@ type RemoteSigner struct { connDeadline time.Duration connRetries int privKey crypto.PrivKeyEd25519 - privVal PrivValidator + privVal types.PrivValidator conn net.Conn } @@ -353,7 +353,7 @@ type RemoteSigner struct { func NewRemoteSigner( logger log.Logger, chainID, socketAddr string, - privVal PrivValidator, + privVal types.PrivValidator, privKey crypto.PrivKeyEd25519, ) *RemoteSigner { rs := &RemoteSigner{ @@ -374,8 +374,8 @@ func NewRemoteSigner( func (rs *RemoteSigner) OnStart() error { conn, err := rs.connect() if err != nil { - rs.Logger.Error("OnStart", "err", errors.Wrap(err, "connect")) - + err = cmn.ErrorWrap(err, "connect") + rs.Logger.Error("OnStart", "err", err) return err } @@ -391,7 +391,7 @@ func (rs *RemoteSigner) OnStop() { } if err := rs.conn.Close(); err != nil { - rs.Logger.Error("OnStop", "err", errors.Wrap(err, "closing listener failed")) + rs.Logger.Error("OnStop", "err", cmn.ErrorWrap(err, "closing listener failed")) } } @@ -404,28 +404,31 @@ func (rs *RemoteSigner) connect() (net.Conn, error) { conn, err := cmn.Connect(rs.addr) if err != nil { + err = cmn.ErrorWrap(err, "connection failed") rs.Logger.Error( "connect", "addr", rs.addr, - "err", errors.Wrap(err, "connection failed"), + "err", err, ) continue } if err := conn.SetDeadline(time.Now().Add(connDeadline)); err != nil { + err = cmn.ErrorWrap(err, "setting connection timeout failed") rs.Logger.Error( "connect", - "err", errors.Wrap(err, "setting connection timeout failed"), + "err", err, ) continue } - conn, err = p2pconn.MakeSecretConnection(conn, rs.privKey.Wrap()) + conn, err = p2pconn.MakeSecretConnection(conn, rs.privKey) if err != nil { + err = cmn.ErrorWrap(err, "encrypting connection failed") rs.Logger.Error( "connect", - "err", errors.Wrap(err, "encrypting connection failed"), + "err", err, ) continue @@ -451,13 +454,12 @@ func (rs *RemoteSigner) handleConnection(conn net.Conn) { return } - var res PrivValMsg + var res SocketPVMsg switch r := req.(type) { case *PubKeyMsg: var p crypto.PubKey - - p, err = rs.privVal.PubKey() + p = rs.privVal.GetPubKey() res = &PubKeyMsg{p} case *SignVoteMsg: err = rs.privVal.SignVote(rs.chainID, r.Vote) @@ -487,23 +489,16 @@ func (rs *RemoteSigner) handleConnection(conn net.Conn) { //--------------------------------------------------------- -const ( - msgTypePubKey = byte(0x01) - msgTypeSignVote = byte(0x10) - msgTypeSignProposal = byte(0x11) - msgTypeSignHeartbeat = byte(0x12) -) +// SocketPVMsg is sent between RemoteSigner and SocketPV. +type SocketPVMsg interface{} -// PrivValMsg is sent between RemoteSigner and SocketClient. -type PrivValMsg interface{} - -var _ = wire.RegisterInterface( - struct{ PrivValMsg }{}, - wire.ConcreteType{&PubKeyMsg{}, msgTypePubKey}, - wire.ConcreteType{&SignVoteMsg{}, msgTypeSignVote}, - wire.ConcreteType{&SignProposalMsg{}, msgTypeSignProposal}, - wire.ConcreteType{&SignHeartbeatMsg{}, msgTypeSignHeartbeat}, -) +func RegisterSocketPVMsg(cdc *amino.Codec) { + cdc.RegisterInterface((*SocketPVMsg)(nil), nil) + cdc.RegisterConcrete(&PubKeyMsg{}, "tendermint/socketpv/PubKeyMsg", nil) + cdc.RegisterConcrete(&SignVoteMsg{}, "tendermint/socketpv/SignVoteMsg", nil) + cdc.RegisterConcrete(&SignProposalMsg{}, "tendermint/socketpv/SignProposalMsg", nil) + cdc.RegisterConcrete(&SignHeartbeatMsg{}, "tendermint/socketpv/SignHeartbeatMsg", nil) +} // PubKeyMsg is a PrivValidatorSocket message containing the public key. type PubKeyMsg struct { @@ -525,40 +520,19 @@ type SignHeartbeatMsg struct { Heartbeat *types.Heartbeat } -func readMsg(r io.Reader) (PrivValMsg, error) { - var ( - n int - err error - ) - - read := wire.ReadBinary(struct{ PrivValMsg }{}, r, 0, &n, &err) - if err != nil { - if _, ok := err.(timeoutError); ok { - return nil, errors.Wrap(ErrConnTimeout, err.Error()) - } - - return nil, err - } - - w, ok := read.(struct{ PrivValMsg }) - if !ok { - return nil, errors.New("unknown type") - } - - return w.PrivValMsg, nil -} - -func writeMsg(w io.Writer, msg interface{}) error { - var ( - err error - n int - ) - - // TODO(xla): This extra wrap should be gone with the sdk-2 update. - wire.WriteBinary(struct{ PrivValMsg }{msg}, w, &n, &err) +func readMsg(r io.Reader) (msg SocketPVMsg, err error) { + const maxSocketPVMsgSize = 1024 * 10 + _, err = cdc.UnmarshalBinaryReader(r, &msg, maxSocketPVMsgSize) if _, ok := err.(timeoutError); ok { - return errors.Wrap(ErrConnTimeout, err.Error()) + err = cmn.ErrorWrap(ErrConnTimeout, err.Error()) } - - return err + return +} + +func writeMsg(w io.Writer, msg interface{}) (err error) { + _, err = cdc.MarshalBinaryWriter(w, msg) + if _, ok := err.(timeoutError); ok { + err = cmn.ErrorWrap(ErrConnTimeout, err.Error()) + } + return } diff --git a/types/priv_validator/socket_tcp.go b/types/priv_validator/socket_tcp.go index 2421eb9f4..b26db00c2 100644 --- a/types/priv_validator/socket_tcp.go +++ b/types/priv_validator/socket_tcp.go @@ -1,4 +1,4 @@ -package types +package privval import ( "net" diff --git a/types/priv_validator/socket_tcp_test.go b/types/priv_validator/socket_tcp_test.go index cd95ab0b9..44a673c0c 100644 --- a/types/priv_validator/socket_tcp_test.go +++ b/types/priv_validator/socket_tcp_test.go @@ -1,4 +1,4 @@ -package types +package privval import ( "net" diff --git a/types/priv_validator/socket_test.go b/types/priv_validator/socket_test.go index 2859c9452..eb0f4c12d 100644 --- a/types/priv_validator/socket_test.go +++ b/types/priv_validator/socket_test.go @@ -1,4 +1,4 @@ -package types +package privval import ( "fmt" @@ -6,11 +6,10 @@ import ( "testing" "time" - "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - crypto "github.com/tendermint/go-crypto" + "github.com/tendermint/go-crypto" cmn "github.com/tendermint/tmlibs/common" "github.com/tendermint/tmlibs/log" @@ -18,7 +17,7 @@ import ( "github.com/tendermint/tendermint/types" ) -func TestSocketClientAddress(t *testing.T) { +func TestSocketPVAddress(t *testing.T) { var ( chainID = cmn.RandStr(12) sc, rs = testSetupSocketPair(t, chainID) @@ -26,10 +25,9 @@ func TestSocketClientAddress(t *testing.T) { defer sc.Stop() defer rs.Stop() - serverAddr, err := rs.privVal.Address() - require.NoError(t, err) + serverAddr := rs.privVal.GetAddress() - clientAddr, err := sc.Address() + clientAddr, err := sc.getAddress() require.NoError(t, err) assert.Equal(t, serverAddr, clientAddr) @@ -39,7 +37,7 @@ func TestSocketClientAddress(t *testing.T) { } -func TestSocketClientPubKey(t *testing.T) { +func TestSocketPVPubKey(t *testing.T) { var ( chainID = cmn.RandStr(12) sc, rs = testSetupSocketPair(t, chainID) @@ -47,11 +45,10 @@ func TestSocketClientPubKey(t *testing.T) { defer sc.Stop() defer rs.Stop() - clientKey, err := sc.PubKey() + clientKey, err := sc.getPubKey() require.NoError(t, err) - privKey, err := rs.privVal.PubKey() - require.NoError(t, err) + privKey := rs.privVal.GetPubKey() assert.Equal(t, privKey, clientKey) @@ -59,7 +56,7 @@ func TestSocketClientPubKey(t *testing.T) { assert.Equal(t, privKey, sc.GetPubKey()) } -func TestSocketClientProposal(t *testing.T) { +func TestSocketPVProposal(t *testing.T) { var ( chainID = cmn.RandStr(12) sc, rs = testSetupSocketPair(t, chainID) @@ -76,7 +73,7 @@ func TestSocketClientProposal(t *testing.T) { assert.Equal(t, privProposal.Signature, clientProposal.Signature) } -func TestSocketClientVote(t *testing.T) { +func TestSocketPVVote(t *testing.T) { var ( chainID = cmn.RandStr(12) sc, rs = testSetupSocketPair(t, chainID) @@ -94,7 +91,7 @@ func TestSocketClientVote(t *testing.T) { assert.Equal(t, want.Signature, have.Signature) } -func TestSocketClientHeartbeat(t *testing.T) { +func TestSocketPVHeartbeat(t *testing.T) { var ( chainID = cmn.RandStr(12) sc, rs = testSetupSocketPair(t, chainID) @@ -110,9 +107,9 @@ func TestSocketClientHeartbeat(t *testing.T) { assert.Equal(t, want.Signature, have.Signature) } -func TestSocketClientAcceptDeadline(t *testing.T) { +func TestSocketPVAcceptDeadline(t *testing.T) { var ( - sc = NewSocketClient( + sc = NewSocketPV( log.TestingLogger(), "127.0.0.1:0", crypto.GenPrivKeyEd25519(), @@ -120,26 +117,26 @@ func TestSocketClientAcceptDeadline(t *testing.T) { ) defer sc.Stop() - SocketClientAcceptDeadline(time.Millisecond)(sc) + SocketPVAcceptDeadline(time.Millisecond)(sc) - assert.Equal(t, errors.Cause(sc.Start()), ErrConnWaitTimeout) + assert.Equal(t, sc.Start().(cmn.Error).Cause(), ErrConnWaitTimeout) } -func TestSocketClientDeadline(t *testing.T) { +func TestSocketPVDeadline(t *testing.T) { var ( addr = testFreeAddr(t) listenc = make(chan struct{}) - sc = NewSocketClient( + sc = NewSocketPV( log.TestingLogger(), addr, crypto.GenPrivKeyEd25519(), ) ) - SocketClientConnDeadline(10 * time.Millisecond)(sc) - SocketClientConnWait(500 * time.Millisecond)(sc) + SocketPVConnDeadline(10 * time.Millisecond)(sc) + SocketPVConnWait(500 * time.Millisecond)(sc) - go func(sc *SocketClient) { + go func(sc *SocketPV) { defer close(listenc) require.NoError(t, sc.Start()) @@ -155,7 +152,7 @@ func TestSocketClientDeadline(t *testing.T) { _, err = p2pconn.MakeSecretConnection( conn, - crypto.GenPrivKeyEd25519().Wrap(), + crypto.GenPrivKeyEd25519(), ) if err == nil { break @@ -167,21 +164,21 @@ func TestSocketClientDeadline(t *testing.T) { // Sleep to guarantee deadline has been hit. time.Sleep(20 * time.Microsecond) - _, err := sc.PubKey() - assert.Equal(t, errors.Cause(err), ErrConnTimeout) + _, err := sc.getPubKey() + assert.Equal(t, err.(cmn.Error).Cause(), ErrConnTimeout) } -func TestSocketClientWait(t *testing.T) { - sc := NewSocketClient( +func TestSocketPVWait(t *testing.T) { + sc := NewSocketPV( log.TestingLogger(), "127.0.0.1:0", crypto.GenPrivKeyEd25519(), ) defer sc.Stop() - SocketClientConnWait(time.Millisecond)(sc) + SocketPVConnWait(time.Millisecond)(sc) - assert.Equal(t, errors.Cause(sc.Start()), ErrConnWaitTimeout) + assert.Equal(t, sc.Start().(cmn.Error).Cause(), ErrConnWaitTimeout) } func TestRemoteSignerRetry(t *testing.T) { @@ -216,7 +213,7 @@ func TestRemoteSignerRetry(t *testing.T) { log.TestingLogger(), cmn.RandStr(12), ln.Addr().String(), - NewTestPrivValidator(types.GenSigner()), + types.NewMockPV(), crypto.GenPrivKeyEd25519(), ) defer rs.Stop() @@ -224,7 +221,7 @@ func TestRemoteSignerRetry(t *testing.T) { RemoteSignerConnDeadline(time.Millisecond)(rs) RemoteSignerConnRetries(retries)(rs) - assert.Equal(t, errors.Cause(rs.Start()), ErrDialRetryMax) + assert.Equal(t, rs.Start().(cmn.Error).Cause(), ErrDialRetryMax) select { case attempts := <-attemptc: @@ -237,12 +234,11 @@ func TestRemoteSignerRetry(t *testing.T) { func testSetupSocketPair( t *testing.T, chainID string, -) (*SocketClient, *RemoteSigner) { +) (*SocketPV, *RemoteSigner) { var ( addr = testFreeAddr(t) logger = log.TestingLogger() - signer = types.GenSigner() - privVal = NewTestPrivValidator(signer) + privVal = types.NewMockPV() readyc = make(chan struct{}) rs = NewRemoteSigner( logger, @@ -251,14 +247,14 @@ func testSetupSocketPair( privVal, crypto.GenPrivKeyEd25519(), ) - sc = NewSocketClient( + sc = NewSocketPV( logger, addr, crypto.GenPrivKeyEd25519(), ) ) - go func(sc *SocketClient) { + go func(sc *SocketPV) { require.NoError(t, sc.Start()) assert.True(t, sc.IsRunning()) diff --git a/types/priv_validator/unencrypted.go b/types/priv_validator/unencrypted.go deleted file mode 100644 index 10a304d9e..000000000 --- a/types/priv_validator/unencrypted.go +++ /dev/null @@ -1,66 +0,0 @@ -package types - -import ( - "fmt" - - crypto "github.com/tendermint/go-crypto" - "github.com/tendermint/tendermint/types" - cmn "github.com/tendermint/tmlibs/common" -) - -//----------------------------------------------------------------- - -var _ types.PrivValidator2 = (*PrivValidatorUnencrypted)(nil) - -// PrivValidatorUnencrypted implements PrivValidator. -// It uses an in-memory crypto.PrivKey that is -// persisted to disk unencrypted. -type PrivValidatorUnencrypted struct { - ID types.ValidatorID `json:"id"` - PrivKey PrivKey `json:"priv_key"` - LastSignedInfo *LastSignedInfo `json:"last_signed_info"` -} - -// NewPrivValidatorUnencrypted returns an instance of PrivValidatorUnencrypted. -func NewPrivValidatorUnencrypted(priv crypto.PrivKey) *PrivValidatorUnencrypted { - return &PrivValidatorUnencrypted{ - ID: types.ValidatorID{ - Address: priv.PubKey().Address(), - PubKey: priv.PubKey(), - }, - PrivKey: PrivKey(priv), - LastSignedInfo: NewLastSignedInfo(), - } -} - -// String returns a string representation of the PrivValidatorUnencrypted -func (upv *PrivValidatorUnencrypted) String() string { - addr, err := upv.Address() - if err != nil { - panic(err) - } - - return fmt.Sprintf("PrivValidator{%v %v}", addr, upv.LastSignedInfo.String()) -} - -func (upv *PrivValidatorUnencrypted) Address() (cmn.HexBytes, error) { - return upv.PrivKey.PubKey().Address(), nil -} - -func (upv *PrivValidatorUnencrypted) PubKey() (crypto.PubKey, error) { - return upv.PrivKey.PubKey(), nil -} - -func (upv *PrivValidatorUnencrypted) SignVote(chainID string, vote *types.Vote) error { - return upv.LastSignedInfo.SignVote(upv.PrivKey, chainID, vote) -} - -func (upv *PrivValidatorUnencrypted) SignProposal(chainID string, proposal *types.Proposal) error { - return upv.LastSignedInfo.SignProposal(upv.PrivKey, chainID, proposal) -} - -func (upv *PrivValidatorUnencrypted) SignHeartbeat(chainID string, heartbeat *types.Heartbeat) error { - var err error - heartbeat.Signature, err = upv.PrivKey.Sign(heartbeat.SignBytes(chainID)) - return err -} diff --git a/types/priv_validator/upgrade.go b/types/priv_validator/upgrade.go deleted file mode 100644 index 063655421..000000000 --- a/types/priv_validator/upgrade.go +++ /dev/null @@ -1,59 +0,0 @@ -package types - -import ( - "encoding/json" - "io/ioutil" - - crypto "github.com/tendermint/go-crypto" - "github.com/tendermint/tendermint/types" - cmn "github.com/tendermint/tmlibs/common" -) - -type PrivValidatorV1 struct { - Address cmn.HexBytes `json:"address"` - PubKey crypto.PubKey `json:"pub_key"` - LastHeight int64 `json:"last_height"` - LastRound int `json:"last_round"` - LastStep int8 `json:"last_step"` - LastSignature crypto.Signature `json:"last_signature,omitempty"` // so we dont lose signatures - LastSignBytes cmn.HexBytes `json:"last_signbytes,omitempty"` // so we dont lose signatures - PrivKey crypto.PrivKey `json:"priv_key"` -} - -func UpgradePrivValidator(filePath string) (*PrivValidatorJSON, error) { - b, err := ioutil.ReadFile(filePath) - if err != nil { - return nil, err - } - - pv := new(PrivValidatorV1) - err = json.Unmarshal(b, pv) - if err != nil { - return nil, err - } - - pvNew := &PrivValidatorJSON{ - PrivValidatorUnencrypted: &PrivValidatorUnencrypted{ - ID: types.ValidatorID{ - Address: pv.Address, - PubKey: pv.PubKey, - }, - PrivKey: PrivKey(pv.PrivKey), - LastSignedInfo: &LastSignedInfo{ - Height: pv.LastHeight, - Round: pv.LastRound, - Step: pv.LastStep, - SignBytes: pv.LastSignBytes, - Signature: pv.LastSignature, - }, - }, - } - - b, err = json.MarshalIndent(pvNew, "", " ") - if err != nil { - return nil, err - } - - err = ioutil.WriteFile(filePath, b, 0600) - return pvNew, err -} diff --git a/types/priv_validator/upgrade_pv/main.go b/types/priv_validator/upgrade_pv/main.go deleted file mode 100644 index 5a0d4f263..000000000 --- a/types/priv_validator/upgrade_pv/main.go +++ /dev/null @@ -1,21 +0,0 @@ -package main - -import ( - "fmt" - "os" - - priv_val "github.com/tendermint/tendermint/types/priv_validator" -) - -func main() { - if len(os.Args) < 2 { - fmt.Println("USAGE: priv_val_converter ") - os.Exit(1) - } - file := os.Args[1] - _, err := priv_val.UpgradePrivValidator(file) - if err != nil { - fmt.Println(err) - os.Exit(1) - } -} diff --git a/types/priv_validator/wire.go b/types/priv_validator/wire.go new file mode 100644 index 000000000..688910834 --- /dev/null +++ b/types/priv_validator/wire.go @@ -0,0 +1,13 @@ +package privval + +import ( + "github.com/tendermint/go-amino" + "github.com/tendermint/go-crypto" +) + +var cdc = amino.NewCodec() + +func init() { + crypto.RegisterAmino(cdc) + RegisterSocketPVMsg(cdc) +} diff --git a/types/priv_validator_test.go b/types/priv_validator_test.go deleted file mode 100644 index edfcdf58c..000000000 --- a/types/priv_validator_test.go +++ /dev/null @@ -1,255 +0,0 @@ -package types - -import ( - "encoding/hex" - "encoding/json" - "fmt" - "os" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - crypto "github.com/tendermint/go-crypto" - cmn "github.com/tendermint/tmlibs/common" -) - -func TestGenLoadValidator(t *testing.T) { - assert := assert.New(t) - - _, tempFilePath := cmn.Tempfile("priv_validator_") - privVal := GenPrivValidatorFS(tempFilePath) - - height := int64(100) - privVal.LastHeight = height - privVal.Save() - addr := privVal.GetAddress() - - privVal = LoadPrivValidatorFS(tempFilePath) - assert.Equal(addr, privVal.GetAddress(), "expected privval addr to be the same") - assert.Equal(height, privVal.LastHeight, "expected privval.LastHeight to have been saved") -} - -func TestLoadOrGenValidator(t *testing.T) { - assert := assert.New(t) - - _, tempFilePath := cmn.Tempfile("priv_validator_") - if err := os.Remove(tempFilePath); err != nil { - t.Error(err) - } - privVal := LoadOrGenPrivValidatorFS(tempFilePath) - addr := privVal.GetAddress() - privVal = LoadOrGenPrivValidatorFS(tempFilePath) - assert.Equal(addr, privVal.GetAddress(), "expected privval addr to be the same") -} - -func TestUnmarshalValidator(t *testing.T) { - assert, require := assert.New(t), require.New(t) - - // create some fixed values - addrStr := "D028C9981F7A87F3093672BF0D5B0E2A1B3ED456" - pubStr := "3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8" - privStr := "27F82582AEFAE7AB151CFB01C48BB6C1A0DA78F9BDDA979A9F70A84D074EB07D3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8" - addrBytes, _ := hex.DecodeString(addrStr) - pubBytes, _ := hex.DecodeString(pubStr) - privBytes, _ := hex.DecodeString(privStr) - - // prepend type byte - pubKey, err := crypto.PubKeyFromBytes(append([]byte{1}, pubBytes...)) - require.Nil(err, "%+v", err) - privKey, err := crypto.PrivKeyFromBytes(append([]byte{1}, privBytes...)) - require.Nil(err, "%+v", err) - - serialized := fmt.Sprintf(`{ - "address": "%s", - "pub_key": { - "type": "ed25519", - "data": "%s" - }, - "last_height": 0, - "last_round": 0, - "last_step": 0, - "last_signature": null, - "priv_key": { - "type": "ed25519", - "data": "%s" - } -}`, addrStr, pubStr, privStr) - - val := PrivValidatorFS{} - err = json.Unmarshal([]byte(serialized), &val) - require.Nil(err, "%+v", err) - - // make sure the values match - assert.EqualValues(addrBytes, val.GetAddress()) - assert.EqualValues(pubKey, val.GetPubKey()) - assert.EqualValues(privKey, val.PrivKey) - - // export it and make sure it is the same - out, err := json.Marshal(val) - require.Nil(err, "%+v", err) - assert.JSONEq(serialized, string(out)) -} - -func TestSignVote(t *testing.T) { - assert := assert.New(t) - - _, tempFilePath := cmn.Tempfile("priv_validator_") - privVal := GenPrivValidatorFS(tempFilePath) - - block1 := BlockID{[]byte{1, 2, 3}, PartSetHeader{}} - block2 := BlockID{[]byte{3, 2, 1}, PartSetHeader{}} - height, round := int64(10), 1 - voteType := VoteTypePrevote - - // sign a vote for first time - vote := newVote(privVal.Address, 0, height, round, voteType, block1) - err := privVal.SignVote("mychainid", vote) - assert.NoError(err, "expected no error signing vote") - - // try to sign the same vote again; should be fine - err = privVal.SignVote("mychainid", vote) - assert.NoError(err, "expected no error on signing same vote") - - // now try some bad votes - cases := []*Vote{ - newVote(privVal.Address, 0, height, round-1, voteType, block1), // round regression - newVote(privVal.Address, 0, height-1, round, voteType, block1), // height regression - newVote(privVal.Address, 0, height-2, round+4, voteType, block1), // height regression and different round - newVote(privVal.Address, 0, height, round, voteType, block2), // different block - } - - for _, c := range cases { - err = privVal.SignVote("mychainid", c) - assert.Error(err, "expected error on signing conflicting vote") - } - - // try signing a vote with a different time stamp - sig := vote.Signature - vote.Timestamp = vote.Timestamp.Add(time.Duration(1000)) - err = privVal.SignVote("mychainid", vote) - assert.NoError(err) - assert.Equal(sig, vote.Signature) -} - -func TestSignProposal(t *testing.T) { - assert := assert.New(t) - - _, tempFilePath := cmn.Tempfile("priv_validator_") - privVal := GenPrivValidatorFS(tempFilePath) - - block1 := PartSetHeader{5, []byte{1, 2, 3}} - block2 := PartSetHeader{10, []byte{3, 2, 1}} - height, round := int64(10), 1 - - // sign a proposal for first time - proposal := newProposal(height, round, block1) - err := privVal.SignProposal("mychainid", proposal) - assert.NoError(err, "expected no error signing proposal") - - // try to sign the same proposal again; should be fine - err = privVal.SignProposal("mychainid", proposal) - assert.NoError(err, "expected no error on signing same proposal") - - // now try some bad Proposals - cases := []*Proposal{ - newProposal(height, round-1, block1), // round regression - newProposal(height-1, round, block1), // height regression - newProposal(height-2, round+4, block1), // height regression and different round - newProposal(height, round, block2), // different block - } - - for _, c := range cases { - err = privVal.SignProposal("mychainid", c) - assert.Error(err, "expected error on signing conflicting proposal") - } - - // try signing a proposal with a different time stamp - sig := proposal.Signature - proposal.Timestamp = proposal.Timestamp.Add(time.Duration(1000)) - err = privVal.SignProposal("mychainid", proposal) - assert.NoError(err) - assert.Equal(sig, proposal.Signature) -} - -func TestDifferByTimestamp(t *testing.T) { - _, tempFilePath := cmn.Tempfile("priv_validator_") - privVal := GenPrivValidatorFS(tempFilePath) - - block1 := PartSetHeader{5, []byte{1, 2, 3}} - height, round := int64(10), 1 - chainID := "mychainid" - - // test proposal - { - proposal := newProposal(height, round, block1) - err := privVal.SignProposal(chainID, proposal) - assert.NoError(t, err, "expected no error signing proposal") - signBytes := proposal.SignBytes(chainID) - sig := proposal.Signature - timeStamp := clipToMS(proposal.Timestamp) - - // manipulate the timestamp. should get changed back - proposal.Timestamp = proposal.Timestamp.Add(time.Millisecond) - var emptySig crypto.Signature - proposal.Signature = emptySig - err = privVal.SignProposal("mychainid", proposal) - assert.NoError(t, err, "expected no error on signing same proposal") - - assert.Equal(t, timeStamp, proposal.Timestamp) - assert.Equal(t, signBytes, proposal.SignBytes(chainID)) - assert.Equal(t, sig, proposal.Signature) - } - - // test vote - { - voteType := VoteTypePrevote - blockID := BlockID{[]byte{1, 2, 3}, PartSetHeader{}} - vote := newVote(privVal.Address, 0, height, round, voteType, blockID) - err := privVal.SignVote("mychainid", vote) - assert.NoError(t, err, "expected no error signing vote") - - signBytes := vote.SignBytes(chainID) - sig := vote.Signature - timeStamp := clipToMS(vote.Timestamp) - - // manipulate the timestamp. should get changed back - vote.Timestamp = vote.Timestamp.Add(time.Millisecond) - var emptySig crypto.Signature - vote.Signature = emptySig - err = privVal.SignVote("mychainid", vote) - assert.NoError(t, err, "expected no error on signing same vote") - - assert.Equal(t, timeStamp, vote.Timestamp) - assert.Equal(t, signBytes, vote.SignBytes(chainID)) - assert.Equal(t, sig, vote.Signature) - } -} - -func newVote(addr Address, idx int, height int64, round int, typ byte, blockID BlockID) *Vote { - return &Vote{ - ValidatorAddress: addr, - ValidatorIndex: idx, - Height: height, - Round: round, - Type: typ, - Timestamp: time.Now().UTC(), - BlockID: blockID, - } -} - -func newProposal(height int64, round int, partsHeader PartSetHeader) *Proposal { - return &Proposal{ - Height: height, - Round: round, - BlockPartsHeader: partsHeader, - Timestamp: time.Now().UTC(), - } -} - -func clipToMS(t time.Time) time.Time { - nano := t.UnixNano() - million := int64(1000000) - nano = (nano / million) * million - return time.Unix(0, nano).UTC() -} diff --git a/types/proposal.go b/types/proposal.go index c240756bb..95008897b 100644 --- a/types/proposal.go +++ b/types/proposal.go @@ -6,7 +6,6 @@ import ( "time" "github.com/tendermint/go-crypto" - "github.com/tendermint/tendermint/wire" ) var ( @@ -51,10 +50,7 @@ func (p *Proposal) String() string { // SignBytes returns the Proposal bytes for signing func (p *Proposal) SignBytes(chainID string) []byte { - bz, err := wire.MarshalJSON(CanonicalJSONOnceProposal{ - ChainID: chainID, - Proposal: CanonicalProposal(p), - }) + bz, err := cdc.MarshalJSON(CanonicalProposal(chainID, p)) if err != nil { panic(err) } diff --git a/types/proposal_test.go b/types/proposal_test.go index 610f76855..43fb7c209 100644 --- a/types/proposal_test.go +++ b/types/proposal_test.go @@ -5,8 +5,6 @@ import ( "time" "github.com/stretchr/testify/require" - - wire "github.com/tendermint/tendermint/wire" ) var testProposal *Proposal @@ -29,7 +27,7 @@ func TestProposalSignable(t *testing.T) { signBytes := testProposal.SignBytes("test_chain_id") signStr := string(signBytes) - expected := `{"chain_id":"test_chain_id","proposal":{"block_parts_header":{"hash":"626C6F636B7061727473","total":111},"height":12345,"pol_block_id":{},"pol_round":-1,"round":23456,"timestamp":"2018-02-11T07:09:22.765Z"}}` + expected := `{"@chain_id":"test_chain_id","@type":"proposal","block_parts_header":{"hash":"626C6F636B7061727473","total":111},"height":12345,"pol_block_id":{},"pol_round":-1,"round":23456,"timestamp":"2018-02-11T07:09:22.765Z"}` if signStr != expected { t.Errorf("Got unexpected sign string for Proposal. Expected:\n%v\nGot:\n%v", expected, signStr) } @@ -37,38 +35,38 @@ func TestProposalSignable(t *testing.T) { func TestProposalString(t *testing.T) { str := testProposal.String() - expected := `Proposal{12345/23456 111:626C6F636B70 (-1,:0:000000000000) {} @ 2018-02-11T07:09:22.765Z}` + expected := `Proposal{12345/23456 111:626C6F636B70 (-1,:0:000000000000) @ 2018-02-11T07:09:22.765Z}` if str != expected { t.Errorf("Got unexpected string for Proposal. Expected:\n%v\nGot:\n%v", expected, str) } } func TestProposalVerifySignature(t *testing.T) { - privVal := GenPrivValidatorFS("") + privVal := NewMockPV() pubKey := privVal.GetPubKey() prop := NewProposal(4, 2, PartSetHeader{777, []byte("proper")}, 2, BlockID{}) signBytes := prop.SignBytes("test_chain_id") // sign it - signature, err := privVal.Signer.Sign(signBytes) + err := privVal.SignProposal("test_chain_id", prop) require.NoError(t, err) // verify the same proposal - valid := pubKey.VerifyBytes(prop.SignBytes("test_chain_id"), signature) + valid := pubKey.VerifyBytes(signBytes, prop.Signature) require.True(t, valid) // serialize, deserialize and verify again.... newProp := new(Proposal) - bs, err := wire.MarshalBinary(prop) + bs, err := cdc.MarshalBinary(prop) require.NoError(t, err) - err = wire.UnmarshalBinary(bs, &newProp) + err = cdc.UnmarshalBinary(bs, &newProp) require.NoError(t, err) // verify the transmitted proposal newSignBytes := newProp.SignBytes("test_chain_id") require.Equal(t, string(signBytes), string(newSignBytes)) - valid = pubKey.VerifyBytes(newSignBytes, signature) + valid = pubKey.VerifyBytes(newSignBytes, newProp.Signature) require.True(t, valid) } @@ -79,9 +77,9 @@ func BenchmarkProposalWriteSignBytes(b *testing.B) { } func BenchmarkProposalSign(b *testing.B) { - privVal := GenPrivValidatorFS("") + privVal := NewMockPV() for i := 0; i < b.N; i++ { - _, err := privVal.Signer.Sign(testProposal.SignBytes("test_chain_id")) + err := privVal.SignProposal("test_chain_id", testProposal) if err != nil { b.Error(err) } @@ -89,12 +87,12 @@ func BenchmarkProposalSign(b *testing.B) { } func BenchmarkProposalVerifySignature(b *testing.B) { - signBytes := testProposal.SignBytes("test_chain_id") - privVal := GenPrivValidatorFS("") - signature, _ := privVal.Signer.Sign(signBytes) + privVal := NewMockPV() + err := privVal.SignProposal("test_chain_id", testProposal) + require.Nil(b, err) pubKey := privVal.GetPubKey() for i := 0; i < b.N; i++ { - pubKey.VerifyBytes(testProposal.SignBytes("test_chain_id"), signature) + pubKey.VerifyBytes(testProposal.SignBytes("test_chain_id"), testProposal.Signature) } } diff --git a/types/results.go b/types/results.go index 71834664d..326cee48d 100644 --- a/types/results.go +++ b/types/results.go @@ -2,7 +2,6 @@ package types import ( abci "github.com/tendermint/abci/types" - wire "github.com/tendermint/tendermint/wire" cmn "github.com/tendermint/tmlibs/common" "github.com/tendermint/tmlibs/merkle" ) @@ -18,7 +17,8 @@ type ABCIResult struct { // Hash returns the canonical hash of the ABCIResult func (a ABCIResult) Hash() []byte { - return tmHash(a) + bz := aminoHash(a) + return bz } // ABCIResults wraps the deliver tx results to return a proof @@ -42,7 +42,7 @@ func NewResultFromResponse(response *abci.ResponseDeliverTx) ABCIResult { // Bytes serializes the ABCIResponse using wire func (a ABCIResults) Bytes() []byte { - bz, err := wire.MarshalBinary(a) + bz, err := cdc.MarshalBinary(a) if err != nil { panic(err) } diff --git a/types/results_test.go b/types/results_test.go index 88624e8c8..8018b4e85 100644 --- a/types/results_test.go +++ b/types/results_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestABCIResults(t *testing.T) { @@ -14,13 +15,15 @@ func TestABCIResults(t *testing.T) { e := ABCIResult{Code: 14, Data: []byte("foo")} f := ABCIResult{Code: 14, Data: []byte("bar")} - // nil and []byte{} should produce same hash - assert.Equal(t, a.Hash(), b.Hash()) + // Nil and []byte{} should not produce the same hash. + require.Equal(t, a.Hash(), a.Hash()) + require.Equal(t, b.Hash(), b.Hash()) + require.NotEqual(t, a.Hash(), b.Hash()) - // a and b should be the same, don't go in results + // a and b should be the same, don't go in results. results := ABCIResults{a, c, d, e, f} - // make sure each result hashes properly + // Make sure each result hashes properly. var last []byte for i, res := range results { h := res.Hash() @@ -28,8 +31,7 @@ func TestABCIResults(t *testing.T) { last = h } - // make sure that we can get a root hash from results - // and verify proofs + // Make sure that we can get a root hash from results and verify proofs. root := results.Hash() assert.NotEmpty(t, root) diff --git a/types/signable.go b/types/signable.go index 19829ede7..cc6498882 100644 --- a/types/signable.go +++ b/types/signable.go @@ -9,8 +9,3 @@ package types type Signable interface { SignBytes(chainID string) []byte } - -// HashSignBytes is a convenience method for getting the hash of the bytes of a signable -func HashSignBytes(chainID string, o Signable) []byte { - return tmHash(o.SignBytes(chainID)) -} diff --git a/types/test_util.go b/types/test_util.go index 73e53eb19..f21c2831f 100644 --- a/types/test_util.go +++ b/types/test_util.go @@ -4,7 +4,7 @@ import "time" func MakeCommit(blockID BlockID, height int64, round int, voteSet *VoteSet, - validators []*PrivValidatorFS) (*Commit, error) { + validators []PrivValidator) (*Commit, error) { // all sign for i := 0; i < len(validators); i++ { @@ -28,8 +28,8 @@ func MakeCommit(blockID BlockID, height int64, round int, return voteSet.MakeCommit(), nil } -func signAddVote(privVal *PrivValidatorFS, vote *Vote, voteSet *VoteSet) (signed bool, err error) { - vote.Signature, err = privVal.Signer.Sign(vote.SignBytes(voteSet.ChainID())) +func signAddVote(privVal PrivValidator, vote *Vote, voteSet *VoteSet) (signed bool, err error) { + err = privVal.SignVote(voteSet.ChainID(), vote) if err != nil { return false, err } diff --git a/types/tx.go b/types/tx.go index fc1d27212..e7247693a 100644 --- a/types/tx.go +++ b/types/tx.go @@ -18,7 +18,7 @@ type Tx []byte // Hash computes the RIPEMD160 hash of the wire encoded transaction. func (tx Tx) Hash() []byte { - return wireHasher(tx).Hash() + return aminoHasher(tx).Hash() } // String returns the hex-encoded transaction as a string. @@ -66,9 +66,7 @@ func (txs Txs) IndexByHash(hash []byte) int { } // Proof returns a simple merkle proof for this node. -// // Panics if i < 0 or i >= len(txs) -// // TODO: optimize this! func (txs Txs) Proof(i int) TxProof { l := len(txs) @@ -95,7 +93,7 @@ type TxProof struct { Proof merkle.SimpleProof } -// LeadHash returns the hash of the transaction this proof refers to. +// LeadHash returns the hash of the this proof refers to. func (tp TxProof) LeafHash() []byte { return tp.Data.Hash() } @@ -106,7 +104,12 @@ func (tp TxProof) Validate(dataHash []byte) error { if !bytes.Equal(dataHash, tp.RootHash) { return errors.New("Proof matches different data hash") } - + if tp.Index < 0 { + return errors.New("Proof index cannot be negative") + } + if tp.Total <= 0 { + return errors.New("Proof total must be positive") + } valid := tp.Proof.Verify(tp.Index, tp.Total, tp.LeafHash(), tp.RootHash) if !valid { return errors.New("Proof is not internally consistent") diff --git a/types/tx_test.go b/types/tx_test.go index 340bd5a6b..2a93ceb31 100644 --- a/types/tx_test.go +++ b/types/tx_test.go @@ -6,7 +6,6 @@ import ( "github.com/stretchr/testify/assert" - wire "github.com/tendermint/tendermint/wire" cmn "github.com/tendermint/tmlibs/common" ctest "github.com/tendermint/tmlibs/test" ) @@ -69,9 +68,9 @@ func TestValidTxProof(t *testing.T) { // read-write must also work var p2 TxProof - bin, err := wire.MarshalBinary(proof) + bin, err := cdc.MarshalBinary(proof) assert.Nil(err) - err = wire.UnmarshalBinary(bin, &p2) + err = cdc.UnmarshalBinary(bin, &p2) if assert.Nil(err, "%d: %d: %+v", h, i, err) { assert.Nil(p2.Validate(root), "%d: %d", h, i) } @@ -97,7 +96,7 @@ func testTxProofUnchangable(t *testing.T) { // make sure it is valid to start with assert.Nil(proof.Validate(root)) - bin, err := wire.MarshalBinary(proof) + bin, err := cdc.MarshalBinary(proof) assert.Nil(err) // try mutating the data and make sure nothing breaks @@ -109,16 +108,17 @@ func testTxProofUnchangable(t *testing.T) { } } -// this make sure the proof doesn't deserialize into something valid +// This makes sure that the proof doesn't deserialize into something valid. func assertBadProof(t *testing.T, root []byte, bad []byte, good TxProof) { var proof TxProof - err := wire.UnmarshalBinary(bad, &proof) + err := cdc.UnmarshalBinary(bad, &proof) if err == nil { err = proof.Validate(root) if err == nil { - // okay, this can happen if we have a slightly different total - // (where the path ends up the same), if it is something else, we have - // a real problem + // XXX Fix simple merkle proofs so the following is *not* OK. + // This can happen if we have a slightly different total (where the + // path ends up the same). If it is something else, we have a real + // problem. assert.NotEqual(t, proof.Total, good.Total, "bad: %#v\ngood: %#v", proof, good) } } diff --git a/types/validator.go b/types/validator.go index 027f7dc3c..e3ccc551a 100644 --- a/types/validator.go +++ b/types/validator.go @@ -70,7 +70,7 @@ func (v *Validator) String() string { // Hash computes the unique ID of a validator with a given voting power. // It excludes the Accum value, which changes with every round. func (v *Validator) Hash() []byte { - return tmHash(struct { + return aminoHash(struct { Address Address PubKey crypto.PubKey VotingPower int64 @@ -81,14 +81,13 @@ func (v *Validator) Hash() []byte { }) } -//-------------------------------------------------------------------------------- -// For testing... +//---------------------------------------- +// RandValidator // RandValidator returns a randomized validator, useful for testing. // UNSTABLE -func RandValidator(randPower bool, minPower int64) (*Validator, *PrivValidatorFS) { - _, tempFilePath := cmn.Tempfile("priv_validator_") - privVal := GenPrivValidatorFS(tempFilePath) +func RandValidator(randPower bool, minPower int64) (*Validator, PrivValidator) { + privVal := NewMockPV() votePower := minPower if randPower { votePower += int64(cmn.RandUint32()) diff --git a/types/validator_set.go b/types/validator_set.go index 83d066ec1..51bf71563 100644 --- a/types/validator_set.go +++ b/types/validator_set.go @@ -7,7 +7,6 @@ import ( "sort" "strings" - "github.com/pkg/errors" cmn "github.com/tendermint/tmlibs/common" "github.com/tendermint/tmlibs/merkle" ) @@ -289,10 +288,10 @@ func (valSet *ValidatorSet) VerifyCommitAny(newSet *ValidatorSet, chainID string blockID BlockID, height int64, commit *Commit) error { if newSet.Size() != len(commit.Precommits) { - return errors.Errorf("Invalid commit -- wrong set size: %v vs %v", newSet.Size(), len(commit.Precommits)) + return cmn.NewError("Invalid commit -- wrong set size: %v vs %v", newSet.Size(), len(commit.Precommits)) } if height != commit.Height() { - return errors.Errorf("Invalid commit -- wrong height: %v vs %v", height, commit.Height()) + return cmn.NewError("Invalid commit -- wrong height: %v vs %v", height, commit.Height()) } oldVotingPower := int64(0) @@ -307,13 +306,13 @@ func (valSet *ValidatorSet) VerifyCommitAny(newSet *ValidatorSet, chainID string } if precommit.Height != height { // return certerr.ErrHeightMismatch(height, precommit.Height) - return errors.Errorf("Blocks don't match - %d vs %d", round, precommit.Round) + return cmn.NewError("Blocks don't match - %d vs %d", round, precommit.Round) } if precommit.Round != round { - return errors.Errorf("Invalid commit -- wrong round: %v vs %v", round, precommit.Round) + return cmn.NewError("Invalid commit -- wrong round: %v vs %v", round, precommit.Round) } if precommit.Type != VoteTypePrecommit { - return errors.Errorf("Invalid commit -- not precommit @ index %v", idx) + return cmn.NewError("Invalid commit -- not precommit @ index %v", idx) } if !blockID.Equals(precommit.BlockID) { continue // Not an error, but doesn't count @@ -329,7 +328,7 @@ func (valSet *ValidatorSet) VerifyCommitAny(newSet *ValidatorSet, chainID string // Validate signature old school precommitSignBytes := precommit.SignBytes(chainID) if !ov.PubKey.VerifyBytes(precommitSignBytes, precommit.Signature) { - return errors.Errorf("Invalid commit -- invalid signature: %v", precommit) + return cmn.NewError("Invalid commit -- invalid signature: %v", precommit) } // Good precommit! oldVotingPower += ov.VotingPower @@ -343,10 +342,10 @@ func (valSet *ValidatorSet) VerifyCommitAny(newSet *ValidatorSet, chainID string } if oldVotingPower <= valSet.TotalVotingPower()*2/3 { - return errors.Errorf("Invalid commit -- insufficient old voting power: got %v, needed %v", + return cmn.NewError("Invalid commit -- insufficient old voting power: got %v, needed %v", oldVotingPower, (valSet.TotalVotingPower()*2/3 + 1)) } else if newVotingPower <= newSet.TotalVotingPower()*2/3 { - return errors.Errorf("Invalid commit -- insufficient cur voting power: got %v, needed %v", + return cmn.NewError("Invalid commit -- insufficient cur voting power: got %v, needed %v", newVotingPower, (newSet.TotalVotingPower()*2/3 + 1)) } return nil @@ -416,9 +415,9 @@ func (ac accumComparable) Less(o interface{}) bool { // RandValidatorSet returns a randomized validator set, useful for testing. // NOTE: PrivValidator are in order. // UNSTABLE -func RandValidatorSet(numValidators int, votingPower int64) (*ValidatorSet, []*PrivValidatorFS) { +func RandValidatorSet(numValidators int, votingPower int64) (*ValidatorSet, []PrivValidator) { vals := make([]*Validator, numValidators) - privValidators := make([]*PrivValidatorFS, numValidators) + privValidators := make([]PrivValidator, numValidators) for i := 0; i < numValidators; i++ { val, privValidator := RandValidator(false, votingPower) vals[i] = val diff --git a/types/validator_set_test.go b/types/validator_set_test.go index b346be1be..edfe4d5ce 100644 --- a/types/validator_set_test.go +++ b/types/validator_set_test.go @@ -11,7 +11,6 @@ import ( "github.com/stretchr/testify/assert" crypto "github.com/tendermint/go-crypto" - wire "github.com/tendermint/tendermint/wire" cmn "github.com/tendermint/tmlibs/common" ) @@ -198,7 +197,7 @@ func newValidator(address []byte, power int64) *Validator { func randPubKey() crypto.PubKey { var pubKey [32]byte copy(pubKey[:], cmn.RandBytes(32)) - return crypto.PubKeyEd25519(pubKey).Wrap() + return crypto.PubKeyEd25519(pubKey) } func randValidator_() *Validator { @@ -216,7 +215,7 @@ func randValidatorSet(numValidators int) *ValidatorSet { } func (valSet *ValidatorSet) toBytes() []byte { - bz, err := wire.MarshalBinary(valSet) + bz, err := cdc.MarshalBinary(valSet) if err != nil { panic(err) } @@ -224,7 +223,7 @@ func (valSet *ValidatorSet) toBytes() []byte { } func (valSet *ValidatorSet) fromBytes(b []byte) { - err := wire.UnmarshalBinary(b, &valSet) + err := cdc.UnmarshalBinary(b, &valSet) if err != nil { // DATA HAS BEEN CORRUPTED OR THE SPEC HAS CHANGED panic(err) diff --git a/types/vote.go b/types/vote.go index ceb6e985e..e4ead612a 100644 --- a/types/vote.go +++ b/types/vote.go @@ -7,7 +7,6 @@ import ( "time" crypto "github.com/tendermint/go-crypto" - "github.com/tendermint/tendermint/wire" cmn "github.com/tendermint/tmlibs/common" ) @@ -73,10 +72,7 @@ type Vote struct { } func (vote *Vote) SignBytes(chainID string) []byte { - bz, err := wire.MarshalJSON(CanonicalJSONOnceVote{ - chainID, - CanonicalVote(vote), - }) + bz, err := cdc.MarshalJSON(CanonicalVote(chainID, vote)) if err != nil { panic(err) } diff --git a/types/vote_set_test.go b/types/vote_set_test.go index 4a09d04d3..d424667b1 100644 --- a/types/vote_set_test.go +++ b/types/vote_set_test.go @@ -11,7 +11,7 @@ import ( ) // NOTE: privValidators are in order -func randVoteSet(height int64, round int, type_ byte, numValidators int, votingPower int64) (*VoteSet, *ValidatorSet, []*PrivValidatorFS) { +func randVoteSet(height int64, round int, type_ byte, numValidators int, votingPower int64) (*VoteSet, *ValidatorSet, []PrivValidator) { valSet, privValidators := RandValidatorSet(numValidators, votingPower) return NewVoteSet("test_chain_id", height, round, type_, valSet), valSet, privValidators } diff --git a/types/vote_test.go b/types/vote_test.go index a4a0f309f..263d2c351 100644 --- a/types/vote_test.go +++ b/types/vote_test.go @@ -5,8 +5,6 @@ import ( "time" "github.com/stretchr/testify/require" - - wire "github.com/tendermint/tendermint/wire" ) func examplePrevote() *Vote { @@ -45,7 +43,7 @@ func TestVoteSignable(t *testing.T) { signBytes := vote.SignBytes("test_chain_id") signStr := string(signBytes) - expected := `{"chain_id":"test_chain_id","vote":{"block_id":{"hash":"68617368","parts":{"hash":"70617274735F68617368","total":1000000}},"height":12345,"round":2,"timestamp":"2017-12-25T03:00:01.234Z","type":2}}` + expected := `{"@chain_id":"test_chain_id","@type":"vote","block_id":{"hash":"68617368","parts":{"hash":"70617274735F68617368","total":1000000}},"height":12345,"round":2,"timestamp":"2017-12-25T03:00:01.234Z","type":2}` if signStr != expected { // NOTE: when this fails, you probably want to fix up consensus/replay_test too t.Errorf("Got unexpected sign string for Vote. Expected:\n%v\nGot:\n%v", expected, signStr) @@ -58,8 +56,8 @@ func TestVoteString(t *testing.T) { in string out string }{ - {"Precommit", examplePrecommit().String(), `Vote{56789:616464720000 12345/02/2(Precommit) 686173680000 {} @ 2017-12-25T03:00:01.234Z}`}, - {"Prevote", examplePrevote().String(), `Vote{56789:616464720000 12345/02/1(Prevote) 686173680000 {} @ 2017-12-25T03:00:01.234Z}`}, + {"Precommit", examplePrecommit().String(), `Vote{56789:616464720000 12345/02/2(Precommit) 686173680000 @ 2017-12-25T03:00:01.234Z}`}, + {"Prevote", examplePrevote().String(), `Vote{56789:616464720000 12345/02/1(Prevote) 686173680000 @ 2017-12-25T03:00:01.234Z}`}, } for _, tt := range tc { @@ -73,31 +71,31 @@ func TestVoteString(t *testing.T) { } func TestVoteVerifySignature(t *testing.T) { - privVal := GenPrivValidatorFS("") + privVal := NewMockPV() pubKey := privVal.GetPubKey() vote := examplePrecommit() signBytes := vote.SignBytes("test_chain_id") // sign it - signature, err := privVal.Signer.Sign(signBytes) + err := privVal.SignVote("test_chain_id", vote) require.NoError(t, err) // verify the same vote - valid := pubKey.VerifyBytes(vote.SignBytes("test_chain_id"), signature) + valid := pubKey.VerifyBytes(vote.SignBytes("test_chain_id"), vote.Signature) require.True(t, valid) // serialize, deserialize and verify again.... precommit := new(Vote) - bs, err := wire.MarshalBinary(vote) + bs, err := cdc.MarshalBinary(vote) require.NoError(t, err) - err = wire.UnmarshalBinary(bs, &precommit) + err = cdc.UnmarshalBinary(bs, &precommit) require.NoError(t, err) // verify the transmitted vote newSignBytes := precommit.SignBytes("test_chain_id") require.Equal(t, string(signBytes), string(newSignBytes)) - valid = pubKey.VerifyBytes(newSignBytes, signature) + valid = pubKey.VerifyBytes(newSignBytes, precommit.Signature) require.True(t, valid) } diff --git a/types/wire.go b/types/wire.go new file mode 100644 index 000000000..bd5c4497d --- /dev/null +++ b/types/wire.go @@ -0,0 +1,12 @@ +package types + +import ( + "github.com/tendermint/go-amino" + "github.com/tendermint/go-crypto" +) + +var cdc = amino.NewCodec() + +func init() { + crypto.RegisterAmino(cdc) +} From 35a1d747b0d1fee110e749dca84a58387588f074 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Sat, 31 Mar 2018 11:51:32 +0200 Subject: [PATCH 017/143] Fix mempool --- mempool/reactor.go | 26 +++++++++----------------- mempool/wire.go | 11 +++++++++++ 2 files changed, 20 insertions(+), 17 deletions(-) create mode 100644 mempool/wire.go diff --git a/mempool/reactor.go b/mempool/reactor.go index 514347e94..acd693e42 100644 --- a/mempool/reactor.go +++ b/mempool/reactor.go @@ -1,13 +1,12 @@ package mempool import ( - "bytes" "fmt" "reflect" "time" abci "github.com/tendermint/abci/types" - wire "github.com/tendermint/go-wire" + "github.com/tendermint/go-amino" "github.com/tendermint/tmlibs/clist" "github.com/tendermint/tmlibs/log" @@ -71,7 +70,7 @@ func (memR *MempoolReactor) RemovePeer(peer p2p.Peer, reason interface{}) { // Receive implements Reactor. // It adds any received transactions to the mempool. func (memR *MempoolReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) { - _, msg, err := DecodeMessage(msgBytes) + msg, err := DecodeMessage(msgBytes) if err != nil { memR.Logger.Error("Error decoding message", "src", src, "chId", chID, "msg", msg, "err", err, "bytes", msgBytes) memR.Switch.StopPeerForError(src, err) @@ -137,7 +136,7 @@ func (memR *MempoolReactor) broadcastTxRoutine(peer p2p.Peer) { } // send memTx msg := &TxMessage{Tx: memTx.tx} - success := peer.Send(MempoolChannel, struct{ MempoolMessage }{msg}) + success := peer.Send(MempoolChannel, cdc.MustMarshalBinaryBare(msg)) if !success { time.Sleep(peerCatchupSleepIntervalMS * time.Millisecond) continue @@ -158,24 +157,17 @@ func (memR *MempoolReactor) broadcastTxRoutine(peer p2p.Peer) { //----------------------------------------------------------------------------- // Messages -const ( - msgTypeTx = byte(0x01) -) - // MempoolMessage is a message sent or received by the MempoolReactor. type MempoolMessage interface{} -var _ = wire.RegisterInterface( - struct{ MempoolMessage }{}, - wire.ConcreteType{&TxMessage{}, msgTypeTx}, -) +func RegisterMempoolMessages(cdc *amino.Codec) { + cdc.RegisterInterface((*MempoolMessage)(nil), nil) + cdc.RegisterConcrete(&TxMessage{}, "tendermint/mempool/TxMessage", nil) +} // DecodeMessage decodes a byte-array into a MempoolMessage. -func DecodeMessage(bz []byte) (msgType byte, msg MempoolMessage, err error) { - msgType = bz[0] - n := new(int) - r := bytes.NewReader(bz) - msg = wire.ReadBinary(struct{ MempoolMessage }{}, r, maxMempoolMessageSize, n, &err).(struct{ MempoolMessage }).MempoolMessage +func DecodeMessage(bz []byte) (msg MempoolMessage, err error) { + err = cdc.UnmarshalBinaryBare(bz, &msg) return } diff --git a/mempool/wire.go b/mempool/wire.go new file mode 100644 index 000000000..ed0897268 --- /dev/null +++ b/mempool/wire.go @@ -0,0 +1,11 @@ +package mempool + +import ( + "github.com/tendermint/go-amino" +) + +var cdc = amino.NewCodec() + +func init() { + RegisterMempoolMessages(cdc) +} From eaee98ee1f4b72a3a4b3a64766b09070b65e2bb6 Mon Sep 17 00:00:00 2001 From: Greg Szabo <16846635+greg-szabo@users.noreply.github.com> Date: Sun, 1 Apr 2018 13:54:48 -0400 Subject: [PATCH 018/143] CGO_ENABLED=0 added for static linking (#1396) --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 851151eeb..c19ce11a6 100644 --- a/Makefile +++ b/Makefile @@ -14,13 +14,13 @@ check: check_tools ensure_deps ### Build build: - go build $(BUILD_FLAGS) -tags '$(BUILD_TAGS)' -o build/tendermint ./cmd/tendermint/ + CGO_ENABLED=0 go build $(BUILD_FLAGS) -tags '$(BUILD_TAGS)' -o build/tendermint ./cmd/tendermint/ build_race: - go build -race $(BUILD_FLAGS) -tags '$(BUILD_TAGS)' -o build/tendermint ./cmd/tendermint + CGO_ENABLED=0 go build -race $(BUILD_FLAGS) -tags '$(BUILD_TAGS)' -o build/tendermint ./cmd/tendermint install: - go install $(BUILD_FLAGS) -tags '$(BUILD_TAGS)' ./cmd/tendermint + CGO_ENABLED=0 go install $(BUILD_FLAGS) -tags '$(BUILD_TAGS)' ./cmd/tendermint ######################################## ### Distribution From 2644a529f0ddfb3fa928e8223ed4f8a7e91bf05b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Corbi=C3=A8re?= Date: Mon, 2 Apr 2018 10:21:17 +0200 Subject: [PATCH 019/143] Fix lint errors (#1390) * use increment and decrement operators. * remove unnecessary else branches. * fix package comment with leading space. * fix receiver names. * fix error strings. * remove omittable code. * remove redundant return statement. * Revert changes (code is generated.) * use cfg as receiver name for all config-related types. * use lsi as the receiver name for the LastSignedInfo type. --- cmd/tendermint/commands/run_node.go | 3 +- config/config.go | 72 ++++++++--------- consensus/common_test.go | 4 +- consensus/mempool_test.go | 11 ++- consensus/reactor.go | 10 +-- consensus/reactor_test.go | 2 +- consensus/replay.go | 12 +-- consensus/replay_file.go | 17 ++-- consensus/replay_test.go | 6 +- consensus/state.go | 19 +++-- node/node.go | 13 ++- p2p/base_reactor.go | 8 +- p2p/conn/connection.go | 3 +- p2p/conn/secret_connection.go | 7 +- p2p/conn/secret_connection_test.go | 2 +- p2p/key.go | 5 +- p2p/netaddress.go | 4 +- p2p/peer_set.go | 3 +- p2p/peer_set_test.go | 2 +- p2p/pex/known_address.go | 2 +- p2p/pex/pex_reactor.go | 3 +- p2p/test_util.go | 2 +- p2p/upnp/upnp.go | 2 +- rpc/core/tx.go | 4 +- rpc/lib/client/ws_client.go | 26 +++--- rpc/lib/types/types.go | 3 +- scripts/wal2json/main.go | 1 + state/state_test.go | 4 +- state/txindex/kv/kv.go | 3 +- state/txindex/kv/kv_test.go | 2 +- types/block.go | 3 +- types/block_test.go | 2 +- types/evidence.go | 2 +- types/genesis.go | 3 +- types/part_set.go | 30 +++---- types/priv_validator.go | 120 ++++++++++++++-------------- types/priv_validator/sign_info.go | 46 +++++------ types/validator_set.go | 49 +++++------- types/validator_set_test.go | 2 +- types/vote_set.go | 49 +++++------- 40 files changed, 262 insertions(+), 299 deletions(-) diff --git a/cmd/tendermint/commands/run_node.go b/cmd/tendermint/commands/run_node.go index 6ad92441b..0d50f9e4b 100644 --- a/cmd/tendermint/commands/run_node.go +++ b/cmd/tendermint/commands/run_node.go @@ -57,9 +57,8 @@ func NewRunNodeCmd(nodeProvider nm.NodeProvider) *cobra.Command { if err := n.Start(); err != nil { return fmt.Errorf("Failed to start node: %v", err) - } else { - logger.Info("Started node", "nodeInfo", n.Switch().NodeInfo()) } + logger.Info("Started node", "nodeInfo", n.Switch().NodeInfo()) // Trap signal, run forever. n.RunForever() diff --git a/config/config.go b/config/config.go index 0844fbd51..c87d56b3d 100644 --- a/config/config.go +++ b/config/config.go @@ -165,28 +165,28 @@ func TestBaseConfig() BaseConfig { return cfg } -func (b BaseConfig) ChainID() string { - return b.chainID +func (cfg BaseConfig) ChainID() string { + return cfg.chainID } // GenesisFile returns the full path to the genesis.json file -func (b BaseConfig) GenesisFile() string { - return rootify(b.Genesis, b.RootDir) +func (cfg BaseConfig) GenesisFile() string { + return rootify(cfg.Genesis, cfg.RootDir) } // PrivValidatorFile returns the full path to the priv_validator.json file -func (b BaseConfig) PrivValidatorFile() string { - return rootify(b.PrivValidator, b.RootDir) +func (cfg BaseConfig) PrivValidatorFile() string { + return rootify(cfg.PrivValidator, cfg.RootDir) } // NodeKeyFile returns the full path to the node_key.json file -func (b BaseConfig) NodeKeyFile() string { - return rootify(b.NodeKey, b.RootDir) +func (cfg BaseConfig) NodeKeyFile() string { + return rootify(cfg.NodeKey, cfg.RootDir) } // DBDir returns the full path to the database directory -func (b BaseConfig) DBDir() string { - return rootify(b.DBPath, b.RootDir) +func (cfg BaseConfig) DBDir() string { + return rootify(cfg.DBPath, cfg.RootDir) } // DefaultLogLevel returns a default log level of "error" @@ -321,8 +321,8 @@ func TestP2PConfig() *P2PConfig { } // AddrBookFile returns the full path to the address book -func (p *P2PConfig) AddrBookFile() string { - return rootify(p.AddrBook, p.RootDir) +func (cfg *P2PConfig) AddrBookFile() string { + return rootify(cfg.AddrBook, cfg.RootDir) } //----------------------------------------------------------------------------- @@ -357,8 +357,8 @@ func TestMempoolConfig() *MempoolConfig { } // WalDir returns the full path to the mempool's write-ahead log -func (m *MempoolConfig) WalDir() string { - return rootify(m.WalPath, m.RootDir) +func (cfg *MempoolConfig) WalDir() string { + return rootify(cfg.WalPath, cfg.RootDir) } //----------------------------------------------------------------------------- @@ -436,56 +436,56 @@ func TestConsensusConfig() *ConsensusConfig { } // WaitForTxs returns true if the consensus should wait for transactions before entering the propose step -func (c *ConsensusConfig) WaitForTxs() bool { - return !c.CreateEmptyBlocks || c.CreateEmptyBlocksInterval > 0 +func (cfg *ConsensusConfig) WaitForTxs() bool { + return !cfg.CreateEmptyBlocks || cfg.CreateEmptyBlocksInterval > 0 } // EmptyBlocks returns the amount of time to wait before proposing an empty block or starting the propose timer if there are no txs available -func (c *ConsensusConfig) EmptyBlocksInterval() time.Duration { - return time.Duration(c.CreateEmptyBlocksInterval) * time.Second +func (cfg *ConsensusConfig) EmptyBlocksInterval() time.Duration { + return time.Duration(cfg.CreateEmptyBlocksInterval) * time.Second } // Propose returns the amount of time to wait for a proposal -func (c *ConsensusConfig) Propose(round int) time.Duration { - return time.Duration(c.TimeoutPropose+c.TimeoutProposeDelta*round) * time.Millisecond +func (cfg *ConsensusConfig) Propose(round int) time.Duration { + return time.Duration(cfg.TimeoutPropose+cfg.TimeoutProposeDelta*round) * time.Millisecond } // Prevote returns the amount of time to wait for straggler votes after receiving any +2/3 prevotes -func (c *ConsensusConfig) Prevote(round int) time.Duration { - return time.Duration(c.TimeoutPrevote+c.TimeoutPrevoteDelta*round) * time.Millisecond +func (cfg *ConsensusConfig) Prevote(round int) time.Duration { + return time.Duration(cfg.TimeoutPrevote+cfg.TimeoutPrevoteDelta*round) * time.Millisecond } // Precommit returns the amount of time to wait for straggler votes after receiving any +2/3 precommits -func (c *ConsensusConfig) Precommit(round int) time.Duration { - return time.Duration(c.TimeoutPrecommit+c.TimeoutPrecommitDelta*round) * time.Millisecond +func (cfg *ConsensusConfig) Precommit(round int) time.Duration { + return time.Duration(cfg.TimeoutPrecommit+cfg.TimeoutPrecommitDelta*round) * time.Millisecond } // Commit returns the amount of time to wait for straggler votes after receiving +2/3 precommits for a single block (ie. a commit). -func (c *ConsensusConfig) Commit(t time.Time) time.Time { - return t.Add(time.Duration(c.TimeoutCommit) * time.Millisecond) +func (cfg *ConsensusConfig) Commit(t time.Time) time.Time { + return t.Add(time.Duration(cfg.TimeoutCommit) * time.Millisecond) } // PeerGossipSleep returns the amount of time to sleep if there is nothing to send from the ConsensusReactor -func (c *ConsensusConfig) PeerGossipSleep() time.Duration { - return time.Duration(c.PeerGossipSleepDuration) * time.Millisecond +func (cfg *ConsensusConfig) PeerGossipSleep() time.Duration { + return time.Duration(cfg.PeerGossipSleepDuration) * time.Millisecond } // PeerQueryMaj23Sleep returns the amount of time to sleep after each VoteSetMaj23Message is sent in the ConsensusReactor -func (c *ConsensusConfig) PeerQueryMaj23Sleep() time.Duration { - return time.Duration(c.PeerQueryMaj23SleepDuration) * time.Millisecond +func (cfg *ConsensusConfig) PeerQueryMaj23Sleep() time.Duration { + return time.Duration(cfg.PeerQueryMaj23SleepDuration) * time.Millisecond } // WalFile returns the full path to the write-ahead log file -func (c *ConsensusConfig) WalFile() string { - if c.walFile != "" { - return c.walFile +func (cfg *ConsensusConfig) WalFile() string { + if cfg.walFile != "" { + return cfg.walFile } - return rootify(c.WalPath, c.RootDir) + return rootify(cfg.WalPath, cfg.RootDir) } // SetWalFile sets the path to the write-ahead log file -func (c *ConsensusConfig) SetWalFile(walFile string) { - c.walFile = walFile +func (cfg *ConsensusConfig) SetWalFile(walFile string) { + cfg.walFile = walFile } //----------------------------------------------------------------------------- diff --git a/consensus/common_test.go b/consensus/common_test.go index 6d16958ca..287852e04 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -101,13 +101,13 @@ func signVotes(voteType byte, hash []byte, header types.PartSetHeader, vss ...*v func incrementHeight(vss ...*validatorStub) { for _, vs := range vss { - vs.Height += 1 + vs.Height++ } } func incrementRound(vss ...*validatorStub) { for _, vs := range vss { - vs.Round += 1 + vs.Round++ } } diff --git a/consensus/mempool_test.go b/consensus/mempool_test.go index d283ff4ba..31e4e1e62 100644 --- a/consensus/mempool_test.go +++ b/consensus/mempool_test.go @@ -200,7 +200,7 @@ func (app *CounterApplication) DeliverTx(tx []byte) abci.ResponseDeliverTx { Code: code.CodeTypeBadNonce, Log: fmt.Sprintf("Invalid nonce. Expected %v, got %v", app.txCount, txValue)} } - app.txCount += 1 + app.txCount++ return abci.ResponseDeliverTx{Code: code.CodeTypeOK} } @@ -211,7 +211,7 @@ func (app *CounterApplication) CheckTx(tx []byte) abci.ResponseCheckTx { Code: code.CodeTypeBadNonce, Log: fmt.Sprintf("Invalid nonce. Expected %v, got %v", app.mempoolTxCount, txValue)} } - app.mempoolTxCount += 1 + app.mempoolTxCount++ return abci.ResponseCheckTx{Code: code.CodeTypeOK} } @@ -225,9 +225,8 @@ func (app *CounterApplication) Commit() abci.ResponseCommit { app.mempoolTxCount = app.txCount if app.txCount == 0 { return abci.ResponseCommit{} - } else { - hash := make([]byte, 8) - binary.BigEndian.PutUint64(hash, uint64(app.txCount)) - return abci.ResponseCommit{Data: hash} } + hash := make([]byte, 8) + binary.BigEndian.PutUint64(hash, uint64(app.txCount)) + return abci.ResponseCommit{Data: hash} } diff --git a/consensus/reactor.go b/consensus/reactor.go index a265f76c0..b13be0e87 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -602,11 +602,9 @@ func (conR *ConsensusReactor) gossipDataForCatchup(logger log.Logger, rs *cstype logger.Debug("Sending block part for catchup failed") } return - } else { - //logger.Info("No parts to send in catch-up, sleeping") - time.Sleep(conR.conS.config.PeerGossipSleep()) - return } + //logger.Info("No parts to send in catch-up, sleeping") + time.Sleep(conR.conS.config.PeerGossipSleep()) } func (conR *ConsensusReactor) gossipVotesRoutine(peer p2p.Peer, ps *PeerState) { @@ -1087,7 +1085,7 @@ func (ps *PeerState) RecordVote(vote *types.Vote) int { return ps.stats.votes } ps.stats.lastVoteHeight = vote.Height - ps.stats.votes += 1 + ps.stats.votes++ return ps.stats.votes } @@ -1106,7 +1104,7 @@ func (ps *PeerState) RecordBlockPart(bp *BlockPartMessage) int { } ps.stats.lastBlockPartHeight = bp.Height - ps.stats.blockParts += 1 + ps.stats.blockParts++ return ps.stats.blockParts } diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index 26fc7e171..8448caf63 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -441,7 +441,7 @@ func waitForAndValidateBlockWithTx(t *testing.T, n int, activeVals map[string]st // but they should be in order. for _, tx := range newBlock.Data.Txs { assert.EqualValues(t, txs[ntxs], tx) - ntxs += 1 + ntxs++ } if ntxs == len(txs) { diff --git a/consensus/replay.go b/consensus/replay.go index 39dd592a4..5b5a48425 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -112,7 +112,7 @@ func (cs *ConsensusState) catchupReplay(csHeight int64) error { } } if found { - return fmt.Errorf("WAL should not contain #ENDHEIGHT %d.", csHeight) + return fmt.Errorf("WAL should not contain #ENDHEIGHT %d", csHeight) } // Search for last height marker @@ -125,7 +125,7 @@ func (cs *ConsensusState) catchupReplay(csHeight int64) error { return err } if !found { - return fmt.Errorf("Cannot replay height %d. WAL does not contain #ENDHEIGHT for %d.", csHeight, csHeight-1) + return fmt.Errorf("Cannot replay height %d. WAL does not contain #ENDHEIGHT for %d", csHeight, csHeight-1) } defer gr.Close() // nolint: errcheck @@ -352,7 +352,7 @@ func (h *Handshaker) replayBlocks(state sm.State, proxyApp proxy.AppConns, appBl var err error finalBlock := storeBlockHeight if mutateState { - finalBlock -= 1 + finalBlock-- } for i := appBlockHeight + 1; i <= finalBlock; i++ { h.logger.Info("Applying block", "height", i) @@ -362,7 +362,7 @@ func (h *Handshaker) replayBlocks(state sm.State, proxyApp proxy.AppConns, appBl return nil, err } - h.nBlocks += 1 + h.nBlocks++ } if mutateState { @@ -390,7 +390,7 @@ func (h *Handshaker) replayBlock(state sm.State, height int64, proxyApp proxy.Ap return sm.State{}, err } - h.nBlocks += 1 + h.nBlocks++ return state, nil } @@ -429,7 +429,7 @@ type mockProxyApp struct { func (mock *mockProxyApp) DeliverTx(tx []byte) abci.ResponseDeliverTx { r := mock.abciResponses.DeliverTx[mock.txCount] - mock.txCount += 1 + mock.txCount++ return *r } diff --git a/consensus/replay_file.go b/consensus/replay_file.go index 2fcd9b472..1fd4f415d 100644 --- a/consensus/replay_file.go +++ b/consensus/replay_file.go @@ -87,9 +87,9 @@ func (cs *ConsensusState) ReplayFile(file string, console bool) error { } if nextN > 0 { - nextN -= 1 + nextN-- } - pb.count += 1 + pb.count++ } return nil } @@ -153,7 +153,7 @@ func (pb *playback) replayReset(count int, newStepCh chan interface{}) error { if err := pb.cs.readReplayMessage(msg, newStepCh); err != nil { return err } - pb.count += 1 + pb.count++ } return nil } @@ -197,13 +197,12 @@ func (pb *playback) replayConsoleLoop() int { if len(tokens) == 1 { return 0 + } + i, err := strconv.Atoi(tokens[1]) + if err != nil { + fmt.Println("next takes an integer argument") } else { - i, err := strconv.Atoi(tokens[1]) - if err != nil { - fmt.Println("next takes an integer argument") - } else { - return i - } + return i } case "back": diff --git a/consensus/replay_test.go b/consensus/replay_test.go index 9d7bbc8e4..606df8260 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -382,9 +382,9 @@ func testHandshakeReplay(t *testing.T, nBlocks int, mode uint) { expectedBlocksToSync := NUM_BLOCKS - nBlocks if nBlocks == NUM_BLOCKS && mode > 0 { - expectedBlocksToSync += 1 + expectedBlocksToSync++ } else if nBlocks > 0 && mode == 1 { - expectedBlocksToSync += 1 + expectedBlocksToSync++ } if handshaker.NBlocks() != expectedBlocksToSync { @@ -533,7 +533,7 @@ func makeBlockchainFromWAL(wal WAL) ([]*types.Block, []*types.Commit, error) { } blocks = append(blocks, block) commits = append(commits, thisBlockCommit) - height += 1 + height++ } case *types.PartSetHeader: thisBlockParts = types.NewPartSetFromHeader(*p) diff --git a/consensus/state.go b/consensus/state.go index b59baf1ee..6acad698b 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -494,7 +494,7 @@ func (cs *ConsensusState) updateToState(state sm.State) { func (cs *ConsensusState) newStep() { rs := cs.RoundStateEvent() cs.wal.Save(rs) - cs.nSteps += 1 + cs.nSteps++ // newStep is called by updateToStep in NewConsensusState before the eventBus is set! if cs.eventBus != nil { cs.eventBus.PublishEventNewRoundStep(rs) @@ -741,7 +741,7 @@ func (cs *ConsensusState) proposalHeartbeat(height int64, round int) { } cs.privValidator.SignHeartbeat(chainID, heartbeat) cs.eventBus.PublishEventProposalHeartbeat(types.EventDataProposalHeartbeat{heartbeat}) - counter += 1 + counter++ time.Sleep(proposalHeartbeatIntervalSeconds * time.Second) } } @@ -852,10 +852,10 @@ func (cs *ConsensusState) isProposalComplete() bool { // make sure we have the prevotes from it too if cs.Proposal.POLRound < 0 { return true - } else { - // if this is false the proposer is lying or we haven't received the POL yet - return cs.Votes.Prevotes(cs.Proposal.POLRound).HasTwoThirdsMajority() } + // if this is false the proposer is lying or we haven't received the POL yet + return cs.Votes.Prevotes(cs.Proposal.POLRound).HasTwoThirdsMajority() + } // Create the next block to propose and return it. @@ -1498,12 +1498,11 @@ func (cs *ConsensusState) signAddVote(type_ byte, hash []byte, header types.Part cs.sendInternalMessage(msgInfo{&VoteMessage{vote}, ""}) cs.Logger.Info("Signed and pushed vote", "height", cs.Height, "round", cs.Round, "vote", vote, "err", err) return vote - } else { - //if !cs.replayMode { - cs.Logger.Error("Error signing vote", "height", cs.Height, "round", cs.Round, "vote", vote, "err", err) - //} - return nil } + //if !cs.replayMode { + cs.Logger.Error("Error signing vote", "height", cs.Height, "round", cs.Round, "vote", vote, "err", err) + //} + return nil } //--------------------------------------------------------- diff --git a/node/node.go b/node/node.go index 1f745bf00..3a59d2f06 100644 --- a/node/node.go +++ b/node/node.go @@ -643,14 +643,13 @@ func loadGenesisDoc(db dbm.DB) (*types.GenesisDoc, error) { bytes := db.Get(genesisDocKey) if len(bytes) == 0 { return nil, errors.New("Genesis doc not found") - } else { - var genDoc *types.GenesisDoc - err := json.Unmarshal(bytes, &genDoc) - if err != nil { - cmn.PanicCrisis(fmt.Sprintf("Failed to load genesis doc due to unmarshaling error: %v (bytes: %X)", err, bytes)) - } - return genDoc, nil } + var genDoc *types.GenesisDoc + err := json.Unmarshal(bytes, &genDoc) + if err != nil { + cmn.PanicCrisis(fmt.Sprintf("Failed to load genesis doc due to unmarshaling error: %v (bytes: %X)", err, bytes)) + } + return genDoc, nil } // panics if failed to marshal the given genesis document diff --git a/p2p/base_reactor.go b/p2p/base_reactor.go index f0dd14147..83c8efa4b 100644 --- a/p2p/base_reactor.go +++ b/p2p/base_reactor.go @@ -47,7 +47,7 @@ func NewBaseReactor(name string, impl Reactor) *BaseReactor { func (br *BaseReactor) SetSwitch(sw *Switch) { br.Switch = sw } -func (_ *BaseReactor) GetChannels() []*conn.ChannelDescriptor { return nil } -func (_ *BaseReactor) AddPeer(peer Peer) {} -func (_ *BaseReactor) RemovePeer(peer Peer, reason interface{}) {} -func (_ *BaseReactor) Receive(chID byte, peer Peer, msgBytes []byte) {} +func (*BaseReactor) GetChannels() []*conn.ChannelDescriptor { return nil } +func (*BaseReactor) AddPeer(peer Peer) {} +func (*BaseReactor) RemovePeer(peer Peer, reason interface{}) {} +func (*BaseReactor) Receive(chID byte, peer Peer, msgBytes []byte) {} diff --git a/p2p/conn/connection.go b/p2p/conn/connection.go index e4ae4df80..9a3360f2d 100644 --- a/p2p/conn/connection.go +++ b/p2p/conn/connection.go @@ -422,9 +422,8 @@ func (c *MConnection) sendMsgPacket() bool { // Nothing to send? if leastChannel == nil { return true - } else { - // c.Logger.Info("Found a msgPacket to send") } + // c.Logger.Info("Found a msgPacket to send") // Make & send a msgPacket from this channel n, err := leastChannel.writeMsgPacketTo(c.bufWriter) diff --git a/p2p/conn/secret_connection.go b/p2p/conn/secret_connection.go index 3495853e0..bc67abf3a 100644 --- a/p2p/conn/secret_connection.go +++ b/p2p/conn/secret_connection.go @@ -113,7 +113,7 @@ func (sc *SecretConnection) RemotePubKey() crypto.PubKey { // CONTRACT: data smaller than dataMaxSize is read atomically. func (sc *SecretConnection) Write(data []byte) (n int, err error) { for 0 < len(data) { - var frame []byte = make([]byte, totalFrameSize) + var frame = make([]byte, totalFrameSize) var chunk []byte if dataMaxSize < len(data) { chunk = data[:dataMaxSize] @@ -136,9 +136,8 @@ func (sc *SecretConnection) Write(data []byte) (n int, err error) { _, err := sc.conn.Write(sealedFrame) if err != nil { return n, err - } else { - n += len(chunk) } + n += len(chunk) } return } @@ -347,7 +346,7 @@ func incr2Nonce(nonce *[24]byte) { // increment nonce big-endian by 1 with wraparound. func incrNonce(nonce *[24]byte) { for i := 23; 0 <= i; i-- { - nonce[i] += 1 + nonce[i]++ if nonce[i] != 0 { return } diff --git a/p2p/conn/secret_connection_test.go b/p2p/conn/secret_connection_test.go index 54c453a7c..edb0de04d 100644 --- a/p2p/conn/secret_connection_test.go +++ b/p2p/conn/secret_connection_test.go @@ -180,7 +180,7 @@ func TestSecretConnectionReadWrite(t *testing.T) { var readCount = 0 for _, readChunk := range reads { read += readChunk - readCount += 1 + readCount++ if len(write) <= len(read) { break } diff --git a/p2p/key.go b/p2p/key.go index ea0f0b071..6d0f28586 100644 --- a/p2p/key.go +++ b/p2p/key.go @@ -53,9 +53,8 @@ func LoadOrGenNodeKey(filePath string) (*NodeKey, error) { return nil, err } return nodeKey, nil - } else { - return genNodeKey(filePath) } + return genNodeKey(filePath) } func loadNodeKey(filePath string) (*NodeKey, error) { @@ -66,7 +65,7 @@ func loadNodeKey(filePath string) (*NodeKey, error) { nodeKey := new(NodeKey) err = json.Unmarshal(jsonBytes, nodeKey) if err != nil { - return nil, fmt.Errorf("Error reading NodeKey from %v: %v\n", filePath, err) + return nil, fmt.Errorf("Error reading NodeKey from %v: %v", filePath, err) } return nodeKey, nil } diff --git a/p2p/netaddress.go b/p2p/netaddress.go index 333d16e5d..c7773a9f8 100644 --- a/p2p/netaddress.go +++ b/p2p/netaddress.go @@ -294,7 +294,7 @@ func (na *NetAddress) RFC6145() bool { return rfc6145.Contains(na.IP) } func removeProtocolIfDefined(addr string) string { if strings.Contains(addr, "://") { return strings.Split(addr, "://")[1] - } else { - return addr } + return addr + } diff --git a/p2p/peer_set.go b/p2p/peer_set.go index dc53174a1..a4565ea1d 100644 --- a/p2p/peer_set.go +++ b/p2p/peer_set.go @@ -68,9 +68,8 @@ func (ps *PeerSet) Get(peerKey ID) Peer { item, ok := ps.lookup[peerKey] if ok { return item.peer - } else { - return nil } + return nil } // Remove discards peer by its Key, if the peer was previously memoized. diff --git a/p2p/peer_set_test.go b/p2p/peer_set_test.go index 174e73c9c..c8d1d64d2 100644 --- a/p2p/peer_set_test.go +++ b/p2p/peer_set_test.go @@ -115,7 +115,7 @@ func TestPeerSetAddDuplicate(t *testing.T) { errsTally := make(map[error]int) for i := 0; i < n; i++ { err := <-errsChan - errsTally[err] += 1 + errsTally[err]++ } // Our next procedure is to ensure that only one addition diff --git a/p2p/pex/known_address.go b/p2p/pex/known_address.go index 085eb10fa..0261e4902 100644 --- a/p2p/pex/known_address.go +++ b/p2p/pex/known_address.go @@ -56,7 +56,7 @@ func (ka *knownAddress) isNew() bool { func (ka *knownAddress) markAttempt() { now := time.Now() ka.LastAttempt = now - ka.Attempts += 1 + ka.Attempts++ } func (ka *knownAddress) markGood() { diff --git a/p2p/pex/pex_reactor.go b/p2p/pex/pex_reactor.go index 45a05bdb7..45f689c05 100644 --- a/p2p/pex/pex_reactor.go +++ b/p2p/pex/pex_reactor.go @@ -467,9 +467,8 @@ func (r *PEXReactor) AttemptsToDial(addr *p2p.NetAddress) int { lAttempts, attempted := r.attemptsToDial.Load(addr.DialString()) if attempted { return lAttempts.(_attemptsToDial).number - } else { - return 0 } + return 0 } //---------------------------------------------------------- diff --git a/p2p/test_util.go b/p2p/test_util.go index 0d6427f43..535b0bd0b 100644 --- a/p2p/test_util.go +++ b/p2p/test_util.go @@ -143,7 +143,7 @@ func MakeSwitch(cfg *cfg.P2PConfig, i int, network, version string, initSwitch f Version: version, ListenAddr: cmn.Fmt("%v:%v", network, rand.Intn(64512)+1023), } - for ch, _ := range sw.reactorsByCh { + for ch := range sw.reactorsByCh { ni.Channels = append(ni.Channels, ch) } sw.SetNodeInfo(ni) diff --git a/p2p/upnp/upnp.go b/p2p/upnp/upnp.go index cac67a730..e98538aae 100644 --- a/p2p/upnp/upnp.go +++ b/p2p/upnp/upnp.go @@ -103,7 +103,7 @@ func Discover() (nat NAT, err error) { return } } - err = errors.New("UPnP port discovery failed.") + err = errors.New("UPnP port discovery failed") return } diff --git a/rpc/core/tx.go b/rpc/core/tx.go index 18120ded1..7ddc70809 100644 --- a/rpc/core/tx.go +++ b/rpc/core/tx.go @@ -73,7 +73,7 @@ func Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { // if index is disabled, return error if _, ok := txIndexer.(*null.TxIndex); ok { - return nil, fmt.Errorf("Transaction indexing is disabled.") + return nil, fmt.Errorf("Transaction indexing is disabled") } r, err := txIndexer.Get(hash) @@ -169,7 +169,7 @@ func Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { func TxSearch(query string, prove bool) ([]*ctypes.ResultTx, error) { // if index is disabled, return error if _, ok := txIndexer.(*null.TxIndex); ok { - return nil, fmt.Errorf("Transaction indexing is disabled.") + return nil, fmt.Errorf("Transaction indexing is disabled") } q, err := tmquery.New(query) diff --git a/rpc/lib/client/ws_client.go b/rpc/lib/client/ws_client.go index fe15cda21..ab2e94d0b 100644 --- a/rpc/lib/client/ws_client.go +++ b/rpc/lib/client/ws_client.go @@ -318,21 +318,21 @@ func (c *WSClient) reconnectRoutine() { c.Logger.Error("failed to reconnect", "err", err, "original_err", originalError) c.Stop() return - } else { - // drain reconnectAfter - LOOP: - for { - select { - case <-c.reconnectAfter: - default: - break LOOP - } - } - err = c.processBacklog() - if err == nil { - c.startReadWriteRoutines() + } + // drain reconnectAfter + LOOP: + for { + select { + case <-c.reconnectAfter: + default: + break LOOP } } + err := c.processBacklog() + if err == nil { + c.startReadWriteRoutines() + } + case <-c.Quit(): return } diff --git a/rpc/lib/types/types.go b/rpc/lib/types/types.go index 37d451457..e4b02c58f 100644 --- a/rpc/lib/types/types.go +++ b/rpc/lib/types/types.go @@ -101,9 +101,8 @@ func NewRPCErrorResponse(id string, code int, msg string, data string) RPCRespon func (resp RPCResponse) String() string { if resp.Error == nil { return fmt.Sprintf("[%s %v]", resp.ID, resp.Result) - } else { - return fmt.Sprintf("[%s %s]", resp.ID, resp.Error) } + return fmt.Sprintf("[%s %s]", resp.ID, resp.Error) } func RPCParseError(id string, err error) RPCResponse { diff --git a/scripts/wal2json/main.go b/scripts/wal2json/main.go index e44ed4b17..f6ffea431 100644 --- a/scripts/wal2json/main.go +++ b/scripts/wal2json/main.go @@ -4,6 +4,7 @@ Usage: wal2json */ + package main import ( diff --git a/state/state_test.go b/state/state_test.go index 1c5a860d5..31f2065cf 100644 --- a/state/state_test.go +++ b/state/state_test.go @@ -222,7 +222,7 @@ func TestOneValidatorChangesSaveLoad(t *testing.T) { // use the next pubkey if changeIndex < len(changeHeights) && i == changeHeights[changeIndex] { changeIndex++ - power += 1 + power++ } header, blockID, responses := makeHeaderPartsResponsesValPowerChange(state, i, power) state, err = updateState(state, blockID, header, responses) @@ -240,7 +240,7 @@ func TestOneValidatorChangesSaveLoad(t *testing.T) { // use the next pubkey (note our counter starts at 0 this time) if changeIndex < len(changeHeights) && i == changeHeights[changeIndex]+1 { changeIndex++ - power += 1 + power++ } testCases[i-1] = power } diff --git a/state/txindex/kv/kv.go b/state/txindex/kv/kv.go index 1f544e58b..74bf4843b 100644 --- a/state/txindex/kv/kv.go +++ b/state/txindex/kv/kv.go @@ -148,9 +148,8 @@ func (txi *TxIndex) Search(q *query.Query) ([]*types.TxResult, error) { res, err := txi.Get(hash) if res == nil { return []*types.TxResult{}, nil - } else { - return []*types.TxResult{res}, errors.Wrap(err, "error while retrieving the result") } + return []*types.TxResult{res}, errors.Wrap(err, "error while retrieving the result") } // conditions to skip because they're handled before "everything else" diff --git a/state/txindex/kv/kv_test.go b/state/txindex/kv/kv_test.go index 810da23bd..97024b30b 100644 --- a/state/txindex/kv/kv_test.go +++ b/state/txindex/kv/kv_test.go @@ -196,7 +196,7 @@ func benchmarkTxIndex(txsCount int, b *testing.B) { if err := batch.Add(txResult); err != nil { b.Fatal(err) } - txResult.Index += 1 + txResult.Index++ } b.ResetTimer() diff --git a/types/block.go b/types/block.go index 31443b1a2..0c1d38e55 100644 --- a/types/block.go +++ b/types/block.go @@ -141,9 +141,8 @@ func (b *Block) StringIndented(indent string) string { func (b *Block) StringShort() string { if b == nil { return "nil-Block" - } else { - return fmt.Sprintf("Block#%v", b.Hash()) } + return fmt.Sprintf("Block#%v", b.Hash()) } //----------------------------------------------------------------------------- diff --git a/types/block_test.go b/types/block_test.go index e3e22743c..4aaa16a97 100644 --- a/types/block_test.go +++ b/types/block_test.go @@ -29,7 +29,7 @@ func TestValidateBlock(t *testing.T) { // tamper with NumTxs block = MakeBlock(h, txs, commit) - block.NumTxs += 1 + block.NumTxs++ err = block.ValidateBasic() require.Error(t, err) diff --git a/types/evidence.go b/types/evidence.go index 9e1f6af0e..0b3496045 100644 --- a/types/evidence.go +++ b/types/evidence.go @@ -144,7 +144,7 @@ func (dve *DuplicateVoteEvidence) Verify(chainID string) error { // BlockIDs must be different if dve.VoteA.BlockID.Equals(dve.VoteB.BlockID) { - return fmt.Errorf("DuplicateVoteEvidence Error: BlockIDs are the same (%v) - not a real duplicate vote!", dve.VoteA.BlockID) + return fmt.Errorf("DuplicateVoteEvidence Error: BlockIDs are the same (%v) - not a real duplicate vote", dve.VoteA.BlockID) } // Signatures must be valid diff --git a/types/genesis.go b/types/genesis.go index 8d885db97..21c61806a 100644 --- a/types/genesis.go +++ b/types/genesis.go @@ -37,9 +37,8 @@ type GenesisDoc struct { func (genDoc *GenesisDoc) AppState() json.RawMessage { if len(genDoc.AppOptions) > 0 { return genDoc.AppOptions - } else { - return genDoc.AppStateJSON } + return genDoc.AppStateJSON } // SaveAs is a utility method for saving GenensisDoc as a JSON file. diff --git a/types/part_set.go b/types/part_set.go index 5c43b1518..749943291 100644 --- a/types/part_set.go +++ b/types/part_set.go @@ -30,12 +30,11 @@ type Part struct { func (part *Part) Hash() []byte { if part.hash != nil { return part.hash - } else { - hasher := ripemd160.New() - hasher.Write(part.Bytes) // nolint: errcheck, gas - part.hash = hasher.Sum(nil) - return part.hash } + hasher := ripemd160.New() + hasher.Write(part.Bytes) // nolint: errcheck, gas + part.hash = hasher.Sum(nil) + return part.hash } func (part *Part) String() string { @@ -129,20 +128,18 @@ func NewPartSetFromHeader(header PartSetHeader) *PartSet { func (ps *PartSet) Header() PartSetHeader { if ps == nil { return PartSetHeader{} - } else { - return PartSetHeader{ - Total: ps.total, - Hash: ps.hash, - } + } + return PartSetHeader{ + Total: ps.total, + Hash: ps.hash, } } func (ps *PartSet) HasHeader(header PartSetHeader) bool { if ps == nil { return false - } else { - return ps.Header().Equals(header) } + return ps.Header().Equals(header) } func (ps *PartSet) BitArray() *cmn.BitArray { @@ -251,7 +248,7 @@ func (psr *PartSetReader) Read(p []byte) (n int, err error) { return n1 + n2, err } - psr.i += 1 + psr.i++ if psr.i >= len(psr.parts) { return 0, io.EOF } @@ -262,9 +259,8 @@ func (psr *PartSetReader) Read(p []byte) (n int, err error) { func (ps *PartSet) StringShort() string { if ps == nil { return "nil-PartSet" - } else { - ps.mtx.Lock() - defer ps.mtx.Unlock() - return fmt.Sprintf("(%v of %v)", ps.Count(), ps.Total()) } + ps.mtx.Lock() + defer ps.mtx.Unlock() + return fmt.Sprintf("(%v of %v)", ps.Count(), ps.Total()) } diff --git a/types/priv_validator.go b/types/priv_validator.go index daa456bc0..052ace978 100644 --- a/types/priv_validator.go +++ b/types/priv_validator.go @@ -211,18 +211,18 @@ func LoadPrivValidatorFSWithSigner(filePath string, signerFunc func(PrivValidato } // Save persists the PrivValidatorFS to disk. -func (privVal *PrivValidatorFS) Save() { - privVal.mtx.Lock() - defer privVal.mtx.Unlock() - privVal.save() +func (pv *PrivValidatorFS) Save() { + pv.mtx.Lock() + defer pv.mtx.Unlock() + pv.save() } -func (privVal *PrivValidatorFS) save() { - outFile := privVal.filePath +func (pv *PrivValidatorFS) save() { + outFile := pv.filePath if outFile == "" { panic("Cannot save PrivValidator: filePath not set") } - jsonBytes, err := json.Marshal(privVal) + jsonBytes, err := json.Marshal(pv) if err != nil { panic(err) } @@ -234,22 +234,22 @@ func (privVal *PrivValidatorFS) save() { // Reset resets all fields in the PrivValidatorFS. // NOTE: Unsafe! -func (privVal *PrivValidatorFS) Reset() { +func (pv *PrivValidatorFS) Reset() { var sig crypto.Signature - privVal.LastHeight = 0 - privVal.LastRound = 0 - privVal.LastStep = 0 - privVal.LastSignature = sig - privVal.LastSignBytes = nil - privVal.Save() + pv.LastHeight = 0 + pv.LastRound = 0 + pv.LastStep = 0 + pv.LastSignature = sig + pv.LastSignBytes = nil + pv.Save() } // SignVote signs a canonical representation of the vote, along with the // chainID. Implements PrivValidator. -func (privVal *PrivValidatorFS) SignVote(chainID string, vote *Vote) error { - privVal.mtx.Lock() - defer privVal.mtx.Unlock() - if err := privVal.signVote(chainID, vote); err != nil { +func (pv *PrivValidatorFS) SignVote(chainID string, vote *Vote) error { + pv.mtx.Lock() + defer pv.mtx.Unlock() + if err := pv.signVote(chainID, vote); err != nil { return errors.New(cmn.Fmt("Error signing vote: %v", err)) } return nil @@ -257,32 +257,32 @@ func (privVal *PrivValidatorFS) SignVote(chainID string, vote *Vote) error { // SignProposal signs a canonical representation of the proposal, along with // the chainID. Implements PrivValidator. -func (privVal *PrivValidatorFS) SignProposal(chainID string, proposal *Proposal) error { - privVal.mtx.Lock() - defer privVal.mtx.Unlock() - if err := privVal.signProposal(chainID, proposal); err != nil { +func (pv *PrivValidatorFS) SignProposal(chainID string, proposal *Proposal) error { + pv.mtx.Lock() + defer pv.mtx.Unlock() + if err := pv.signProposal(chainID, proposal); err != nil { return fmt.Errorf("Error signing proposal: %v", err) } return nil } // returns error if HRS regression or no LastSignBytes. returns true if HRS is unchanged -func (privVal *PrivValidatorFS) checkHRS(height int64, round int, step int8) (bool, error) { - if privVal.LastHeight > height { +func (pv *PrivValidatorFS) checkHRS(height int64, round int, step int8) (bool, error) { + if pv.LastHeight > height { return false, errors.New("Height regression") } - if privVal.LastHeight == height { - if privVal.LastRound > round { + if pv.LastHeight == height { + if pv.LastRound > round { return false, errors.New("Round regression") } - if privVal.LastRound == round { - if privVal.LastStep > step { + if pv.LastRound == round { + if pv.LastStep > step { return false, errors.New("Step regression") - } else if privVal.LastStep == step { - if privVal.LastSignBytes != nil { - if privVal.LastSignature.Empty() { + } else if pv.LastStep == step { + if pv.LastSignBytes != nil { + if pv.LastSignature.Empty() { panic("privVal: LastSignature is nil but LastSignBytes is not!") } return true, nil @@ -297,11 +297,11 @@ func (privVal *PrivValidatorFS) checkHRS(height int64, round int, step int8) (bo // signVote checks if the vote is good to sign and sets the vote signature. // It may need to set the timestamp as well if the vote is otherwise the same as // a previously signed vote (ie. we crashed after signing but before the vote hit the WAL). -func (privVal *PrivValidatorFS) signVote(chainID string, vote *Vote) error { +func (pv *PrivValidatorFS) signVote(chainID string, vote *Vote) error { height, round, step := vote.Height, vote.Round, voteToStep(vote) signBytes := vote.SignBytes(chainID) - sameHRS, err := privVal.checkHRS(height, round, step) + sameHRS, err := pv.checkHRS(height, round, step) if err != nil { return err } @@ -312,11 +312,11 @@ func (privVal *PrivValidatorFS) signVote(chainID string, vote *Vote) error { // If they only differ by timestamp, use last timestamp and signature // Otherwise, return error if sameHRS { - if bytes.Equal(signBytes, privVal.LastSignBytes) { - vote.Signature = privVal.LastSignature - } else if timestamp, ok := checkVotesOnlyDifferByTimestamp(privVal.LastSignBytes, signBytes); ok { + if bytes.Equal(signBytes, pv.LastSignBytes) { + vote.Signature = pv.LastSignature + } else if timestamp, ok := checkVotesOnlyDifferByTimestamp(pv.LastSignBytes, signBytes); ok { vote.Timestamp = timestamp - vote.Signature = privVal.LastSignature + vote.Signature = pv.LastSignature } else { err = fmt.Errorf("Conflicting data") } @@ -324,11 +324,11 @@ func (privVal *PrivValidatorFS) signVote(chainID string, vote *Vote) error { } // It passed the checks. Sign the vote - sig, err := privVal.Sign(signBytes) + sig, err := pv.Sign(signBytes) if err != nil { return err } - privVal.saveSigned(height, round, step, signBytes, sig) + pv.saveSigned(height, round, step, signBytes, sig) vote.Signature = sig return nil } @@ -336,11 +336,11 @@ func (privVal *PrivValidatorFS) signVote(chainID string, vote *Vote) error { // signProposal checks if the proposal is good to sign and sets the proposal signature. // It may need to set the timestamp as well if the proposal is otherwise the same as // a previously signed proposal ie. we crashed after signing but before the proposal hit the WAL). -func (privVal *PrivValidatorFS) signProposal(chainID string, proposal *Proposal) error { +func (pv *PrivValidatorFS) signProposal(chainID string, proposal *Proposal) error { height, round, step := proposal.Height, proposal.Round, stepPropose signBytes := proposal.SignBytes(chainID) - sameHRS, err := privVal.checkHRS(height, round, step) + sameHRS, err := pv.checkHRS(height, round, step) if err != nil { return err } @@ -351,11 +351,11 @@ func (privVal *PrivValidatorFS) signProposal(chainID string, proposal *Proposal) // If they only differ by timestamp, use last timestamp and signature // Otherwise, return error if sameHRS { - if bytes.Equal(signBytes, privVal.LastSignBytes) { - proposal.Signature = privVal.LastSignature - } else if timestamp, ok := checkProposalsOnlyDifferByTimestamp(privVal.LastSignBytes, signBytes); ok { + if bytes.Equal(signBytes, pv.LastSignBytes) { + proposal.Signature = pv.LastSignature + } else if timestamp, ok := checkProposalsOnlyDifferByTimestamp(pv.LastSignBytes, signBytes); ok { proposal.Timestamp = timestamp - proposal.Signature = privVal.LastSignature + proposal.Signature = pv.LastSignature } else { err = fmt.Errorf("Conflicting data") } @@ -363,40 +363,40 @@ func (privVal *PrivValidatorFS) signProposal(chainID string, proposal *Proposal) } // It passed the checks. Sign the proposal - sig, err := privVal.Sign(signBytes) + sig, err := pv.Sign(signBytes) if err != nil { return err } - privVal.saveSigned(height, round, step, signBytes, sig) + pv.saveSigned(height, round, step, signBytes, sig) proposal.Signature = sig return nil } // Persist height/round/step and signature -func (privVal *PrivValidatorFS) saveSigned(height int64, round int, step int8, +func (pv *PrivValidatorFS) saveSigned(height int64, round int, step int8, signBytes []byte, sig crypto.Signature) { - privVal.LastHeight = height - privVal.LastRound = round - privVal.LastStep = step - privVal.LastSignature = sig - privVal.LastSignBytes = signBytes - privVal.save() + pv.LastHeight = height + pv.LastRound = round + pv.LastStep = step + pv.LastSignature = sig + pv.LastSignBytes = signBytes + pv.save() } // SignHeartbeat signs a canonical representation of the heartbeat, along with the chainID. // Implements PrivValidator. -func (privVal *PrivValidatorFS) SignHeartbeat(chainID string, heartbeat *Heartbeat) error { - privVal.mtx.Lock() - defer privVal.mtx.Unlock() +func (pv *PrivValidatorFS) SignHeartbeat(chainID string, heartbeat *Heartbeat) error { + pv.mtx.Lock() + defer pv.mtx.Unlock() var err error - heartbeat.Signature, err = privVal.Sign(heartbeat.SignBytes(chainID)) + heartbeat.Signature, err = pv.Sign(heartbeat.SignBytes(chainID)) return err } // String returns a string representation of the PrivValidatorFS. -func (privVal *PrivValidatorFS) String() string { - return fmt.Sprintf("PrivValidator{%v LH:%v, LR:%v, LS:%v}", privVal.GetAddress(), privVal.LastHeight, privVal.LastRound, privVal.LastStep) +func (pv *PrivValidatorFS) String() string { + return fmt.Sprintf("PrivValidator{%v LH:%v, LR:%v, LS:%v}", pv.GetAddress(), pv.LastHeight, pv.LastRound, pv.LastStep) } //------------------------------------- diff --git a/types/priv_validator/sign_info.go b/types/priv_validator/sign_info.go index 746131a96..8b135df63 100644 --- a/types/priv_validator/sign_info.go +++ b/types/priv_validator/sign_info.go @@ -49,30 +49,30 @@ func NewLastSignedInfo() *LastSignedInfo { } } -func (info *LastSignedInfo) String() string { - return fmt.Sprintf("LH:%v, LR:%v, LS:%v", info.Height, info.Round, info.Step) +func (lsi *LastSignedInfo) String() string { + return fmt.Sprintf("LH:%v, LR:%v, LS:%v", lsi.Height, lsi.Round, lsi.Step) } // Verify returns an error if there is a height/round/step regression // or if the HRS matches but there are no LastSignBytes. // It returns true if HRS matches exactly and the LastSignature exists. // It panics if the HRS matches, the LastSignBytes are not empty, but the LastSignature is empty. -func (info LastSignedInfo) Verify(height int64, round int, step int8) (bool, error) { - if info.Height > height { +func (lsi LastSignedInfo) Verify(height int64, round int, step int8) (bool, error) { + if lsi.Height > height { return false, errors.New("Height regression") } - if info.Height == height { - if info.Round > round { + if lsi.Height == height { + if lsi.Round > round { return false, errors.New("Round regression") } - if info.Round == round { - if info.Step > step { + if lsi.Round == round { + if lsi.Step > step { return false, errors.New("Step regression") - } else if info.Step == step { - if info.SignBytes != nil { - if info.Signature.Empty() { + } else if lsi.Step == step { + if lsi.SignBytes != nil { + if lsi.Signature.Empty() { panic("info: LastSignature is nil but LastSignBytes is not!") } return true, nil @@ -85,24 +85,24 @@ func (info LastSignedInfo) Verify(height int64, round int, step int8) (bool, err } // Set height/round/step and signature on the info -func (info *LastSignedInfo) Set(height int64, round int, step int8, +func (lsi *LastSignedInfo) Set(height int64, round int, step int8, signBytes []byte, sig crypto.Signature) { - info.Height = height - info.Round = round - info.Step = step - info.Signature = sig - info.SignBytes = signBytes + lsi.Height = height + lsi.Round = round + lsi.Step = step + lsi.Signature = sig + lsi.SignBytes = signBytes } // Reset resets all the values. // XXX: Unsafe. -func (info *LastSignedInfo) Reset() { - info.Height = 0 - info.Round = 0 - info.Step = 0 - info.Signature = crypto.Signature{} - info.SignBytes = nil +func (lsi *LastSignedInfo) Reset() { + lsi.Height = 0 + lsi.Round = 0 + lsi.Step = 0 + lsi.Signature = crypto.Signature{} + lsi.SignBytes = nil } // SignVote checks the height/round/step (HRS) are greater than the latest state of the LastSignedInfo. diff --git a/types/validator_set.go b/types/validator_set.go index 83d066ec1..dc6b66d68 100644 --- a/types/validator_set.go +++ b/types/validator_set.go @@ -96,9 +96,8 @@ func (valSet *ValidatorSet) GetByAddress(address []byte) (index int, val *Valida }) if idx != len(valSet.Validators) && bytes.Equal(valSet.Validators[idx].Address, address) { return idx, valSet.Validators[idx].Copy() - } else { - return 0, nil } + return 0, nil } // GetByIndex returns the validator by index. @@ -187,13 +186,12 @@ func (valSet *ValidatorSet) Update(val *Validator) (updated bool) { index, sameVal := valSet.GetByAddress(val.Address) if sameVal == nil { return false - } else { - valSet.Validators[index] = val.Copy() - // Invalidate cache - valSet.Proposer = nil - valSet.totalVotingPower = 0 - return true } + valSet.Validators[index] = val.Copy() + // Invalidate cache + valSet.Proposer = nil + valSet.totalVotingPower = 0 + return true } func (valSet *ValidatorSet) Remove(address []byte) (val *Validator, removed bool) { @@ -202,18 +200,17 @@ func (valSet *ValidatorSet) Remove(address []byte) (val *Validator, removed bool }) if idx == len(valSet.Validators) || !bytes.Equal(valSet.Validators[idx].Address, address) { return nil, false - } else { - removedVal := valSet.Validators[idx] - newValidators := valSet.Validators[:idx] - if idx+1 < len(valSet.Validators) { - newValidators = append(newValidators, valSet.Validators[idx+1:]...) - } - valSet.Validators = newValidators - // Invalidate cache - valSet.Proposer = nil - valSet.totalVotingPower = 0 - return removedVal, true } + removedVal := valSet.Validators[idx] + newValidators := valSet.Validators[:idx] + if idx+1 < len(valSet.Validators) { + newValidators = append(newValidators, valSet.Validators[idx+1:]...) + } + valSet.Validators = newValidators + // Invalidate cache + valSet.Proposer = nil + valSet.totalVotingPower = 0 + return removedVal, true } func (valSet *ValidatorSet) Iterate(fn func(index int, val *Validator) bool) { @@ -266,10 +263,9 @@ func (valSet *ValidatorSet) VerifyCommit(chainID string, blockID BlockID, height if talliedVotingPower > valSet.TotalVotingPower()*2/3 { return nil - } else { - return fmt.Errorf("Invalid commit -- insufficient voting power: got %v, needed %v", - talliedVotingPower, (valSet.TotalVotingPower()*2/3 + 1)) } + return fmt.Errorf("Invalid commit -- insufficient voting power: got %v, needed %v", + talliedVotingPower, (valSet.TotalVotingPower()*2/3 + 1)) } // VerifyCommitAny will check to see if the set would @@ -472,9 +468,8 @@ func safeMulClip(a, b int64) int64 { if overflow { if (a < 0 || b < 0) && !(a < 0 && b < 0) { return math.MinInt64 - } else { - return math.MaxInt64 } + return math.MaxInt64 } return c } @@ -484,9 +479,8 @@ func safeAddClip(a, b int64) int64 { if overflow { if b < 0 { return math.MinInt64 - } else { - return math.MaxInt64 } + return math.MaxInt64 } return c } @@ -496,9 +490,8 @@ func safeSubClip(a, b int64) int64 { if overflow { if b > 0 { return math.MinInt64 - } else { - return math.MaxInt64 } + return math.MaxInt64 } return c } diff --git a/types/validator_set_test.go b/types/validator_set_test.go index b346be1be..8769ac5ff 100644 --- a/types/validator_set_test.go +++ b/types/validator_set_test.go @@ -127,7 +127,7 @@ func TestProposerSelection2(t *testing.T) { for i := 0; i < 120*N; i++ { prop := vals.GetProposer() ii := prop.Address[19] - propCount[ii] += 1 + propCount[ii]++ vals.IncrementAccum(1) } diff --git a/types/vote_set.go b/types/vote_set.go index 37f26f4a5..e255488d6 100644 --- a/types/vote_set.go +++ b/types/vote_set.go @@ -94,33 +94,29 @@ func (voteSet *VoteSet) ChainID() string { func (voteSet *VoteSet) Height() int64 { if voteSet == nil { return 0 - } else { - return voteSet.height } + return voteSet.height } func (voteSet *VoteSet) Round() int { if voteSet == nil { return -1 - } else { - return voteSet.round } + return voteSet.round } func (voteSet *VoteSet) Type() byte { if voteSet == nil { return 0x00 - } else { - return voteSet.type_ } + return voteSet.type_ } func (voteSet *VoteSet) Size() int { if voteSet == nil { return 0 - } else { - return voteSet.valSet.Size() } + return voteSet.valSet.Size() } // Returns added=true if vote is valid and new. @@ -185,9 +181,8 @@ func (voteSet *VoteSet) addVote(vote *Vote) (added bool, err error) { if existing, ok := voteSet.getVote(valIndex, blockKey); ok { if existing.Signature.Equals(vote.Signature) { return false, nil // duplicate - } else { - return false, errors.Wrapf(ErrVoteNonDeterministicSignature, "Existing vote: %v; New vote: %v", existing, vote) } + return false, errors.Wrapf(ErrVoteNonDeterministicSignature, "Existing vote: %v; New vote: %v", existing, vote) } // Check signature. @@ -199,13 +194,11 @@ func (voteSet *VoteSet) addVote(vote *Vote) (added bool, err error) { added, conflicting := voteSet.addVerifiedVote(vote, blockKey, val.VotingPower) if conflicting != nil { return added, NewConflictingVoteError(val, conflicting, vote) - } else { - if !added { - cmn.PanicSanity("Expected to add non-conflicting vote") - } - return added, nil } - + if !added { + cmn.PanicSanity("Expected to add non-conflicting vote") + } + return added, nil } // Returns (vote, true) if vote exists for valIndex and blockKey @@ -257,13 +250,12 @@ func (voteSet *VoteSet) addVerifiedVote(vote *Vote, blockKey string, votingPower // ... and there's a conflicting vote. // We're not even tracking this blockKey, so just forget it. return false, conflicting - } else { - // ... and there's no conflicting vote. - // Start tracking this blockKey - votesByBlock = newBlockVotes(false, voteSet.valSet.Size()) - voteSet.votesByBlock[blockKey] = votesByBlock - // We'll add the vote in a bit. } + // ... and there's no conflicting vote. + // Start tracking this blockKey + votesByBlock = newBlockVotes(false, voteSet.valSet.Size()) + voteSet.votesByBlock[blockKey] = votesByBlock + // We'll add the vote in a bit. } // Before adding to votesByBlock, see if we'll exceed quorum @@ -309,10 +301,9 @@ func (voteSet *VoteSet) SetPeerMaj23(peerID P2PID, blockID BlockID) error { if existing, ok := voteSet.peerMaj23s[peerID]; ok { if existing.Equals(blockID) { return nil // Nothing to do - } else { - return fmt.Errorf("SetPeerMaj23: Received conflicting blockID from peer %v. Got %v, expected %v", - peerID, blockID, existing) } + return fmt.Errorf("SetPeerMaj23: Received conflicting blockID from peer %v. Got %v, expected %v", + peerID, blockID, existing) } voteSet.peerMaj23s[peerID] = blockID @@ -321,10 +312,9 @@ func (voteSet *VoteSet) SetPeerMaj23(peerID P2PID, blockID BlockID) error { if ok { if votesByBlock.peerMaj23 { return nil // Nothing to do - } else { - votesByBlock.peerMaj23 = true - // No need to copy votes, already there. } + votesByBlock.peerMaj23 = true + // No need to copy votes, already there. } else { votesByBlock = newBlockVotes(true, voteSet.valSet.Size()) voteSet.votesByBlock[blockKey] = votesByBlock @@ -422,9 +412,8 @@ func (voteSet *VoteSet) TwoThirdsMajority() (blockID BlockID, ok bool) { defer voteSet.mtx.Unlock() if voteSet.maj23 != nil { return *voteSet.maj23, true - } else { - return BlockID{}, false } + return BlockID{}, false } func (voteSet *VoteSet) String() string { From 70e7454c218bccebb6c5e125e34288efd4d09626 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 27 Mar 2018 11:44:53 -0400 Subject: [PATCH 020/143] comment out test_libs because of gcc dep in tmlibs --- .circleci/config.yml | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cb2577908..11ad5f33f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -130,18 +130,18 @@ jobs: paths: - "profiles/*" - test_libs: - <<: *defaults - steps: - - attach_workspace: - at: /tmp/workspace - - restore_cache: - key: v1-pkg-cache - - restore_cache: - key: v1-tree-{{ .Environment.CIRCLE_SHA1 }} - - run: - name: Run tests - command: bash test/test_libs.sh + #test_libs: + # <<: *defaults + # steps: + # - attach_workspace: + # at: /tmp/workspace + # - restore_cache: + # key: v1-pkg-cache + # - restore_cache: + # key: v1-tree-{{ .Environment.CIRCLE_SHA1 }} + # - run: + # name: Run tests + # command: bash test/test_libs.sh test_persistence: <<: *defaults @@ -205,14 +205,14 @@ workflows: - test_cover: requires: - setup_dependencies - - test_libs: - filters: - branches: - only: - - develop - - master - requires: - - setup_dependencies + #- test_libs: + #- filters: + #- branches: + #- only: + #- - develop + #- - master + #- requires: + #- - setup_dependencies - test_persistence: requires: - setup_abci From 5243e54641dc2437a6aac0f379d4f234948bcbf8 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Mon, 2 Apr 2018 11:23:18 +0200 Subject: [PATCH 021/143] [codecov] ignore docs, scripts and DOCKER dirs --- codecov.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/codecov.yml b/codecov.yml index 6db0f15d5..b190853de 100644 --- a/codecov.yml +++ b/codecov.yml @@ -16,3 +16,8 @@ comment: require_changes: no require_base: no require_head: yes + +ignore: + - "docs" + - "DOCKER" + - "scripts" From 9c757108ca8bc849a04c23e6800e53155863e933 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Mon, 2 Apr 2018 11:29:03 +0200 Subject: [PATCH 022/143] [test] remove test_libs Reasons: 1) all deps we're using should be passing tests (including external) 2) deps can require complicated setup for testing 3) the person responsible for releasing Tendermint should be cautious when updating a dep --- .circleci/config.yml | 21 --------------------- Makefile | 7 +------ scripts/dep_utils/checkout.sh | 35 ----------------------------------- test/README.md | 4 ---- test/test_libs.sh | 34 ---------------------------------- 5 files changed, 1 insertion(+), 100 deletions(-) delete mode 100644 scripts/dep_utils/checkout.sh delete mode 100644 test/test_libs.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 11ad5f33f..8dac95a8e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -130,19 +130,6 @@ jobs: paths: - "profiles/*" - #test_libs: - # <<: *defaults - # steps: - # - attach_workspace: - # at: /tmp/workspace - # - restore_cache: - # key: v1-pkg-cache - # - restore_cache: - # key: v1-tree-{{ .Environment.CIRCLE_SHA1 }} - # - run: - # name: Run tests - # command: bash test/test_libs.sh - test_persistence: <<: *defaults steps: @@ -205,14 +192,6 @@ workflows: - test_cover: requires: - setup_dependencies - #- test_libs: - #- filters: - #- branches: - #- only: - #- - develop - #- - master - #- requires: - #- - setup_dependencies - test_persistence: requires: - setup_abci diff --git a/Makefile b/Makefile index c19ce11a6..3e51c834e 100644 --- a/Makefile +++ b/Makefile @@ -119,11 +119,6 @@ test_integrations: make test_persistence make test_p2p -test_libs: - # checkout every github.com/tendermint dir and run its tests - # NOTE: on release-* or master branches only (set by Jenkins) - docker run --name run_libs -t tester bash test/test_libs.sh - test_release: @go test -tags release $(PACKAGES) @@ -186,4 +181,4 @@ metalinter_all: # To avoid unintended conflicts with file names, always add to .PHONY # unless there is a reason not to. # https://www.gnu.org/software/make/manual/html_node/Phony-Targets.html -.PHONY: check build build_race dist install check_tools get_tools update_tools get_vendor_deps draw_deps test_cover test_apps test_persistence test_p2p test test_race test_libs test_integrations test_release test100 vagrant_test fmt +.PHONY: check build build_race dist install check_tools get_tools update_tools get_vendor_deps draw_deps test_cover test_apps test_persistence test_p2p test test_race test_integrations test_release test100 vagrant_test fmt diff --git a/scripts/dep_utils/checkout.sh b/scripts/dep_utils/checkout.sh deleted file mode 100644 index 1d35e97ab..000000000 --- a/scripts/dep_utils/checkout.sh +++ /dev/null @@ -1,35 +0,0 @@ -#! /bin/bash - -set -ex - -set +u -if [[ "$DEP" == "" ]]; then - DEP=$GOPATH/src/github.com/tendermint/tendermint/Gopkg.lock -fi -set -u - - -set -u - -function getVendoredVersion() { - grep -A100 "$LIB" "$DEP" | grep revision | head -n1 | grep -o '"[^"]\+"' | cut -d '"' -f 2 -} - - -# fetch and checkout vendored dep - -lib=$1 - -echo "----------------------------------" -echo "Getting $lib ..." -go get -t "github.com/tendermint/$lib/..." - -VENDORED=$(getVendoredVersion "$lib") -cd "$GOPATH/src/github.com/tendermint/$lib" || exit -MASTER=$(git rev-parse origin/master) - -if [[ "$VENDORED" != "$MASTER" ]]; then - echo "... VENDORED != MASTER ($VENDORED != $MASTER)" - echo "... Checking out commit $VENDORED" - git checkout "$VENDORED" &> /dev/null -fi diff --git a/test/README.md b/test/README.md index d9f09fef7..ee53905bf 100644 --- a/test/README.md +++ b/test/README.md @@ -19,7 +19,3 @@ and run the following tests in docker containers: - send a tx on each node and ensure the state root is updated on all of them - crash and restart nodes one at a time and ensure they can sync back up (via fastsync) - crash and restart all nodes at once and ensure they can sync back up - -If on a `release-x.x.x` branch, we also run - -- `go test` for all our dependency libs (test/test_libs.sh) diff --git a/test/test_libs.sh b/test/test_libs.sh deleted file mode 100644 index 893bf35dc..000000000 --- a/test/test_libs.sh +++ /dev/null @@ -1,34 +0,0 @@ -#! /bin/bash -set -ex - -export PATH="$GOBIN:$PATH" - -# Get the parent directory of where this script is. -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done -DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )" - -#################### -# libs we depend on -#################### - -# All libs should define `make test` and `make get_vendor_deps` -LIBS=(tmlibs go-wire go-crypto abci) -for lib in "${LIBS[@]}"; do - # checkout vendored version of lib - bash scripts/dep_utils/checkout.sh "$lib" - - echo "Testing $lib ..." - cd "$GOPATH/src/github.com/tendermint/$lib" - make get_tools - make get_vendor_deps - make test - if [[ "$?" != 0 ]]; then - echo "FAIL" - exit 1 - fi - cd "$DIR" -done - -echo "" -echo "PASS" From ecdc1b9bb07329b31ff999dce5424c3c406caeb0 Mon Sep 17 00:00:00 2001 From: Zaki Manian Date: Mon, 2 Apr 2018 02:36:09 -0700 Subject: [PATCH 023/143] Add a method for creating an https server (#1403) --- rpc/lib/server/http_server.go | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/rpc/lib/server/http_server.go b/rpc/lib/server/http_server.go index 515baf5dd..ce7673644 100644 --- a/rpc/lib/server/http_server.go +++ b/rpc/lib/server/http_server.go @@ -48,6 +48,39 @@ func StartHTTPServer(listenAddr string, handler http.Handler, logger log.Logger) return listener, nil } +func StartHTTPAndTLSServer(listenAddr string, handler http.Handler, cert_path string, key_path string, logger log.Logger) (listener net.Listener, err error) { + // listenAddr should be fully formed including tcp:// or unix:// prefix + var proto, addr string + parts := strings.SplitN(listenAddr, "://", 2) + if len(parts) != 2 { + logger.Error("WARNING (tendermint/rpc/lib): Please use fully formed listening addresses, including the tcp:// or unix:// prefix") + // we used to allow addrs without tcp/unix prefix by checking for a colon + // TODO: Deprecate + proto = types.SocketType(listenAddr) + addr = listenAddr + // return nil, errors.Errorf("Invalid listener address %s", lisenAddr) + } else { + proto, addr = parts[0], parts[1] + } + + logger.Info(fmt.Sprintf("Starting RPC HTTPS server on %s socket %v", proto, addr)) + listener, err = net.Listen(proto, addr) + if err != nil { + return nil, errors.Errorf("Failed to listen to %v: %v", listenAddr, err) + } + + go func() { + res := http.ServeTLS( + listener, + RecoverAndLogHandler(handler, logger), + cert_path, + key_path, + ) + logger.Error("RPC HTTPS server stopped", "result", res) + }() + return listener, nil +} + func WriteRPCResponseHTTPError(w http.ResponseWriter, httpCode int, res types.RPCResponse) { jsonBytes, err := json.MarshalIndent(res, "", " ") if err != nil { From d694d47d2245c10ef1788abacb2a206b7a32157d Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Mon, 2 Apr 2018 11:40:47 +0200 Subject: [PATCH 024/143] [rpc/lib] rename vars according to Go conventions --- rpc/lib/server/http_server.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rpc/lib/server/http_server.go b/rpc/lib/server/http_server.go index ce7673644..df4ab8a10 100644 --- a/rpc/lib/server/http_server.go +++ b/rpc/lib/server/http_server.go @@ -48,7 +48,7 @@ func StartHTTPServer(listenAddr string, handler http.Handler, logger log.Logger) return listener, nil } -func StartHTTPAndTLSServer(listenAddr string, handler http.Handler, cert_path string, key_path string, logger log.Logger) (listener net.Listener, err error) { +func StartHTTPAndTLSServer(listenAddr string, handler http.Handler, certFile, keyFile string, logger log.Logger) (listener net.Listener, err error) { // listenAddr should be fully formed including tcp:// or unix:// prefix var proto, addr string parts := strings.SplitN(listenAddr, "://", 2) @@ -73,8 +73,8 @@ func StartHTTPAndTLSServer(listenAddr string, handler http.Handler, cert_path st res := http.ServeTLS( listener, RecoverAndLogHandler(handler, logger), - cert_path, - key_path, + certFile, + keyFile, ) logger.Error("RPC HTTPS server stopped", "result", res) }() From 5ef8a6e887580b60a34217816fb165e37fd7857b Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Mon, 2 Apr 2018 11:47:04 +0200 Subject: [PATCH 025/143] deprecate not fully formed addresses --- rpc/lib/server/http_server.go | 34 ++++++++++------------------------ 1 file changed, 10 insertions(+), 24 deletions(-) diff --git a/rpc/lib/server/http_server.go b/rpc/lib/server/http_server.go index df4ab8a10..01ad0809d 100644 --- a/rpc/lib/server/http_server.go +++ b/rpc/lib/server/http_server.go @@ -18,65 +18,51 @@ import ( ) func StartHTTPServer(listenAddr string, handler http.Handler, logger log.Logger) (listener net.Listener, err error) { - // listenAddr should be fully formed including tcp:// or unix:// prefix var proto, addr string parts := strings.SplitN(listenAddr, "://", 2) if len(parts) != 2 { - logger.Error("WARNING (tendermint/rpc/lib): Please use fully formed listening addresses, including the tcp:// or unix:// prefix") - // we used to allow addrs without tcp/unix prefix by checking for a colon - // TODO: Deprecate - proto = types.SocketType(listenAddr) - addr = listenAddr - // return nil, errors.Errorf("Invalid listener address %s", lisenAddr) - } else { - proto, addr = parts[0], parts[1] + return nil, errors.Errorf("Invalid listening address %s (use fully formed addresses, including the tcp:// or unix:// prefix)", listenAddr) } + proto, addr = parts[0], parts[1] logger.Info(fmt.Sprintf("Starting RPC HTTP server on %s socket %v", proto, addr)) listener, err = net.Listen(proto, addr) if err != nil { - return nil, errors.Errorf("Failed to listen to %v: %v", listenAddr, err) + return nil, errors.Errorf("Failed to listen on %v: %v", listenAddr, err) } go func() { - res := http.Serve( + err := http.Serve( listener, RecoverAndLogHandler(handler, logger), ) - logger.Error("RPC HTTP server stopped", "result", res) + logger.Error("RPC HTTP server stopped", "err", err) }() return listener, nil } func StartHTTPAndTLSServer(listenAddr string, handler http.Handler, certFile, keyFile string, logger log.Logger) (listener net.Listener, err error) { - // listenAddr should be fully formed including tcp:// or unix:// prefix var proto, addr string parts := strings.SplitN(listenAddr, "://", 2) if len(parts) != 2 { - logger.Error("WARNING (tendermint/rpc/lib): Please use fully formed listening addresses, including the tcp:// or unix:// prefix") - // we used to allow addrs without tcp/unix prefix by checking for a colon - // TODO: Deprecate - proto = types.SocketType(listenAddr) - addr = listenAddr - // return nil, errors.Errorf("Invalid listener address %s", lisenAddr) - } else { - proto, addr = parts[0], parts[1] + return nil, errors.Errorf("Invalid listening address %s (use fully formed addresses, including the tcp:// or unix:// prefix)", listenAddr) } + proto, addr = parts[0], parts[1] logger.Info(fmt.Sprintf("Starting RPC HTTPS server on %s socket %v", proto, addr)) listener, err = net.Listen(proto, addr) if err != nil { - return nil, errors.Errorf("Failed to listen to %v: %v", listenAddr, err) + return nil, errors.Errorf("Failed to listen on %v: %v", listenAddr, err) } go func() { - res := http.ServeTLS( + err := http.ServeTLS( listener, RecoverAndLogHandler(handler, logger), certFile, keyFile, ) - logger.Error("RPC HTTPS server stopped", "result", res) + logger.Error("RPC HTTPS server stopped", "err", err) }() return listener, nil } From 491c8ab4c1e867c160b2a550356cfb18f9e2f047 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Mon, 2 Apr 2018 11:55:55 +0200 Subject: [PATCH 026/143] [rpc/lib] log cert and key files in StartHTTPAndTLSServer --- rpc/lib/server/http_server.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rpc/lib/server/http_server.go b/rpc/lib/server/http_server.go index 01ad0809d..3f54c61ef 100644 --- a/rpc/lib/server/http_server.go +++ b/rpc/lib/server/http_server.go @@ -25,7 +25,7 @@ func StartHTTPServer(listenAddr string, handler http.Handler, logger log.Logger) } proto, addr = parts[0], parts[1] - logger.Info(fmt.Sprintf("Starting RPC HTTP server on %s socket %v", proto, addr)) + logger.Info(fmt.Sprintf("Starting RPC HTTP server on %s", listenAddr)) listener, err = net.Listen(proto, addr) if err != nil { return nil, errors.Errorf("Failed to listen on %v: %v", listenAddr, err) @@ -49,7 +49,7 @@ func StartHTTPAndTLSServer(listenAddr string, handler http.Handler, certFile, ke } proto, addr = parts[0], parts[1] - logger.Info(fmt.Sprintf("Starting RPC HTTPS server on %s socket %v", proto, addr)) + logger.Info(fmt.Sprintf("Starting RPC HTTPS server on %s (cert: %q, key: %q)", listenAddr, certFile, keyFile)) listener, err = net.Listen(proto, addr) if err != nil { return nil, errors.Errorf("Failed to listen on %v: %v", listenAddr, err) From 641476d40f6e8a51013b10809ff3fbc3edf0ec90 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Mon, 2 Apr 2018 16:55:43 +0200 Subject: [PATCH 027/143] update docker to use alpine 3.7 --- DOCKER/Dockerfile | 2 +- DOCKER/Dockerfile.develop | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/DOCKER/Dockerfile b/DOCKER/Dockerfile index c00318fba..9934ff722 100644 --- a/DOCKER/Dockerfile +++ b/DOCKER/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.6 +FROM alpine:3.7 # This is the release of tendermint to pull in. ENV TM_VERSION 0.15.0 diff --git a/DOCKER/Dockerfile.develop b/DOCKER/Dockerfile.develop index e6ee6607e..4f8ae0b2e 100644 --- a/DOCKER/Dockerfile.develop +++ b/DOCKER/Dockerfile.develop @@ -1,4 +1,4 @@ -FROM alpine:3.6 +FROM alpine:3.7 ENV DATA_ROOT /tendermint ENV TMHOME $DATA_ROOT From 208ac32fa266657bd6c304e84ec828aa252bb0b8 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Mon, 2 Apr 2018 16:56:07 +0200 Subject: [PATCH 028/143] update Dockerfile to point to 0.17.1 release --- DOCKER/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/DOCKER/Dockerfile b/DOCKER/Dockerfile index 9934ff722..cf45ac2f1 100644 --- a/DOCKER/Dockerfile +++ b/DOCKER/Dockerfile @@ -1,8 +1,8 @@ FROM alpine:3.7 # This is the release of tendermint to pull in. -ENV TM_VERSION 0.15.0 -ENV TM_SHA256SUM 71cc271c67eca506ca492c8b90b090132f104bf5dbfe0af2702a50886e88de17 +ENV TM_VERSION 0.17.1 +ENV TM_SHA256SUM d57008c63d2d9176861137e38ed203da486febf20ae7d388fb810a75afff8f24 # Tendermint will be looking for genesis file in /tendermint (unless you change # `genesis_file` in config.toml). You can put your config.toml and private @@ -26,7 +26,7 @@ RUN mkdir -p $DATA_ROOT && \ RUN apk add --no-cache bash curl jq RUN apk add --no-cache openssl && \ - wget https://s3-us-west-2.amazonaws.com/tendermint/binaries/tendermint/v${TM_VERSION}/tendermint_${TM_VERSION}_linux_amd64.zip && \ + wget https://github.com/tendermint/tendermint/releases/download/v${TM_VERSION}/tendermint_${TM_VERSION}_linux_amd64.zip && \ echo "${TM_SHA256SUM} tendermint_${TM_VERSION}_linux_amd64.zip" | sha256sum -c && \ unzip -d /bin tendermint_${TM_VERSION}_linux_amd64.zip && \ apk del openssl && \ From b800b4ec1d389c9891ce65f2c0633a1fad6f2af1 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Mon, 2 Apr 2018 16:57:25 +0200 Subject: [PATCH 029/143] update docker readme --- DOCKER/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/DOCKER/README.md b/DOCKER/README.md index 24cd38a51..2fe3db866 100644 --- a/DOCKER/README.md +++ b/DOCKER/README.md @@ -1,6 +1,7 @@ # Supported tags and respective `Dockerfile` links -- `0.15.0`, `latest` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/170777300ea92dc21a8aec1abc16cb51812513a4/DOCKER/Dockerfile) +- `0.17.1`, `latest` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/208ac32fa266657bd6c304e84ec828aa252bb0b8/DOCKER/Dockerfile) +- `0.15.0` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/170777300ea92dc21a8aec1abc16cb51812513a4/DOCKER/Dockerfile) - `0.13.0` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/a28b3fff49dce2fb31f90abb2fc693834e0029c2/DOCKER/Dockerfile) - `0.12.1` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/457c688346b565e90735431619ca3ca597ef9007/DOCKER/Dockerfile) - `0.12.0` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/70d8afa6e952e24c573ece345560a5971bf2cc0e/DOCKER/Dockerfile) From 315c475b791c9c48a35f52f79fef6afc2208138d Mon Sep 17 00:00:00 2001 From: Zach Ramsay Date: Tue, 3 Apr 2018 04:48:40 -0700 Subject: [PATCH 030/143] docs: build updates ref: https://github.com/tendermint/tools/pull/79 --- .gitignore | 1 + docs/conf.py | 5 ++--- docs/index.rst | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 7b7b746fb..e76fb1fc5 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ test/logs coverage.txt docs/_build docs/tools +docs/abci-spec.rst *.log scripts/wal2json/wal2json diff --git a/docs/conf.py b/docs/conf.py index 6122e90a3..9d4f6b017 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -196,9 +196,8 @@ urllib.urlretrieve(tools_repo+tools_branch+'/mintnet-kubernetes/assets/statefuls urllib.urlretrieve(tools_repo+tools_branch+'/mintnet-kubernetes/assets/t_plus_k.png', filename=assets_dir+'/t_plus_k.png') urllib.urlretrieve(tools_repo+tools_branch+'/terraform-digitalocean/README.rst', filename=tools_dir+'/terraform-digitalocean.rst') -urllib.urlretrieve(tools_repo+tools_branch+'/tm-bench/README.rst', filename=tools_dir+'/benchmarking-and-monitoring.rst') -# the readme for below is included in tm-bench -# urllib.urlretrieve('https://raw.githubusercontent.com/tendermint/tools/master/tm-monitor/README.rst', filename='tools/tm-monitor.rst') +urllib.urlretrieve(tools_repo+tools_branch+'/tm-bench/README.rst', filename=tools_dir+'/benchmarking.rst') +urllib.urlretrieve('https://raw.githubusercontent.com/tendermint/tools/master/tm-monitor/README.rst', filename='tools/monitoring.rst') #### abci spec ################################# diff --git a/docs/index.rst b/docs/index.rst index f2f70d194..a89adb296 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -44,7 +44,8 @@ Tendermint Tools tools/docker.rst tools/mintnet-kubernetes.rst tools/terraform-digitalocean.rst - tools/benchmarking-and-monitoring.rst + tools/benchmarking.rst + tools/monitoring.rst Tendermint 102 -------------- From 89cdde7f1e13781a896c31c4d1b5c7a3feac771a Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Tue, 3 Apr 2018 06:50:53 -0700 Subject: [PATCH 031/143] Fix state tests --- config/toml.go | 12 +++++----- state/execution.go | 2 +- state/execution_test.go | 4 ++-- state/state.go | 13 ++++------- state/state_test.go | 33 ++++++++++++-------------- state/store.go | 46 +++++++++++++++---------------------- state/txindex/kv/kv.go | 8 +++---- state/txindex/kv/kv_test.go | 11 +++++---- state/txindex/kv/wire.go | 10 ++++++++ state/wire.go | 12 ++++++++++ types/block.go | 16 +++++++------ types/params.go | 6 ++--- 12 files changed, 90 insertions(+), 83 deletions(-) create mode 100644 state/txindex/kv/wire.go create mode 100644 state/wire.go diff --git a/config/toml.go b/config/toml.go index af5be2345..882d2289a 100644 --- a/config/toml.go +++ b/config/toml.go @@ -276,8 +276,8 @@ var testGenesis = `{ "validators": [ { "pub_key": { - "type": "ed25519", - "data":"3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8" + "type": "AC26791624DE60", + "value":"AT/+aaL1eB0477Mud9JMm8Sh8BIvOYlPGC9KkIUmFaE=" }, "power": 10, "name": "" @@ -289,12 +289,12 @@ var testGenesis = `{ var testPrivValidator = `{ "address": "D028C9981F7A87F3093672BF0D5B0E2A1B3ED456", "pub_key": { - "type": "ed25519", - "data": "3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8" + "type": "AC26791624DE60", + "value": "AT/+aaL1eB0477Mud9JMm8Sh8BIvOYlPGC9KkIUmFaE=" }, "priv_key": { - "type": "ed25519", - "data": "27F82582AEFAE7AB151CFB01C48BB6C1A0DA78F9BDDA979A9F70A84D074EB07D3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8" + "type": "954568A3288910", + "value": "EVkqJO/jIXp3rkASXfh9YnyToYXRXhBr6g9cQVxPFnQBP/5povV4HTjvsy530kybxKHwEi85iU8YL0qQhSYVoQ==" }, "last_height": 0, "last_round": 0, diff --git a/state/execution.go b/state/execution.go index 64db9f31a..0ce5e44f1 100644 --- a/state/execution.go +++ b/state/execution.go @@ -245,7 +245,7 @@ func execBlockOnProxyApp(logger log.Logger, proxyAppConn proxy.AppConnConsensus, // ./lite/doc.go for details on how a light client tracks validators. func updateValidators(currentSet *types.ValidatorSet, updates []abci.Validator) error { for _, v := range updates { - pubkey, err := crypto.PubKeyFromBytes(v.PubKey) // NOTE: expects go-wire encoded pubkey + pubkey, err := crypto.PubKeyFromBytes(v.PubKey) // NOTE: expects go-amino encoded pubkey if err != nil { return err } diff --git a/state/execution_test.go b/state/execution_test.go index e508c0fa8..09c40b5a0 100644 --- a/state/execution_test.go +++ b/state/execution_test.go @@ -66,8 +66,8 @@ func TestBeginBlockAbsentValidators(t *testing.T) { lastCommitPrecommits []*types.Vote expectedAbsentValidators []int32 }{ - {"none absent", []*types.Vote{{ValidatorIndex: 0, Timestamp: now}, {ValidatorIndex: 1, Timestamp: now}}, []int32{}}, - {"one absent", []*types.Vote{{ValidatorIndex: 0, Timestamp: now}, nil}, []int32{1}}, + {"none absent", []*types.Vote{{ValidatorIndex: 0, Timestamp: now, Type: types.VoteTypePrecommit}, {ValidatorIndex: 1, Timestamp: now}}, []int32{}}, + {"one absent", []*types.Vote{{ValidatorIndex: 0, Timestamp: now, Type: types.VoteTypePrecommit}, nil}, []int32{1}}, {"multiple absent", []*types.Vote{nil, nil}, []int32{0, 1}}, } diff --git a/state/state.go b/state/state.go index fb5d78c47..aa6e04b66 100644 --- a/state/state.go +++ b/state/state.go @@ -6,8 +6,6 @@ import ( "io/ioutil" "time" - wire "github.com/tendermint/go-wire" - "github.com/tendermint/tendermint/types" ) @@ -81,16 +79,13 @@ func (s State) Copy() State { // Equals returns true if the States are identical. func (s State) Equals(s2 State) bool { - return bytes.Equal(s.Bytes(), s2.Bytes()) + sbz, s2bz := s.Bytes(), s2.Bytes() + return bytes.Equal(sbz, s2bz) } -// Bytes serializes the State using go-wire. +// Bytes serializes the State using go-amino. func (s State) Bytes() []byte { - bz, err := wire.MarshalBinary(s) - if err != nil { - panic(err) - } - return bz + return cdc.MustMarshalBinaryBare(s) } // IsEmpty returns true if the State is equal to the empty State. diff --git a/state/state_test.go b/state/state_test.go index 02c94253e..27a605249 100644 --- a/state/state_test.go +++ b/state/state_test.go @@ -7,11 +7,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - abci "github.com/tendermint/abci/types" - - crypto "github.com/tendermint/go-crypto" - + "github.com/tendermint/go-crypto" cmn "github.com/tendermint/tmlibs/common" dbm "github.com/tendermint/tmlibs/db" @@ -42,7 +39,7 @@ func TestStateCopy(t *testing.T) { stateCopy := state.Copy() assert.True(state.Equals(stateCopy), - cmn.Fmt(`expected state and its copy to be identical. got %v\n expected %v\n`, + cmn.Fmt("expected state and its copy to be identical.\ngot: %v\nexpected: %v\n", stateCopy, state)) stateCopy.LastBlockHeight++ @@ -62,7 +59,7 @@ func TestStateSaveLoad(t *testing.T) { loadedState := LoadState(stateDB) assert.True(state.Equals(loadedState), - cmn.Fmt(`expected state and its copy to be identical. got %v\n expected %v\n`, + cmn.Fmt("expected state and its copy to be identical.\ngot: %v\nexpected: %v\n", loadedState, state)) } @@ -78,8 +75,8 @@ func TestABCIResponsesSaveLoad1(t *testing.T) { // build mock responses block := makeBlock(state, 2) abciResponses := NewABCIResponses(block) - abciResponses.DeliverTx[0] = &abci.ResponseDeliverTx{Data: []byte("foo"), Tags: []cmn.KVPair{}} - abciResponses.DeliverTx[1] = &abci.ResponseDeliverTx{Data: []byte("bar"), Log: "ok", Tags: []cmn.KVPair{}} + abciResponses.DeliverTx[0] = &abci.ResponseDeliverTx{Data: []byte("foo"), Tags: nil} + abciResponses.DeliverTx[1] = &abci.ResponseDeliverTx{Data: []byte("bar"), Log: "ok", Tags: nil} abciResponses.EndBlock = &abci.ResponseEndBlock{ValidatorUpdates: []abci.Validator{ { PubKey: crypto.GenPrivKeyEd25519().PubKey().Bytes(), @@ -88,11 +85,11 @@ func TestABCIResponsesSaveLoad1(t *testing.T) { }} saveABCIResponses(stateDB, block.Height, abciResponses) - loadedAbciResponses, err := LoadABCIResponses(stateDB, block.Height) + loadedABCIResponses, err := LoadABCIResponses(stateDB, block.Height) assert.Nil(err) - assert.Equal(abciResponses, loadedAbciResponses, - cmn.Fmt(`ABCIResponses don't match: Got %v, Expected %v`, loadedAbciResponses, - abciResponses)) + assert.Equal(abciResponses, loadedABCIResponses, + cmn.Fmt("ABCIResponses don't match:\ngot: %v\nexpected: %v\n", + loadedABCIResponses, abciResponses)) } // TestResultsSaveLoad tests saving and loading abci results. @@ -109,8 +106,8 @@ func TestABCIResponsesSaveLoad2(t *testing.T) { expected types.ABCIResults }{ 0: { - []*abci.ResponseDeliverTx{}, - types.ABCIResults{}, + nil, + nil, }, 1: { []*abci.ResponseDeliverTx{ @@ -129,12 +126,12 @@ func TestABCIResponsesSaveLoad2(t *testing.T) { }}, }, types.ABCIResults{ - {383, []byte{}}, + {383, nil}, {0, []byte("Gotcha!")}, }}, 3: { nil, - types.ABCIResults{}, + nil, }, } @@ -430,7 +427,7 @@ func makeHeaderPartsResponsesValPubKeyChange(state State, height int64, block := makeBlock(state, height) abciResponses := &ABCIResponses{ - EndBlock: &abci.ResponseEndBlock{ValidatorUpdates: []abci.Validator{}}, + EndBlock: &abci.ResponseEndBlock{ValidatorUpdates: nil}, } // if the pubkey is new, remove the old and add the new @@ -452,7 +449,7 @@ func makeHeaderPartsResponsesValPowerChange(state State, height int64, block := makeBlock(state, height) abciResponses := &ABCIResponses{ - EndBlock: &abci.ResponseEndBlock{ValidatorUpdates: []abci.Validator{}}, + EndBlock: &abci.ResponseEndBlock{ValidatorUpdates: nil}, } // if the pubkey is new, remove the old and add the new diff --git a/state/store.go b/state/store.go index df07ec540..ee0619d34 100644 --- a/state/store.go +++ b/state/store.go @@ -4,7 +4,6 @@ import ( "fmt" abci "github.com/tendermint/abci/types" - wire "github.com/tendermint/go-wire" "github.com/tendermint/tendermint/types" cmn "github.com/tendermint/tmlibs/common" dbm "github.com/tendermint/tmlibs/db" @@ -69,7 +68,7 @@ func loadState(db dbm.DB, key []byte) (state State) { return state } - err := wire.UnmarshalBinary(buf, &state) + err := cdc.UnmarshalBinaryBare(buf, &state) if err != nil { // DATA HAS BEEN CORRUPTED OR THE SPEC HAS CHANGED cmn.Exit(cmn.Fmt(`LoadState: Data has been corrupted or its spec has changed: @@ -104,22 +103,23 @@ type ABCIResponses struct { // NewABCIResponses returns a new ABCIResponses func NewABCIResponses(block *types.Block) *ABCIResponses { + resDeliverTxs := make([]*abci.ResponseDeliverTx, block.NumTxs) + if block.NumTxs == 0 { + // This makes Amino encoding/decoding consistent. + resDeliverTxs = nil + } return &ABCIResponses{ - DeliverTx: make([]*abci.ResponseDeliverTx, block.NumTxs), + DeliverTx: resDeliverTxs, } } -// Bytes serializes the ABCIResponse using go-wire -func (a *ABCIResponses) Bytes() []byte { - bz, err := wire.MarshalBinary(*a) - if err != nil { - panic(err) - } - return bz +// Bytes serializes the ABCIResponse using go-amino. +func (arz *ABCIResponses) Bytes() []byte { + return cdc.MustMarshalBinaryBare(arz) } -func (a *ABCIResponses) ResultsHash() []byte { - results := types.NewResults(a.DeliverTx) +func (arz *ABCIResponses) ResultsHash() []byte { + results := types.NewResults(arz.DeliverTx) return results.Hash() } @@ -133,7 +133,7 @@ func LoadABCIResponses(db dbm.DB, height int64) (*ABCIResponses, error) { } abciResponses := new(ABCIResponses) - err := wire.UnmarshalBinary(buf, abciResponses) + err := cdc.UnmarshalBinaryBare(buf, abciResponses) if err != nil { // DATA HAS BEEN CORRUPTED OR THE SPEC HAS CHANGED cmn.Exit(cmn.Fmt(`LoadABCIResponses: Data has been corrupted or its spec has @@ -159,13 +159,9 @@ type ValidatorsInfo struct { LastHeightChanged int64 } -// Bytes serializes the ValidatorsInfo using go-wire +// Bytes serializes the ValidatorsInfo using go-amino. func (valInfo *ValidatorsInfo) Bytes() []byte { - bz, err := wire.MarshalBinary(*valInfo) - if err != nil { - panic(err) - } - return bz + return cdc.MustMarshalBinaryBare(valInfo) } // LoadValidators loads the ValidatorSet for a given height. @@ -194,7 +190,7 @@ func loadValidatorsInfo(db dbm.DB, height int64) *ValidatorsInfo { } v := new(ValidatorsInfo) - err := wire.UnmarshalBinary(buf, v) + err := cdc.UnmarshalBinaryBare(buf, v) if err != nil { // DATA HAS BEEN CORRUPTED OR THE SPEC HAS CHANGED cmn.Exit(cmn.Fmt(`LoadValidators: Data has been corrupted or its spec has changed: @@ -227,13 +223,9 @@ type ConsensusParamsInfo struct { LastHeightChanged int64 } -// Bytes serializes the ConsensusParamsInfo using go-wire +// Bytes serializes the ConsensusParamsInfo using go-amino. func (params ConsensusParamsInfo) Bytes() []byte { - bz, err := wire.MarshalBinary(params) - if err != nil { - panic(err) - } - return bz + return cdc.MustMarshalBinaryBare(params) } // LoadConsensusParams loads the ConsensusParams for a given height. @@ -263,7 +255,7 @@ func loadConsensusParamsInfo(db dbm.DB, height int64) *ConsensusParamsInfo { } paramsInfo := new(ConsensusParamsInfo) - err := wire.UnmarshalBinary(buf, paramsInfo) + err := cdc.UnmarshalBinaryBare(buf, paramsInfo) if err != nil { // DATA HAS BEEN CORRUPTED OR THE SPEC HAS CHANGED cmn.Exit(cmn.Fmt(`LoadConsensusParams: Data has been corrupted or its spec has changed: diff --git a/state/txindex/kv/kv.go b/state/txindex/kv/kv.go index 24e982725..9131b24fc 100644 --- a/state/txindex/kv/kv.go +++ b/state/txindex/kv/kv.go @@ -9,8 +9,6 @@ import ( "time" "github.com/pkg/errors" - - wire "github.com/tendermint/go-wire" cmn "github.com/tendermint/tmlibs/common" dbm "github.com/tendermint/tmlibs/db" "github.com/tendermint/tmlibs/pubsub/query" @@ -68,7 +66,7 @@ func (txi *TxIndex) Get(hash []byte) (*types.TxResult, error) { } txResult := new(types.TxResult) - err := wire.UnmarshalBinary(rawBytes, &txResult) + err := cdc.UnmarshalBinaryBare(rawBytes, &txResult) if err != nil { return nil, fmt.Errorf("Error reading TxResult: %v", err) } @@ -91,7 +89,7 @@ func (txi *TxIndex) AddBatch(b *txindex.Batch) error { } // index tx by hash - rawBytes, err := wire.MarshalBinary(result) + rawBytes, err := cdc.MarshalBinaryBare(result) if err != nil { return err } @@ -116,7 +114,7 @@ func (txi *TxIndex) Index(result *types.TxResult) error { } // index tx by hash - rawBytes, err := wire.MarshalBinary(result) + rawBytes, err := cdc.MarshalBinaryBare(result) if err != nil { return err } diff --git a/state/txindex/kv/kv_test.go b/state/txindex/kv/kv_test.go index ba2830e81..c14b59aed 100644 --- a/state/txindex/kv/kv_test.go +++ b/state/txindex/kv/kv_test.go @@ -9,18 +9,19 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" abci "github.com/tendermint/abci/types" - "github.com/tendermint/tendermint/state/txindex" - "github.com/tendermint/tendermint/types" cmn "github.com/tendermint/tmlibs/common" db "github.com/tendermint/tmlibs/db" "github.com/tendermint/tmlibs/pubsub/query" + + "github.com/tendermint/tendermint/state/txindex" + "github.com/tendermint/tendermint/types" ) func TestTxIndex(t *testing.T) { indexer := NewTxIndex(db.NewMemDB()) tx := types.Tx("HELLO WORLD") - txResult := &types.TxResult{1, 0, tx, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeTypeOK, Log: "", Tags: []cmn.KVPair{}}} + txResult := &types.TxResult{1, 0, tx, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeTypeOK, Log: "", Tags: nil}} hash := tx.Hash() batch := txindex.NewBatch(1) @@ -35,7 +36,7 @@ func TestTxIndex(t *testing.T) { assert.Equal(t, txResult, loadedTxResult) tx2 := types.Tx("BYE BYE WORLD") - txResult2 := &types.TxResult{1, 0, tx2, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeTypeOK, Log: "", Tags: []cmn.KVPair{}}} + txResult2 := &types.TxResult{1, 0, tx2, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeTypeOK, Log: "", Tags: nil}} hash2 := tx2.Hash() err = indexer.Index(txResult2) @@ -151,7 +152,7 @@ func txResultWithTags(tags []cmn.KVPair) *types.TxResult { func benchmarkTxIndex(txsCount int, b *testing.B) { tx := types.Tx("HELLO WORLD") - txResult := &types.TxResult{1, 0, tx, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeTypeOK, Log: "", Tags: []cmn.KVPair{}}} + txResult := &types.TxResult{1, 0, tx, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeTypeOK, Log: "", Tags: nil}} dir, err := ioutil.TempDir("", "tx_index_db") if err != nil { diff --git a/state/txindex/kv/wire.go b/state/txindex/kv/wire.go new file mode 100644 index 000000000..ccca75254 --- /dev/null +++ b/state/txindex/kv/wire.go @@ -0,0 +1,10 @@ +package kv + +import ( + "github.com/tendermint/go-amino" +) + +var cdc = amino.NewCodec() + +func init() { +} diff --git a/state/wire.go b/state/wire.go new file mode 100644 index 000000000..3e8b544d9 --- /dev/null +++ b/state/wire.go @@ -0,0 +1,12 @@ +package state + +import ( + "github.com/tendermint/go-amino" + "github.com/tendermint/go-crypto" +) + +var cdc = amino.NewCodec() + +func init() { + crypto.RegisterAmino(cdc) +} diff --git a/types/block.go b/types/block.go index 9d8bb42ad..44e616e68 100644 --- a/types/block.go +++ b/types/block.go @@ -518,13 +518,15 @@ type hasher struct { func (h hasher) Hash() []byte { hasher := ripemd160.New() - bz, err := cdc.MarshalBinaryBare(h.item) - if err != nil { - panic(err) - } - _, err = hasher.Write(bz) - if err != nil { - panic(err) + if h.item != nil && !cmn.IsTypedNil(h.item) { + bz, err := cdc.MarshalBinaryBare(h.item) + if err != nil { + panic(err) + } + _, err = hasher.Write(bz) + if err != nil { + panic(err) + } } return hasher.Sum(nil) diff --git a/types/params.go b/types/params.go index 0626663e5..2df092d62 100644 --- a/types/params.go +++ b/types/params.go @@ -7,7 +7,7 @@ import ( ) const ( - maxBlockSizeBytes = 104857600 // 100MB + MaxBlockSizeBytes = 104857600 // 100MB ) // ConsensusParams contains consensus critical parameters @@ -95,9 +95,9 @@ func (params *ConsensusParams) Validate() error { } // ensure blocks aren't too big - if params.BlockSize.MaxBytes > maxBlockSizeBytes { + if params.BlockSize.MaxBytes > MaxBlockSizeBytes { return cmn.NewError("BlockSize.MaxBytes is too big. %d > %d", - params.BlockSize.MaxBytes, maxBlockSizeBytes) + params.BlockSize.MaxBytes, MaxBlockSizeBytes) } return nil } From 47b8bd1728c0f88eff57249223a87ddcda5fbaf4 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Tue, 27 Mar 2018 18:18:24 +0200 Subject: [PATCH 032/143] wrote a test for EventBus#PublishEventTx Refs #1369 --- types/event_bus_test.go | 49 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/types/event_bus_test.go b/types/event_bus_test.go index aa97092f6..c1b4c1b0f 100644 --- a/types/event_bus_test.go +++ b/types/event_bus_test.go @@ -7,9 +7,58 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + abci "github.com/tendermint/abci/types" + cmn "github.com/tendermint/tmlibs/common" tmpubsub "github.com/tendermint/tmlibs/pubsub" + tmquery "github.com/tendermint/tmlibs/pubsub/query" ) +func TestEventBusPublishEventTx(t *testing.T) { + eventBus := NewEventBus() + err := eventBus.Start() + require.NoError(t, err) + defer eventBus.Stop() + + tx := Tx("foo") + result := abci.ResponseDeliverTx{Data: []byte("bar"), Tags: []cmn.KVPair{}, Fee: cmn.KI64Pair{Key: []uint8{}, Value: 0}} + + txEventsCh := make(chan interface{}) + + // PublishEventTx adds all these 3 tags, so the query below should work + query := fmt.Sprintf("tm.event='Tx' AND tx.height=1 AND tx.hash='%X'", tx.Hash()) + err = eventBus.Subscribe(context.Background(), "test", tmquery.MustParse(query), txEventsCh) + require.NoError(t, err) + + done := make(chan struct{}) + go func() { + for e := range txEventsCh { + edt := e.(TMEventData).Unwrap().(EventDataTx) + assert.Equal(t, int64(1), edt.Height) + assert.Equal(t, uint32(0), edt.Index) + assert.Equal(t, tx, edt.Tx) + assert.Equal(t, result, edt.Result) + close(done) + } + }() + + err = eventBus.PublishEventTx(EventDataTx{TxResult{ + Height: 1, + Index: 0, + Tx: tx, + Result: result, + }}) + assert.NoError(t, err) + + select { + case <-done: + case <-time.After(1 * time.Second): + t.Fatal("did not receive a transaction after 1 sec.") + } +} + func BenchmarkEventBus(b *testing.B) { benchmarks := []struct { name string From 8462493cbfefbc0222dc26be892eb39e79072691 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Tue, 27 Mar 2018 19:00:24 +0200 Subject: [PATCH 033/143] [rpc] fix subscribing using an abci.ResponseDeliverTx tag Refs #1369 --- CHANGELOG.md | 4 +++- types/event_bus.go | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 47b39b23a..e7c61489c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,13 +27,15 @@ BUG FIXES: ## 0.17.2 (TBD) +BUG FIXES: +- [rpc] fix subscribing using an abci.ResponseDeliverTx tag + IMPROVEMENTS: - [rpc] `/tx` and `/tx_search` responses now include the transaction hash ## 0.17.1 (March 27th, 2018) BUG FIXES: - - [types] Actually support `app_state` in genesis as `AppStateJSON` ## 0.17.0 (March 27th, 2018) diff --git a/types/event_bus.go b/types/event_bus.go index 4edaea588..37bd5619d 100644 --- a/types/event_bus.go +++ b/types/event_bus.go @@ -101,7 +101,7 @@ func (b *EventBus) PublishEventTx(event EventDataTx) error { b.Logger.Info("Got tag with an empty key (skipping)", "tag", tag, "tx", event.Tx) continue } - tags[string(tag.Key)] = tag.Value + tags[string(tag.Key)] = string(tag.Value) } // add predefined tags From 196f8410bacbbac38571fd448a535648c9d731f4 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Tue, 3 Apr 2018 07:03:08 -0700 Subject: [PATCH 034/143] WIP commit; Fix types/results_test --- blockchain/reactor.go | 101 ++++++++---------------- blockchain/reactor_test.go | 42 +++++----- blockchain/store.go | 93 +++++++--------------- blockchain/store_test.go | 50 +----------- blockchain/wire.go | 11 +++ consensus/common_test.go | 6 +- consensus/state.go | 10 +-- consensus/types/height_vote_set_test.go | 2 +- types/results_test.go | 4 +- 9 files changed, 111 insertions(+), 208 deletions(-) create mode 100644 blockchain/wire.go diff --git a/blockchain/reactor.go b/blockchain/reactor.go index 2b334c232..314e0b0e0 100644 --- a/blockchain/reactor.go +++ b/blockchain/reactor.go @@ -1,21 +1,16 @@ package blockchain import ( - "bytes" - "errors" "fmt" "reflect" - "sync" "time" - wire "github.com/tendermint/go-wire" - - cmn "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tmlibs/log" - + "github.com/tendermint/go-amino" "github.com/tendermint/tendermint/p2p" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" + cmn "github.com/tendermint/tmlibs/common" + "github.com/tendermint/tmlibs/log" ) const ( @@ -52,9 +47,6 @@ func (e peerError) Error() string { type BlockchainReactor struct { p2p.BaseReactor - mtx sync.Mutex - params types.ConsensusParams - // immutable initialState sm.State @@ -87,7 +79,6 @@ func NewBlockchainReactor(state sm.State, blockExec *sm.BlockExecutor, store *Bl ) bcR := &BlockchainReactor{ - params: state.ConsensusParams, initialState: state, blockExec: blockExec, store: store, @@ -140,8 +131,8 @@ func (bcR *BlockchainReactor) GetChannels() []*p2p.ChannelDescriptor { // AddPeer implements Reactor by sending our state to peer. func (bcR *BlockchainReactor) AddPeer(peer p2p.Peer) { - if !peer.Send(BlockchainChannel, - struct{ BlockchainMessage }{&bcStatusResponseMessage{bcR.store.Height()}}) { + msgBytes := cdc.MustMarshalBinaryBare(&bcStatusResponseMessage{bcR.store.Height()}) + if !peer.Send(BlockchainChannel, msgBytes) { // doing nothing, will try later in `poolRoutine` } // peer is added to the pool once we receive the first @@ -162,20 +153,19 @@ func (bcR *BlockchainReactor) respondToPeer(msg *bcBlockRequestMessage, block := bcR.store.LoadBlock(msg.Height) if block != nil { - msg := &bcBlockResponseMessage{Block: block} - return src.TrySend(BlockchainChannel, struct{ BlockchainMessage }{msg}) + msgBytes := cdc.MustMarshalBinaryBare(&bcBlockResponseMessage{Block: block}) + return src.TrySend(BlockchainChannel, msgBytes) } bcR.Logger.Info("Peer asking for a block we don't have", "src", src, "height", msg.Height) - return src.TrySend(BlockchainChannel, struct{ BlockchainMessage }{ - &bcNoBlockResponseMessage{Height: msg.Height}, - }) + msgBytes := cdc.MustMarshalBinaryBare(&bcNoBlockResponseMessage{Height: msg.Height}) + return src.TrySend(BlockchainChannel, msgBytes) } // Receive implements Reactor by handling 4 types of messages (look below). func (bcR *BlockchainReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) { - _, msg, err := DecodeMessage(msgBytes, bcR.maxMsgSize()) + msg, err := DecodeMessage(msgBytes) if err != nil { bcR.Logger.Error("Error decoding message", "src", src, "chId", chID, "msg", msg, "err", err, "bytes", msgBytes) bcR.Switch.StopPeerForError(src, err) @@ -194,8 +184,8 @@ func (bcR *BlockchainReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) bcR.pool.AddBlock(src.ID(), msg.Block, len(msgBytes)) case *bcStatusRequestMessage: // Send peer our state. - queued := src.TrySend(BlockchainChannel, - struct{ BlockchainMessage }{&bcStatusResponseMessage{bcR.store.Height()}}) + msgBytes := cdc.MustMarshalBinaryBare(&bcStatusResponseMessage{bcR.store.Height()}) + queued := src.TrySend(BlockchainChannel, msgBytes) if !queued { // sorry } @@ -207,21 +197,6 @@ func (bcR *BlockchainReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) } } -// maxMsgSize returns the maximum allowable size of a -// message on the blockchain reactor. -func (bcR *BlockchainReactor) maxMsgSize() int { - bcR.mtx.Lock() - defer bcR.mtx.Unlock() - return bcR.params.BlockSize.MaxBytes + 2 -} - -// updateConsensusParams updates the internal consensus params -func (bcR *BlockchainReactor) updateConsensusParams(params types.ConsensusParams) { - bcR.mtx.Lock() - defer bcR.mtx.Unlock() - bcR.params = params -} - // Handle messages from the poolReactor telling the reactor what to do. // NOTE: Don't sleep in the FOR_LOOP or otherwise slow it down! // (Except for the SYNC_LOOP, which is the primary purpose and must be synchronous.) @@ -247,8 +222,8 @@ FOR_LOOP: if peer == nil { continue FOR_LOOP // Peer has since been disconnected. } - msg := &bcBlockRequestMessage{request.Height} - queued := peer.TrySend(BlockchainChannel, struct{ BlockchainMessage }{msg}) + msgBytes := cdc.MustMarshalBinaryBare(&bcBlockRequestMessage{request.Height}) + queued := peer.TrySend(BlockchainChannel, msgBytes) if !queued { // We couldn't make the request, send-queue full. // The pool handles timeouts, just let it go. @@ -321,9 +296,6 @@ FOR_LOOP: } blocksSynced++ - // update the consensus params - bcR.updateConsensusParams(state.ConsensusParams) - if blocksSynced%100 == 0 { lastRate = 0.9*lastRate + 0.1*(100/time.Since(lastHundred).Seconds()) bcR.Logger.Info("Fast Sync Rate", "height", bcR.pool.height, @@ -341,43 +313,37 @@ FOR_LOOP: // BroadcastStatusRequest broadcasts `BlockStore` height. func (bcR *BlockchainReactor) BroadcastStatusRequest() error { - bcR.Switch.Broadcast(BlockchainChannel, - struct{ BlockchainMessage }{&bcStatusRequestMessage{bcR.store.Height()}}) + msgBytes := cdc.MustMarshalBinaryBare(&bcStatusRequestMessage{bcR.store.Height()}) + bcR.Switch.Broadcast(BlockchainChannel, msgBytes) return nil } //----------------------------------------------------------------------------- // Messages -const ( - msgTypeBlockRequest = byte(0x10) - msgTypeBlockResponse = byte(0x11) - msgTypeNoBlockResponse = byte(0x12) - msgTypeStatusResponse = byte(0x20) - msgTypeStatusRequest = byte(0x21) -) - // BlockchainMessage is a generic message for this reactor. type BlockchainMessage interface{} -var _ = wire.RegisterInterface( - struct{ BlockchainMessage }{}, - wire.ConcreteType{&bcBlockRequestMessage{}, msgTypeBlockRequest}, - wire.ConcreteType{&bcBlockResponseMessage{}, msgTypeBlockResponse}, - wire.ConcreteType{&bcNoBlockResponseMessage{}, msgTypeNoBlockResponse}, - wire.ConcreteType{&bcStatusResponseMessage{}, msgTypeStatusResponse}, - wire.ConcreteType{&bcStatusRequestMessage{}, msgTypeStatusRequest}, -) +func RegisterBlockchainMessages(cdc *amino.Codec) { + cdc.RegisterInterface((*BlockchainMessage)(nil), nil) + cdc.RegisterConcrete(&bcBlockRequestMessage{}, "tendermint/mempool/BlockRequest", nil) + cdc.RegisterConcrete(&bcBlockResponseMessage{}, "tendermint/mempool/BlockResponse", nil) + cdc.RegisterConcrete(&bcNoBlockResponseMessage{}, "tendermint/mempool/NoBlockResponse", nil) + cdc.RegisterConcrete(&bcStatusResponseMessage{}, "tendermint/mempool/StatusResponse", nil) + cdc.RegisterConcrete(&bcStatusRequestMessage{}, "tendermint/mempool/StatusRequest", nil) +} // DecodeMessage decodes BlockchainMessage. // TODO: ensure that bz is completely read. -func DecodeMessage(bz []byte, maxSize int) (msgType byte, msg BlockchainMessage, err error) { - msgType = bz[0] - n := int(0) - r := bytes.NewReader(bz) - msg = wire.ReadBinary(struct{ BlockchainMessage }{}, r, maxSize, &n, &err).(struct{ BlockchainMessage }).BlockchainMessage - if err != nil && n != len(bz) { - err = errors.New("DecodeMessage() had bytes left over") +func DecodeMessage(bz []byte) (msg BlockchainMessage, err error) { + const ( + prefixSize = 4 + fieldKeySize = 1 + maxMessageSize = types.MaxBlockSizeBytes + prefixSize + fieldKeySize + ) + err = cdc.UnmarshalBinaryBare(bz, &msg) + if err != nil { + err = cmn.ErrorWrap(err, "DecodeMessage() had bytes left over") } return } @@ -402,7 +368,6 @@ func (brm *bcNoBlockResponseMessage) String() string { //------------------------------------- -// NOTE: keep up-to-date with maxBlockchainResponseSize type bcBlockResponseMessage struct { Block *types.Block } diff --git a/blockchain/reactor_test.go b/blockchain/reactor_test.go index 263ca0f05..d72ff8c30 100644 --- a/blockchain/reactor_test.go +++ b/blockchain/reactor_test.go @@ -3,8 +3,6 @@ package blockchain import ( "testing" - wire "github.com/tendermint/go-wire" - cmn "github.com/tendermint/tmlibs/common" dbm "github.com/tendermint/tmlibs/db" "github.com/tendermint/tmlibs/log" @@ -19,7 +17,10 @@ import ( func makeStateAndBlockStore(logger log.Logger) (sm.State, *BlockStore) { config := cfg.ResetTestRoot("blockchain_reactor_test") blockStore := NewBlockStore(dbm.NewMemDB()) - state, _ := sm.LoadStateFromDBOrGenesisFile(dbm.NewMemDB(), config.GenesisFile()) + state, err := sm.LoadStateFromDBOrGenesisFile(dbm.NewMemDB(), config.GenesisFile()) + if err != nil { + panic(cmn.ErrorWrap(err, "error constructing state from genesis file")) + } return state, blockStore } @@ -76,10 +77,9 @@ func TestNoBlockResponse(t *testing.T) { // wait for our response to be received on the peer for _, tt := range tests { reqBlockMsg := &bcBlockRequestMessage{tt.height} - reqBlockBytes := wire.BinaryBytes(struct{ BlockchainMessage }{reqBlockMsg}) + reqBlockBytes := cdc.MustMarshalBinaryBare(reqBlockMsg) bcr.Receive(chID, peer, reqBlockBytes) - value := peer.lastValue() - msg := value.(struct{ BlockchainMessage }).BlockchainMessage + msg := peer.lastBlockchainMessage() if tt.existent { if blockMsg, ok := msg.(*bcBlockResponseMessage); !ok { @@ -173,26 +173,30 @@ func newbcrTestPeer(id p2p.ID) *bcrTestPeer { return bcr } -func (tp *bcrTestPeer) lastValue() interface{} { return <-tp.ch } +func (tp *bcrTestPeer) lastBlockchainMessage() interface{} { return <-tp.ch } -func (tp *bcrTestPeer) TrySend(chID byte, value interface{}) bool { - if _, ok := value.(struct{ BlockchainMessage }). - BlockchainMessage.(*bcStatusResponseMessage); ok { +func (tp *bcrTestPeer) TrySend(chID byte, msgBytes []byte) bool { + var msg BlockchainMessage + err := cdc.UnmarshalBinaryBare(msgBytes, &msg) + if err != nil { + panic(cmn.ErrorWrap(err, "Error while trying to parse a BlockchainMessage")) + } + if _, ok := msg.(*bcStatusResponseMessage); ok { // Discard status response messages since they skew our results // We only want to deal with: // + bcBlockResponseMessage // + bcNoBlockResponseMessage } else { - tp.ch <- value + tp.ch <- msg } return true } -func (tp *bcrTestPeer) Send(chID byte, data interface{}) bool { return tp.TrySend(chID, data) } -func (tp *bcrTestPeer) NodeInfo() p2p.NodeInfo { return p2p.NodeInfo{} } -func (tp *bcrTestPeer) Status() p2p.ConnectionStatus { return p2p.ConnectionStatus{} } -func (tp *bcrTestPeer) ID() p2p.ID { return tp.id } -func (tp *bcrTestPeer) IsOutbound() bool { return false } -func (tp *bcrTestPeer) IsPersistent() bool { return true } -func (tp *bcrTestPeer) Get(s string) interface{} { return s } -func (tp *bcrTestPeer) Set(string, interface{}) {} +func (tp *bcrTestPeer) Send(chID byte, msgBytes []byte) bool { return tp.TrySend(chID, msgBytes) } +func (tp *bcrTestPeer) NodeInfo() p2p.NodeInfo { return p2p.NodeInfo{} } +func (tp *bcrTestPeer) Status() p2p.ConnectionStatus { return p2p.ConnectionStatus{} } +func (tp *bcrTestPeer) ID() p2p.ID { return tp.id } +func (tp *bcrTestPeer) IsOutbound() bool { return false } +func (tp *bcrTestPeer) IsPersistent() bool { return true } +func (tp *bcrTestPeer) Get(s string) interface{} { return s } +func (tp *bcrTestPeer) Set(string, interface{}) {} diff --git a/blockchain/store.go b/blockchain/store.go index b949bc904..f699c1c7c 100644 --- a/blockchain/store.go +++ b/blockchain/store.go @@ -1,14 +1,10 @@ package blockchain import ( - "bytes" "encoding/json" "fmt" - "io" "sync" - wire "github.com/tendermint/go-wire" - cmn "github.com/tendermint/tmlibs/common" dbm "github.com/tendermint/tmlibs/db" @@ -54,38 +50,25 @@ func (bs *BlockStore) Height() int64 { return bs.height } -// GetReader returns the value associated with the given key wrapped in an io.Reader. -// If no value is found, it returns nil. -// It's mainly for use with wire.ReadBinary. -func (bs *BlockStore) GetReader(key []byte) io.Reader { - bytez := bs.db.Get(key) - if bytez == nil { - return nil - } - return bytes.NewReader(bytez) -} - // LoadBlock returns the block with the given height. // If no block is found for that height, it returns nil. func (bs *BlockStore) LoadBlock(height int64) *types.Block { - var n int - var err error - r := bs.GetReader(calcBlockMetaKey(height)) - if r == nil { - return nil - } - blockMeta := wire.ReadBinary(&types.BlockMeta{}, r, 0, &n, &err).(*types.BlockMeta) + var blockMeta *types.BlockMeta + bz := bs.db.Get(calcBlockMetaKey(height)) + err := cdc.UnmarshalBinaryBare(bz, blockMeta) if err != nil { - panic(fmt.Sprintf("Error reading block meta: %v", err)) + panic(cmn.ErrorWrap(err, "Error reading block meta")) } - bytez := []byte{} + + var block *types.Block + buf := []byte{} for i := 0; i < blockMeta.BlockID.PartsHeader.Total; i++ { part := bs.LoadBlockPart(height, i) - bytez = append(bytez, part.Bytes...) + buf = append(buf, part.Bytes...) } - block := wire.ReadBinary(&types.Block{}, bytes.NewReader(bytez), 0, &n, &err).(*types.Block) + err = cdc.UnmarshalBinaryBare(buf, block) if err != nil { - panic(fmt.Sprintf("Error reading block: %v", err)) + panic(cmn.ErrorWrap(err, "Error reading block")) } return block } @@ -94,15 +77,11 @@ func (bs *BlockStore) LoadBlock(height int64) *types.Block { // from the block at the given height. // If no part is found for the given height and index, it returns nil. func (bs *BlockStore) LoadBlockPart(height int64, index int) *types.Part { - var n int - var err error - r := bs.GetReader(calcBlockPartKey(height, index)) - if r == nil { - return nil - } - part := wire.ReadBinary(&types.Part{}, r, 0, &n, &err).(*types.Part) + var part *types.Part + bz := bs.db.Get(calcBlockPartKey(height, index)) + err := cdc.UnmarshalBinaryBare(bz, part) if err != nil { - panic(fmt.Sprintf("Error reading block part: %v", err)) + panic(cmn.ErrorWrap(err, "Error reading block part")) } return part } @@ -110,15 +89,11 @@ func (bs *BlockStore) LoadBlockPart(height int64, index int) *types.Part { // LoadBlockMeta returns the BlockMeta for the given height. // If no block is found for the given height, it returns nil. func (bs *BlockStore) LoadBlockMeta(height int64) *types.BlockMeta { - var n int - var err error - r := bs.GetReader(calcBlockMetaKey(height)) - if r == nil { - return nil - } - blockMeta := wire.ReadBinary(&types.BlockMeta{}, r, 0, &n, &err).(*types.BlockMeta) + var blockMeta *types.BlockMeta + bz := bs.db.Get(calcBlockMetaKey(height)) + err := cdc.UnmarshalBinaryBare(bz, blockMeta) if err != nil { - panic(fmt.Sprintf("Error reading block meta: %v", err)) + panic(cmn.ErrorWrap(err, "Error reading block meta")) } return blockMeta } @@ -128,15 +103,11 @@ func (bs *BlockStore) LoadBlockMeta(height int64) *types.BlockMeta { // and it comes from the block.LastCommit for `height+1`. // If no commit is found for the given height, it returns nil. func (bs *BlockStore) LoadBlockCommit(height int64) *types.Commit { - var n int - var err error - r := bs.GetReader(calcBlockCommitKey(height)) - if r == nil { - return nil - } - commit := wire.ReadBinary(&types.Commit{}, r, 0, &n, &err).(*types.Commit) + var commit *types.Commit + bz := bs.db.Get(calcBlockCommitKey(height)) + err := cdc.UnmarshalBinaryBare(bz, commit) if err != nil { - panic(fmt.Sprintf("Error reading commit: %v", err)) + panic(cmn.ErrorWrap(err, "Error reading block commit")) } return commit } @@ -145,15 +116,11 @@ func (bs *BlockStore) LoadBlockCommit(height int64) *types.Commit { // This is useful when we've seen a commit, but there has not yet been // a new block at `height + 1` that includes this commit in its block.LastCommit. func (bs *BlockStore) LoadSeenCommit(height int64) *types.Commit { - var n int - var err error - r := bs.GetReader(calcSeenCommitKey(height)) - if r == nil { - return nil - } - commit := wire.ReadBinary(&types.Commit{}, r, 0, &n, &err).(*types.Commit) + var commit *types.Commit + bz := bs.db.Get(calcSeenCommitKey(height)) + err := cdc.UnmarshalBinaryBare(bz, commit) if err != nil { - panic(fmt.Sprintf("Error reading commit: %v", err)) + panic(cmn.ErrorWrap(err, "Error reading block commit")) } return commit } @@ -178,7 +145,7 @@ func (bs *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, s // Save block meta blockMeta := types.NewBlockMeta(block, blockParts) - metaBytes := wire.BinaryBytes(blockMeta) + metaBytes := cdc.MustMarshalBinaryBare(blockMeta) bs.db.Set(calcBlockMetaKey(height), metaBytes) // Save block parts @@ -187,12 +154,12 @@ func (bs *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, s } // Save block commit (duplicate and separate from the Block) - blockCommitBytes := wire.BinaryBytes(block.LastCommit) + blockCommitBytes := cdc.MustMarshalBinaryBare(block.LastCommit) bs.db.Set(calcBlockCommitKey(height-1), blockCommitBytes) // Save seen commit (seen +2/3 precommits for block) // NOTE: we can delete this at a later height - seenCommitBytes := wire.BinaryBytes(seenCommit) + seenCommitBytes := cdc.MustMarshalBinaryBare(seenCommit) bs.db.Set(calcSeenCommitKey(height), seenCommitBytes) // Save new BlockStoreStateJSON descriptor @@ -211,7 +178,7 @@ func (bs *BlockStore) saveBlockPart(height int64, index int, part *types.Part) { if height != bs.Height()+1 { cmn.PanicSanity(cmn.Fmt("BlockStore can only save contiguous blocks. Wanted %v, got %v", bs.Height()+1, height)) } - partBytes := wire.BinaryBytes(part) + partBytes := cdc.MustMarshalBinaryBare(part) bs.db.Set(calcBlockPartKey(height, index), partBytes) } diff --git a/blockchain/store_test.go b/blockchain/store_test.go index 16185ca0d..ac0f1a430 100644 --- a/blockchain/store_test.go +++ b/blockchain/store_test.go @@ -3,7 +3,6 @@ package blockchain import ( "bytes" "fmt" - "io/ioutil" "runtime/debug" "strings" "testing" @@ -11,9 +10,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - - wire "github.com/tendermint/go-wire" - "github.com/tendermint/tmlibs/db" "github.com/tendermint/tmlibs/log" @@ -61,38 +57,6 @@ func TestNewBlockStore(t *testing.T) { assert.Equal(t, bs.Height(), int64(0), "expecting nil bytes to be unmarshaled alright") } -func TestBlockStoreGetReader(t *testing.T) { - db := db.NewMemDB() - // Initial setup - db.Set([]byte("Foo"), []byte("Bar")) - db.Set([]byte("Foo1"), nil) - - bs := NewBlockStore(db) - - tests := [...]struct { - key []byte - want []byte - }{ - 0: {key: []byte("Foo"), want: []byte("Bar")}, - 1: {key: []byte("KnoxNonExistent"), want: nil}, - 2: {key: []byte("Foo1"), want: []byte{}}, - } - - for i, tt := range tests { - r := bs.GetReader(tt.key) - if r == nil { - assert.Nil(t, tt.want, "#%d: expected a non-nil reader", i) - continue - } - slurp, err := ioutil.ReadAll(r) - if err != nil { - t.Errorf("#%d: unexpected Read err: %v", i, err) - } else { - assert.Equal(t, slurp, tt.want, "#%d: mismatch", i) - } - } -} - func freshBlockStore() (*BlockStore, db.DB) { db := db.NewMemDB() return NewBlockStore(db), db @@ -305,14 +269,6 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) { } } -func binarySerializeIt(v interface{}) []byte { - var n int - var err error - buf := new(bytes.Buffer) - wire.WriteBinary(v, buf, &n, &err) - return buf.Bytes() -} - func TestLoadBlockPart(t *testing.T) { bs, db := freshBlockStore() height, index := int64(10), 1 @@ -334,7 +290,7 @@ func TestLoadBlockPart(t *testing.T) { require.Contains(t, panicErr.Error(), "Error reading block part") // 3. A good block serialized and saved to the DB should be retrievable - db.Set(calcBlockPartKey(height, index), binarySerializeIt(part1)) + db.Set(calcBlockPartKey(height, index), cdc.MustMarshalBinaryBare(part1)) gotPart, _, panicErr := doFn(loadPart) require.Nil(t, panicErr, "an existent and proper block should not panic") require.Nil(t, res, "a properly saved block should return a proper block") @@ -364,11 +320,11 @@ func TestLoadBlockMeta(t *testing.T) { // 3. A good blockMeta serialized and saved to the DB should be retrievable meta := &types.BlockMeta{} - db.Set(calcBlockMetaKey(height), binarySerializeIt(meta)) + db.Set(calcBlockMetaKey(height), cdc.MustMarshalBinaryBare(meta)) gotMeta, _, panicErr := doFn(loadMeta) require.Nil(t, panicErr, "an existent and proper block should not panic") require.Nil(t, res, "a properly saved blockMeta should return a proper blocMeta ") - require.Equal(t, binarySerializeIt(meta), binarySerializeIt(gotMeta), + require.Equal(t, cdc.MustMarshalBinaryBare(meta), cdc.MustMarshalBinaryBare(gotMeta), "expecting successful retrieval of previously saved blockMeta") } diff --git a/blockchain/wire.go b/blockchain/wire.go new file mode 100644 index 000000000..72431db82 --- /dev/null +++ b/blockchain/wire.go @@ -0,0 +1,11 @@ +package blockchain + +import ( + "github.com/tendermint/go-amino" +) + +var cdc = amino.NewCodec() + +func init() { + RegisterBlockchainMessages(cdc) +} diff --git a/consensus/common_test.go b/consensus/common_test.go index 6d16958ca..dbe39dda1 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -405,9 +405,9 @@ func getSwitchIndex(switches []*p2p.Switch, peer p2p.Peer) int { //------------------------------------------------------------------------------- // genesis -func randGenesisDoc(numValidators int, randPower bool, minPower int64) (*types.GenesisDoc, []*types.PrivValidatorFS) { +func randGenesisDoc(numValidators int, randPower bool, minPower int64) (*types.GenesisDoc, []types.PrivValidator) { validators := make([]types.GenesisValidator, numValidators) - privValidators := make([]*types.PrivValidatorFS, numValidators) + privValidators := make([]types.PrivValidator, numValidators) for i := 0; i < numValidators; i++ { val, privVal := types.RandValidator(randPower, minPower) validators[i] = types.GenesisValidator{ @@ -425,7 +425,7 @@ func randGenesisDoc(numValidators int, randPower bool, minPower int64) (*types.G }, privValidators } -func randGenesisState(numValidators int, randPower bool, minPower int64) (sm.State, []*types.PrivValidatorFS) { +func randGenesisState(numValidators int, randPower bool, minPower int64) (sm.State, []types.PrivValidator) { genDoc, privValidators := randGenesisDoc(numValidators, randPower, minPower) s0, _ := sm.MakeGenesisState(genDoc) db := dbm.NewMemDB() diff --git a/consensus/state.go b/consensus/state.go index 3cde13bd2..7722990b2 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -1301,10 +1301,10 @@ func (cs *ConsensusState) addProposalBlockPart(height int64, part *types.Part, v } if added && cs.ProposalBlockParts.IsComplete() { // Added and completed! - var n int - var err error - cs.ProposalBlock = wire.ReadBinary(&types.Block{}, cs.ProposalBlockParts.GetReader(), - cs.state.ConsensusParams.BlockSize.MaxBytes, &n, &err).(*types.Block) + err = cdc.UnmarshalBinaryBare(cs.ProposalBlockParts.GetReader(), &cs.ProposalBlock, cs.state.ConsensusParams.BlockSize.MaxBytes) + if err != nil { + return true, err + } // NOTE: it's possible to receive complete proposal blocks for future rounds without having the proposal cs.Logger.Info("Received complete proposal block", "height", cs.ProposalBlock.Height, "hash", cs.ProposalBlock.Hash()) if cs.Step == cstypes.RoundStepPropose && cs.isProposalComplete() { @@ -1314,7 +1314,7 @@ func (cs *ConsensusState) addProposalBlockPart(height int64, part *types.Part, v // If we're waiting on the proposal block... cs.tryFinalizeCommit(height) } - return true, err + return true, nil } return added, nil } diff --git a/consensus/types/height_vote_set_test.go b/consensus/types/height_vote_set_test.go index 246c0b711..678d34759 100644 --- a/consensus/types/height_vote_set_test.go +++ b/consensus/types/height_vote_set_test.go @@ -48,7 +48,7 @@ func TestPeerCatchupRounds(t *testing.T) { } -func makeVoteHR(t *testing.T, height int64, round int, privVals []*types.PrivValidatorFS, valIndex int) *types.Vote { +func makeVoteHR(t *testing.T, height int64, round int, privVals []types.PrivValidator, valIndex int) *types.Vote { privVal := privVals[valIndex] vote := &types.Vote{ ValidatorAddress: privVal.GetAddress(), diff --git a/types/results_test.go b/types/results_test.go index 8018b4e85..009e2693d 100644 --- a/types/results_test.go +++ b/types/results_test.go @@ -15,10 +15,10 @@ func TestABCIResults(t *testing.T) { e := ABCIResult{Code: 14, Data: []byte("foo")} f := ABCIResult{Code: 14, Data: []byte("bar")} - // Nil and []byte{} should not produce the same hash. + // Nil and []byte{} should produce the same hash. require.Equal(t, a.Hash(), a.Hash()) require.Equal(t, b.Hash(), b.Hash()) - require.NotEqual(t, a.Hash(), b.Hash()) + require.Equal(t, a.Hash(), b.Hash()) // a and b should be the same, don't go in results. results := ABCIResults{a, c, d, e, f} From 39ff4d22e93495e11df6f9f75b0e1335590ef744 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 3 Apr 2018 22:34:18 +0300 Subject: [PATCH 035/143] minor cleanup --- lite/proxy/validate_test.go | 55 +++++++++++++++++-------------------- types/block.go | 4 ++- 2 files changed, 28 insertions(+), 31 deletions(-) diff --git a/lite/proxy/validate_test.go b/lite/proxy/validate_test.go index bd57994d7..782a6aabb 100644 --- a/lite/proxy/validate_test.go +++ b/lite/proxy/validate_test.go @@ -12,14 +12,15 @@ import ( ) var ( - deadBeefTxs = types.Txs{[]byte("DE"), []byte("AD"), []byte("BE"), []byte("EF")} - - deadBeefRipEmd160Hash = deadBeefTxs.Hash() + deadBeefTxs = types.Txs{[]byte("DE"), []byte("AD"), []byte("BE"), []byte("EF")} + deadBeefHash = deadBeefTxs.Hash() + testTime1 = time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC) + testTime2 = time.Date(2017, 1, 2, 1, 1, 1, 1, time.UTC) ) -var hdrHeight11Tendermint = &types.Header{ +var hdrHeight11 = &types.Header{ Height: 11, - Time: time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC), + Time: testTime1, ValidatorsHash: []byte("Tendermint"), } @@ -54,14 +55,14 @@ func TestValidateBlock(t *testing.T) { // Start Header.Hash mismatch test { - block: &types.Block{Header: hdrHeight11Tendermint}, + block: &types.Block{Header: hdrHeight11}, commit: lite.Commit{Header: &types.Header{Height: 11}}, wantErr: "Headers don't match", }, { - block: &types.Block{Header: hdrHeight11Tendermint}, - commit: lite.Commit{Header: hdrHeight11Tendermint}, + block: &types.Block{Header: hdrHeight11}, + commit: lite.Commit{Header: hdrHeight11}, }, // End Header.Hash mismatch test @@ -79,7 +80,7 @@ func TestValidateBlock(t *testing.T) { }, { block: &types.Block{ - Header: &types.Header{Height: 11, DataHash: deadBeefRipEmd160Hash}, + Header: &types.Header{Height: 11, DataHash: deadBeefHash}, Data: &types.Data{Txs: deadBeefTxs}, }, commit: lite.Commit{ @@ -136,33 +137,27 @@ func TestValidateBlockMeta(t *testing.T) { // Start Headers don't match test { - meta: &types.BlockMeta{Header: hdrHeight11Tendermint}, + meta: &types.BlockMeta{Header: hdrHeight11}, commit: lite.Commit{Header: &types.Header{Height: 11}}, wantErr: "Headers don't match", }, { - meta: &types.BlockMeta{Header: hdrHeight11Tendermint}, - commit: lite.Commit{Header: hdrHeight11Tendermint}, + meta: &types.BlockMeta{Header: hdrHeight11}, + commit: lite.Commit{Header: hdrHeight11}, }, { meta: &types.BlockMeta{ Header: &types.Header{ - Height: 11, - // TODO: (@odeke-em) inquire why ValidatorsHash has to be non-blank - // for the Header to be hashed. Perhaps this is a security hole because - // an aggressor could perhaps pass in headers that don't have - // ValidatorsHash set and we won't be able to validate blocks. + Height: 11, ValidatorsHash: []byte("lite-test"), - // TODO: (@odeke-em) file an issue with Tendermint to get them to update - // to the latest go-wire, then no more need for this value fill to avoid - // the time zero value of less than 1970. - Time: time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC), + // TODO: should be able to use empty time after Amino upgrade + Time: testTime1, }, }, commit: lite.Commit{ - Header: &types.Header{Height: 11, DataHash: deadBeefRipEmd160Hash}, + Header: &types.Header{Height: 11, DataHash: deadBeefHash}, }, wantErr: "Headers don't match", }, @@ -170,16 +165,16 @@ func TestValidateBlockMeta(t *testing.T) { { meta: &types.BlockMeta{ Header: &types.Header{ - Height: 11, DataHash: deadBeefRipEmd160Hash, + Height: 11, DataHash: deadBeefHash, ValidatorsHash: []byte("Tendermint"), - Time: time.Date(2017, 1, 2, 1, 1, 1, 1, time.UTC), + Time: testTime1, }, }, commit: lite.Commit{ Header: &types.Header{ - Height: 11, DataHash: deadBeefRipEmd160Hash, + Height: 11, DataHash: deadBeefHash, ValidatorsHash: []byte("Tendermint"), - Time: time.Date(2018, 1, 2, 1, 1, 1, 1, time.UTC), + Time: testTime2, }, Commit: &types.Commit{BlockID: types.BlockID{Hash: []byte("DEADBEEF")}}, }, @@ -189,16 +184,16 @@ func TestValidateBlockMeta(t *testing.T) { { meta: &types.BlockMeta{ Header: &types.Header{ - Height: 11, DataHash: deadBeefRipEmd160Hash, + Height: 11, DataHash: deadBeefHash, ValidatorsHash: []byte("Tendermint"), - Time: time.Date(2017, 1, 2, 1, 1, 1, 1, time.UTC), + Time: testTime2, }, }, commit: lite.Commit{ Header: &types.Header{ - Height: 11, DataHash: deadBeefRipEmd160Hash, + Height: 11, DataHash: deadBeefHash, ValidatorsHash: []byte("Tendermint-x"), - Time: time.Date(2017, 1, 2, 1, 1, 1, 1, time.UTC), + Time: testTime2, }, Commit: &types.Commit{BlockID: types.BlockID{Hash: []byte("DEADBEEF")}}, }, diff --git a/types/block.go b/types/block.go index 970ca36f4..774c69f32 100644 --- a/types/block.go +++ b/types/block.go @@ -177,7 +177,9 @@ type Header struct { } // Hash returns the hash of the header. -// Returns nil if ValidatorHash is missing. +// Returns nil if ValidatorHash is missing, +// since a Header is not valid unless there is +// a ValidaotrsHash (corresponding to the validator set). func (h *Header) Hash() cmn.HexBytes { if h == nil || len(h.ValidatorsHash) == 0 { return nil From 5f548c7679dbc4c9ab0ff6ead0dacc77bbf719db Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 3 Apr 2018 22:56:46 +0300 Subject: [PATCH 036/143] consensus: close pubsub channels. fixes #1372 --- consensus/reactor.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/consensus/reactor.go b/consensus/reactor.go index b13be0e87..c71f9e0c4 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -371,19 +371,21 @@ func (conR *ConsensusReactor) startBroadcastRoutine() error { } go func() { + var data interface{} + var ok bool for { select { - case data, ok := <-stepsCh: + case data, ok = <-stepsCh: if ok { // a receive from a closed channel returns the zero value immediately edrs := data.(types.TMEventData).Unwrap().(types.EventDataRoundState) conR.broadcastNewRoundStep(edrs.RoundState.(*cstypes.RoundState)) } - case data, ok := <-votesCh: + case data, ok = <-votesCh: if ok { edv := data.(types.TMEventData).Unwrap().(types.EventDataVote) conR.broadcastHasVoteMessage(edv.Vote) } - case data, ok := <-heartbeatsCh: + case data, ok = <-heartbeatsCh: if ok { edph := data.(types.TMEventData).Unwrap().(types.EventDataProposalHeartbeat) conR.broadcastProposalHeartbeatMessage(edph) @@ -392,6 +394,10 @@ func (conR *ConsensusReactor) startBroadcastRoutine() error { conR.eventBus.UnsubscribeAll(ctx, subscriber) return } + if !ok { + conR.eventBus.UnsubscribeAll(ctx, subscriber) + return + } } }() From 37ce6b195a2c26748bb7cc61466116748a589da0 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 29 Mar 2018 13:32:14 +0200 Subject: [PATCH 037/143] ValidatorSet#GetByAddress: return -1 if no validator was found --- consensus/state.go | 6 +----- types/validator_set.go | 5 ++++- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/consensus/state.go b/consensus/state.go index 6acad698b..57c7b32f0 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -720,11 +720,7 @@ func (cs *ConsensusState) needProofBlock(height int64) bool { func (cs *ConsensusState) proposalHeartbeat(height int64, round int) { counter := 0 addr := cs.privValidator.GetAddress() - valIndex, v := cs.Validators.GetByAddress(addr) - if v == nil { - // not a validator - valIndex = -1 - } + valIndex, _ := cs.Validators.GetByAddress(addr) chainID := cs.state.ChainID for { rs := cs.GetRoundState() diff --git a/types/validator_set.go b/types/validator_set.go index dc6b66d68..ce32772da 100644 --- a/types/validator_set.go +++ b/types/validator_set.go @@ -90,14 +90,17 @@ func (valSet *ValidatorSet) HasAddress(address []byte) bool { return idx != len(valSet.Validators) && bytes.Equal(valSet.Validators[idx].Address, address) } +// GetByAddress returns an index of the validator with address and validator +// itself if found. Otherwise, -1 and nil are returned. func (valSet *ValidatorSet) GetByAddress(address []byte) (index int, val *Validator) { idx := sort.Search(len(valSet.Validators), func(i int) bool { return bytes.Compare(address, valSet.Validators[i].Address) <= 0 }) if idx != len(valSet.Validators) && bytes.Equal(valSet.Validators[idx].Address, address) { return idx, valSet.Validators[idx].Copy() + } else { + return -1, nil } - return 0, nil } // GetByIndex returns the validator by index. From 39a496378203bbc36e6e9e3cac057e21d1174c33 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 29 Mar 2018 13:35:15 +0200 Subject: [PATCH 038/143] document funcs in validator_set.go --- types/validator_set.go | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/types/validator_set.go b/types/validator_set.go index ce32772da..10183beca 100644 --- a/types/validator_set.go +++ b/types/validator_set.go @@ -83,6 +83,8 @@ func (valSet *ValidatorSet) Copy() *ValidatorSet { } } +// HasAddress returns true if address given is in the validator set, false - +// otherwise. func (valSet *ValidatorSet) HasAddress(address []byte) bool { idx := sort.Search(len(valSet.Validators), func(i int) bool { return bytes.Compare(address, valSet.Validators[i].Address) <= 0 @@ -96,16 +98,16 @@ func (valSet *ValidatorSet) GetByAddress(address []byte) (index int, val *Valida idx := sort.Search(len(valSet.Validators), func(i int) bool { return bytes.Compare(address, valSet.Validators[i].Address) <= 0 }) - if idx != len(valSet.Validators) && bytes.Equal(valSet.Validators[idx].Address, address) { + if idx < len(valSet.Validators) && bytes.Equal(valSet.Validators[idx].Address, address) { return idx, valSet.Validators[idx].Copy() } else { return -1, nil } } -// GetByIndex returns the validator by index. -// It returns nil values if index < 0 or -// index >= len(ValidatorSet.Validators) +// GetByIndex returns the validator's address and validator itself by index. +// It returns nil values if index is less than 0 or greater or equal to +// len(ValidatorSet.Validators). func (valSet *ValidatorSet) GetByIndex(index int) (address []byte, val *Validator) { if index < 0 || index >= len(valSet.Validators) { return nil, nil @@ -114,10 +116,12 @@ func (valSet *ValidatorSet) GetByIndex(index int) (address []byte, val *Validato return val.Address, val.Copy() } +// Size returns the length of the validator set. func (valSet *ValidatorSet) Size() int { return len(valSet.Validators) } +// TotalVotingPower returns the sum of the voting powers of all validators. func (valSet *ValidatorSet) TotalVotingPower() int64 { if valSet.totalVotingPower == 0 { for _, val := range valSet.Validators { @@ -128,6 +132,8 @@ func (valSet *ValidatorSet) TotalVotingPower() int64 { return valSet.totalVotingPower } +// GetProposer returns the current proposer. If the validator set is empty, nil +// is returned. func (valSet *ValidatorSet) GetProposer() (proposer *Validator) { if len(valSet.Validators) == 0 { return nil @@ -148,6 +154,8 @@ func (valSet *ValidatorSet) findProposer() *Validator { return proposer } +// Hash returns the Merkle root hash build using validators (as leaves) in the +// set. func (valSet *ValidatorSet) Hash() []byte { if len(valSet.Validators) == 0 { return nil @@ -159,6 +167,8 @@ func (valSet *ValidatorSet) Hash() []byte { return merkle.SimpleHashFromHashers(hashers) } +// Add adds val to the validator set and returns true. It returns false if val +// is already in the set. func (valSet *ValidatorSet) Add(val *Validator) (added bool) { val = val.Copy() idx := sort.Search(len(valSet.Validators), func(i int) bool { @@ -185,6 +195,8 @@ func (valSet *ValidatorSet) Add(val *Validator) (added bool) { } } +// Update updates val and returns true. It returns false if val is not present +// in the set. func (valSet *ValidatorSet) Update(val *Validator) (updated bool) { index, sameVal := valSet.GetByAddress(val.Address) if sameVal == nil { @@ -197,6 +209,8 @@ func (valSet *ValidatorSet) Update(val *Validator) (updated bool) { return true } +// Remove deletes the validator with address. It returns the validator removed +// and true. If returns nil and false if validator is not present in the set. func (valSet *ValidatorSet) Remove(address []byte) (val *Validator, removed bool) { idx := sort.Search(len(valSet.Validators), func(i int) bool { return bytes.Compare(address, valSet.Validators[i].Address) <= 0 @@ -216,6 +230,7 @@ func (valSet *ValidatorSet) Remove(address []byte) (val *Validator, removed bool return removedVal, true } +// Iterate will run the given function over the set. func (valSet *ValidatorSet) Iterate(fn func(index int, val *Validator) bool) { for i, val := range valSet.Validators { stop := fn(i, val.Copy()) From 0732526465b45e7db85e0ce1b992837d48b4f777 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 29 Mar 2018 13:37:12 +0200 Subject: [PATCH 039/143] use more relaxing < and >= ops instead of != an example of Search from godocs: ``` package main import ( "fmt" "sort" ) func main() { a := []int{1, 3, 6, 10, 15, 21, 28, 36, 45, 55} x := 6 i := sort.Search(len(a), func(i int) bool { return a[i] >= x }) if i < len(a) && a[i] == x { fmt.Printf("found %d at index %d in %v\n", x, i, a) } else { fmt.Printf("%d not found in %v\n", x, a) } } ``` --- types/validator_set.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/types/validator_set.go b/types/validator_set.go index 10183beca..887580d69 100644 --- a/types/validator_set.go +++ b/types/validator_set.go @@ -89,7 +89,7 @@ func (valSet *ValidatorSet) HasAddress(address []byte) bool { idx := sort.Search(len(valSet.Validators), func(i int) bool { return bytes.Compare(address, valSet.Validators[i].Address) <= 0 }) - return idx != len(valSet.Validators) && bytes.Equal(valSet.Validators[idx].Address, address) + return idx < len(valSet.Validators) && bytes.Equal(valSet.Validators[idx].Address, address) } // GetByAddress returns an index of the validator with address and validator @@ -174,7 +174,7 @@ func (valSet *ValidatorSet) Add(val *Validator) (added bool) { idx := sort.Search(len(valSet.Validators), func(i int) bool { return bytes.Compare(val.Address, valSet.Validators[i].Address) <= 0 }) - if idx == len(valSet.Validators) { + if idx >= len(valSet.Validators) { valSet.Validators = append(valSet.Validators, val) // Invalidate cache valSet.Proposer = nil @@ -215,7 +215,7 @@ func (valSet *ValidatorSet) Remove(address []byte) (val *Validator, removed bool idx := sort.Search(len(valSet.Validators), func(i int) bool { return bytes.Compare(address, valSet.Validators[i].Address) <= 0 }) - if idx == len(valSet.Validators) || !bytes.Equal(valSet.Validators[idx].Address, address) { + if idx >= len(valSet.Validators) || !bytes.Equal(valSet.Validators[idx].Address, address) { return nil, false } removedVal := valSet.Validators[idx] From ed782e7508f430c2ece4fb2ebce82e2d4c2a6623 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 29 Mar 2018 13:39:15 +0200 Subject: [PATCH 040/143] include validator's voting power in /status Refs #581 --- CHANGELOG.md | 1 + rpc/core/status.go | 49 ++++++++++++++++++++++++++++++++++--- rpc/core/types/responses.go | 19 ++++++++------ types/validator_set.go | 3 +-- 4 files changed, 60 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e7c61489c..cf01bdbb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ BUG FIXES: IMPROVEMENTS: - [rpc] `/tx` and `/tx_search` responses now include the transaction hash +- [rpc] include validator power in `/status` ## 0.17.1 (March 27th, 2018) diff --git a/rpc/core/status.go b/rpc/core/status.go index a8771c8f7..b4543a61c 100644 --- a/rpc/core/status.go +++ b/rpc/core/status.go @@ -1,9 +1,11 @@ package core import ( + "bytes" "time" ctypes "github.com/tendermint/tendermint/rpc/core/types" + sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" cmn "github.com/tendermint/tmlibs/common" ) @@ -48,7 +50,10 @@ import ( // "remote_addr": "", // "network": "test-chain-qhVCa2", // "moniker": "vagrant-ubuntu-trusty-64", -// "pub_key": "844981FE99ABB19F7816F2D5E94E8A74276AB1153760A7799E925C75401856C6" +// "pub_key": "844981FE99ABB19F7816F2D5E94E8A74276AB1153760A7799E925C75401856C6", +// "validator_status": { +// "voting_power": 10 +// } // } // }, // "id": "", @@ -72,12 +77,50 @@ func Status() (*ctypes.ResultStatus, error) { latestBlockTime := time.Unix(0, latestBlockTimeNano) - return &ctypes.ResultStatus{ + result := &ctypes.ResultStatus{ NodeInfo: p2pSwitch.NodeInfo(), PubKey: pubKey, LatestBlockHash: latestBlockHash, LatestAppHash: latestAppHash, LatestBlockHeight: latestHeight, LatestBlockTime: latestBlockTime, - Syncing: consensusReactor.FastSync()}, nil + Syncing: consensusReactor.FastSync(), + } + + // add ValidatorStatus if node is a validator + if val := validatorAtHeight(latestHeight); val != nil { + result.ValidatorStatus = ctypes.ValidatorStatus{ + VotingPower: val.VotingPower, + } + } + + return result, nil +} + +func validatorAtHeight(h int64) *types.Validator { + lastBlockHeight, vals := consensusState.GetValidators() + + privValAddress := pubKey.Address() + + // if we're still at height h, search in the current validator set + if lastBlockHeight == h { + for _, val := range vals { + if bytes.Equal(val.Address, privValAddress) { + return val + } + } + } + + // if we've moved to the next height, retrieve the validator set from DB + if lastBlockHeight > h { + vals, err := sm.LoadValidators(stateDB, h) + if err != nil { + // should not happen + return nil + } + _, val := vals.GetByAddress(privValAddress) + return val + } + + return nil } diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index 5b49e1af6..48f8723d6 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -54,14 +54,19 @@ func NewResultCommit(header *types.Header, commit *types.Commit, } } +type ValidatorStatus struct { + VotingPower int64 `json:"voting_power"` +} + type ResultStatus struct { - NodeInfo p2p.NodeInfo `json:"node_info"` - PubKey crypto.PubKey `json:"pub_key"` - LatestBlockHash cmn.HexBytes `json:"latest_block_hash"` - LatestAppHash cmn.HexBytes `json:"latest_app_hash"` - LatestBlockHeight int64 `json:"latest_block_height"` - LatestBlockTime time.Time `json:"latest_block_time"` - Syncing bool `json:"syncing"` + NodeInfo p2p.NodeInfo `json:"node_info"` + PubKey crypto.PubKey `json:"pub_key"` + LatestBlockHash cmn.HexBytes `json:"latest_block_hash"` + LatestAppHash cmn.HexBytes `json:"latest_app_hash"` + LatestBlockHeight int64 `json:"latest_block_height"` + LatestBlockTime time.Time `json:"latest_block_time"` + Syncing bool `json:"syncing"` + ValidatorStatus ValidatorStatus `json:"validator_status,omitempty"` } func (s *ResultStatus) TxIndexEnabled() bool { diff --git a/types/validator_set.go b/types/validator_set.go index 887580d69..4b84f85d7 100644 --- a/types/validator_set.go +++ b/types/validator_set.go @@ -100,9 +100,8 @@ func (valSet *ValidatorSet) GetByAddress(address []byte) (index int, val *Valida }) if idx < len(valSet.Validators) && bytes.Equal(valSet.Validators[idx].Address, address) { return idx, valSet.Validators[idx].Copy() - } else { - return -1, nil } + return -1, nil } // GetByIndex returns the validator's address and validator itself by index. From f907113c197a296e448d3734d26e8a06ec1cba6b Mon Sep 17 00:00:00 2001 From: Zaki Manian Date: Thu, 5 Apr 2018 02:02:23 -0700 Subject: [PATCH 041/143] Net_info should print the ID of peers (#1312) --- rpc/core/net.go | 1 + rpc/core/types/responses.go | 1 + 2 files changed, 2 insertions(+) diff --git a/rpc/core/net.go b/rpc/core/net.go index 9b04926ab..1918abf11 100644 --- a/rpc/core/net.go +++ b/rpc/core/net.go @@ -43,6 +43,7 @@ func NetInfo() (*ctypes.ResultNetInfo, error) { for _, peer := range p2pSwitch.Peers().List() { peers = append(peers, ctypes.Peer{ NodeInfo: peer.NodeInfo(), + ID: peer.ID(), IsOutbound: peer.IsOutbound(), ConnectionStatus: peer.Status(), }) diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index 48f8723d6..8a6fff63c 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -98,6 +98,7 @@ type ResultDialPeers struct { type Peer struct { p2p.NodeInfo `json:"node_info"` + p2p.ID `json:"node_id"` IsOutbound bool `json:"is_outbound"` ConnectionStatus p2p.ConnectionStatus `json:"connection_status"` } From a506cf47add829f72224531f173526e84af4826b Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 5 Apr 2018 11:42:45 +0200 Subject: [PATCH 042/143] protect Record* peerStateStats functions by mutex Fixes #1414 DATA RACE: ``` Read at 0x00c4214ee940 by goroutine 146: github.com/tendermint/tendermint/consensus.(*peerStateStats).String() :1 +0x57 fmt.(*pp).handleMethods() /usr/local/go/src/fmt/print.go:596 +0x3f4 fmt.(*pp).printArg() /usr/local/go/src/fmt/print.go:679 +0x11f fmt.(*pp).doPrintf() /usr/local/go/src/fmt/print.go:996 +0x319 fmt.Sprintf() /usr/local/go/src/fmt/print.go:196 +0x73 github.com/tendermint/tendermint/consensus.(*PeerState).StringIndented() github.com/tendermint/tendermint/consensus/_test/_obj_test/reactor.go:1426 +0x573 github.com/tendermint/tendermint/consensus.(*PeerState).String() github.com/tendermint/tendermint/consensus/_test/_obj_test/reactor.go:1419 +0x66 github.com/go-logfmt/logfmt.safeString() /home/ubuntu/go/src/github.com/go-logfmt/logfmt/encode.go:299 +0x9d github.com/go-logfmt/logfmt.writeValue() /home/ubuntu/go/src/github.com/go-logfmt/logfmt/encode.go:217 +0x5a0 github.com/go-logfmt/logfmt.(*Encoder).EncodeKeyval() /home/ubuntu/go/src/github.com/go-logfmt/logfmt/encode.go:61 +0x1dd github.com/tendermint/tmlibs/log.tmfmtLogger.Log() /home/ubuntu/go/src/github.com/tendermint/tmlibs/log/tmfmt_logger.go:107 +0x1001 github.com/tendermint/tmlibs/log.(*tmfmtLogger).Log() :1 +0x93 github.com/go-kit/kit/log.(*context).Log() /home/ubuntu/go/src/github.com/go-kit/kit/log/log.go:124 +0x248 github.com/tendermint/tmlibs/log.(*tmLogger).Debug() /home/ubuntu/go/src/github.com/tendermint/tmlibs/log/tm_logger.go:64 +0x1d0 github.com/tendermint/tendermint/consensus.(*PeerState).PickSendVote() github.com/tendermint/tendermint/consensus/_test/_obj_test/reactor.go:1059 +0x242 github.com/tendermint/tendermint/consensus.(*ConsensusReactor).gossipVotesForHeight() github.com/tendermint/tendermint/consensus/_test/_obj_test/reactor.go:789 +0x6ef github.com/tendermint/tendermint/consensus.(*ConsensusReactor).gossipVotesRoutine() github.com/tendermint/tendermint/consensus/_test/_obj_test/reactor.go:723 +0x1039 Previous write at 0x00c4214ee940 by goroutine 21: github.com/tendermint/tendermint/consensus.(*PeerState).RecordVote() github.com/tendermint/tendermint/consensus/_test/_obj_test/reactor.go:1242 +0x15a github.com/tendermint/tendermint/consensus.(*ConsensusReactor).Receive() github.com/tendermint/tendermint/consensus/_test/_obj_test/reactor.go:309 +0x32e6 github.com/tendermint/tendermint/p2p.createMConnection.func1() /home/ubuntu/go/src/github.com/tendermint/tendermint/p2p/peer.go:365 +0xea github.com/tendermint/tendermint/p2p/conn.(*MConnection).recvRoutine() /home/ubuntu/go/src/github.com/tendermint/tendermint/p2p/conn/connection.go:531 +0x779 ``` --- consensus/reactor.go | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/consensus/reactor.go b/consensus/reactor.go index c71f9e0c4..70a79d86a 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -1087,6 +1087,9 @@ func (ps *PeerState) ensureVoteBitArrays(height int64, numValidators int) { // It returns the total number of votes (1 per block). This essentially means // the number of blocks for which peer has been sending us votes. func (ps *PeerState) RecordVote(vote *types.Vote) int { + ps.mtx.Lock() + defer ps.mtx.Unlock() + if ps.stats.lastVoteHeight >= vote.Height { return ps.stats.votes } @@ -1098,13 +1101,20 @@ func (ps *PeerState) RecordVote(vote *types.Vote) int { // VotesSent returns the number of blocks for which peer has been sending us // votes. func (ps *PeerState) VotesSent() int { + ps.mtx.Lock() + defer ps.mtx.Unlock() + return ps.stats.votes } -// RecordVote updates internal statistics for this peer by recording the block part. -// It returns the total number of block parts (1 per block). This essentially means -// the number of blocks for which peer has been sending us block parts. +// RecordBlockPart updates internal statistics for this peer by recording the +// block part. It returns the total number of block parts (1 per block). This +// essentially means the number of blocks for which peer has been sending us +// block parts. func (ps *PeerState) RecordBlockPart(bp *BlockPartMessage) int { + ps.mtx.Lock() + defer ps.mtx.Unlock() + if ps.stats.lastBlockPartHeight >= bp.Height { return ps.stats.blockParts } @@ -1117,6 +1127,9 @@ func (ps *PeerState) RecordBlockPart(bp *BlockPartMessage) int { // BlockPartsSent returns the number of blocks for which peer has been sending // us block parts. func (ps *PeerState) BlockPartsSent() int { + ps.mtx.Lock() + defer ps.mtx.Unlock() + return ps.stats.blockParts } From 904a3115a623ad0b7a30d87d3472339ae7d1b5dc Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 22 Mar 2018 11:27:10 +0100 Subject: [PATCH 043/143] require addresses to have an ID by default Refs #1228 --- CHANGELOG.md | 9 +++-- p2p/listener.go | 2 +- p2p/netaddress.go | 37 +++++++++++++-------- p2p/netaddress_test.go | 75 ++++++++++++++++++++++++++---------------- 4 files changed, 77 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf01bdbb1..68416cbeb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,15 +25,18 @@ BUG FIXES: - Graceful handling/recovery for apps that have non-determinism or fail to halt - Graceful handling/recovery for violations of safety, or liveness -## 0.17.2 (TBD) +## 0.18.0 (TBD) -BUG FIXES: -- [rpc] fix subscribing using an abci.ResponseDeliverTx tag +BREAKING: +- [p2p] require all addresses come with an ID no matter what IMPROVEMENTS: - [rpc] `/tx` and `/tx_search` responses now include the transaction hash - [rpc] include validator power in `/status` +BUG FIXES: +- [rpc] fix subscribing using an abci.ResponseDeliverTx tag + ## 0.17.1 (March 27th, 2018) BUG FIXES: diff --git a/p2p/listener.go b/p2p/listener.go index 884c45ee8..e698765cd 100644 --- a/p2p/listener.go +++ b/p2p/listener.go @@ -72,7 +72,7 @@ func NewDefaultListener(protocol string, lAddr string, skipUPNP bool, logger log // Determine internal address... var intAddr *NetAddress - intAddr, err = NewNetAddressString(lAddr) + intAddr, err = NewNetAddressStringWithOptionalID(lAddr) if err != nil { panic(err) } diff --git a/p2p/netaddress.go b/p2p/netaddress.go index c7773a9f8..a77090a78 100644 --- a/p2p/netaddress.go +++ b/p2p/netaddress.go @@ -49,33 +49,45 @@ func NewNetAddress(id ID, addr net.Addr) *NetAddress { } ip := tcpAddr.IP port := uint16(tcpAddr.Port) - netAddr := NewNetAddressIPPort(ip, port) - netAddr.ID = id - return netAddr + na := NewNetAddressIPPort(ip, port) + na.ID = id + return na } -// NewNetAddressString returns a new NetAddress using the provided -// address in the form of "ID@IP:Port", where the ID is optional. +// NewNetAddressString returns a new NetAddress using the provided address in +// the form of "ID@IP:Port". // Also resolves the host if host is not an IP. func NewNetAddressString(addr string) (*NetAddress, error) { - addr = removeProtocolIfDefined(addr) + spl := strings.Split(addr, "@") + if len(spl) < 2 { + return nil, fmt.Errorf("Address (%s) does not contain ID", addr) + } + return NewNetAddressStringWithOptionalID(addr) +} + +// NewNetAddressStringWithOptionalID returns a new NetAddress using the +// provided address in the form of "ID@IP:Port", where the ID is optional. +// Also resolves the host if host is not an IP. +func NewNetAddressStringWithOptionalID(addr string) (*NetAddress, error) { + addrWithoutProtocol := removeProtocolIfDefined(addr) var id ID - spl := strings.Split(addr, "@") + spl := strings.Split(addrWithoutProtocol, "@") if len(spl) == 2 { idStr := spl[0] idBytes, err := hex.DecodeString(idStr) if err != nil { - return nil, errors.Wrap(err, fmt.Sprintf("Address (%s) contains invalid ID", addr)) + return nil, errors.Wrapf(err, "Address (%s) contains invalid ID", addrWithoutProtocol) } if len(idBytes) != IDByteLength { return nil, fmt.Errorf("Address (%s) contains ID of invalid length (%d). Should be %d hex-encoded bytes", - addr, len(idBytes), IDByteLength) + addrWithoutProtocol, len(idBytes), IDByteLength) } - id, addr = ID(idStr), spl[1] + + id, addrWithoutProtocol = ID(idStr), spl[1] } - host, portStr, err := net.SplitHostPort(addr) + host, portStr, err := net.SplitHostPort(addrWithoutProtocol) if err != nil { return nil, err } @@ -120,11 +132,10 @@ func NewNetAddressStrings(addrs []string) ([]*NetAddress, []error) { // NewNetAddressIPPort returns a new NetAddress using the provided IP // and port number. func NewNetAddressIPPort(ip net.IP, port uint16) *NetAddress { - na := &NetAddress{ + return &NetAddress{ IP: ip, Port: port, } - return na } // Equals reports whether na and other are the same addresses, diff --git a/p2p/netaddress_test.go b/p2p/netaddress_test.go index 6c1930a2f..653b436a6 100644 --- a/p2p/netaddress_test.go +++ b/p2p/netaddress_test.go @@ -9,20 +9,18 @@ import ( ) func TestNewNetAddress(t *testing.T) { - assert, require := assert.New(t), require.New(t) - tcpAddr, err := net.ResolveTCPAddr("tcp", "127.0.0.1:8080") - require.Nil(err) + require.Nil(t, err) addr := NewNetAddress("", tcpAddr) - assert.Equal("127.0.0.1:8080", addr.String()) + assert.Equal(t, "127.0.0.1:8080", addr.String()) - assert.NotPanics(func() { + assert.NotPanics(t, func() { NewNetAddress("", &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: 8000}) }, "Calling NewNetAddress with UDPAddr should not panic in testing") } -func TestNewNetAddressString(t *testing.T) { +func TestNewNetAddressStringWithOptionalID(t *testing.T) { testCases := []struct { addr string expected string @@ -57,6 +55,28 @@ func TestNewNetAddressString(t *testing.T) { {" @ ", "", false}, } + for _, tc := range testCases { + addr, err := NewNetAddressStringWithOptionalID(tc.addr) + if tc.correct { + if assert.Nil(t, err, tc.addr) { + assert.Equal(t, tc.expected, addr.String()) + } + } else { + assert.NotNil(t, err, tc.addr) + } + } +} + +func TestNewNetAddressString(t *testing.T) { + testCases := []struct { + addr string + expected string + correct bool + }{ + {"127.0.0.1:8080", "127.0.0.1:8080", false}, + {"deadbeefdeadbeefdeadbeefdeadbeefdeadbeef@127.0.0.1:8080", "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef@127.0.0.1:8080", true}, + } + for _, tc := range testCases { addr, err := NewNetAddressString(tc.addr) if tc.correct { @@ -70,23 +90,22 @@ func TestNewNetAddressString(t *testing.T) { } func TestNewNetAddressStrings(t *testing.T) { - addrs, errs := NewNetAddressStrings([]string{"127.0.0.1:8080", "127.0.0.2:8080"}) - assert.Len(t, errs, 0) + addrs, errs := NewNetAddressStrings([]string{ + "127.0.0.1:8080", + "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef@127.0.0.1:8080", + "deadbeefdeadbeefdeadbeefdeadbeefdeadbeed@127.0.0.2:8080"}) + assert.Len(t, errs, 1) assert.Equal(t, 2, len(addrs)) } func TestNewNetAddressIPPort(t *testing.T) { - assert := assert.New(t) addr := NewNetAddressIPPort(net.ParseIP("127.0.0.1"), 8080) - - assert.Equal("127.0.0.1:8080", addr.String()) + assert.Equal(t, "127.0.0.1:8080", addr.String()) } func TestNetAddressProperties(t *testing.T) { - assert, require := assert.New(t), require.New(t) - // TODO add more test cases - tests := []struct { + testCases := []struct { addr string valid bool local bool @@ -96,21 +115,19 @@ func TestNetAddressProperties(t *testing.T) { {"ya.ru:80", true, false, true}, } - for _, t := range tests { - addr, err := NewNetAddressString(t.addr) - require.Nil(err) + for _, tc := range testCases { + addr, err := NewNetAddressStringWithOptionalID(tc.addr) + require.Nil(t, err) - assert.Equal(t.valid, addr.Valid()) - assert.Equal(t.local, addr.Local()) - assert.Equal(t.routable, addr.Routable()) + assert.Equal(t, tc.valid, addr.Valid()) + assert.Equal(t, tc.local, addr.Local()) + assert.Equal(t, tc.routable, addr.Routable()) } } func TestNetAddressReachabilityTo(t *testing.T) { - assert, require := assert.New(t), require.New(t) - // TODO add more test cases - tests := []struct { + testCases := []struct { addr string other string reachability int @@ -119,13 +136,13 @@ func TestNetAddressReachabilityTo(t *testing.T) { {"ya.ru:80", "127.0.0.1:8080", 1}, } - for _, t := range tests { - addr, err := NewNetAddressString(t.addr) - require.Nil(err) + for _, tc := range testCases { + addr, err := NewNetAddressStringWithOptionalID(tc.addr) + require.Nil(t, err) - other, err := NewNetAddressString(t.other) - require.Nil(err) + other, err := NewNetAddressStringWithOptionalID(tc.other) + require.Nil(t, err) - assert.Equal(t.reachability, addr.ReachabilityTo(other)) + assert.Equal(t, tc.reachability, addr.ReachabilityTo(other)) } } From 8e699c2bfdeed06a9994e6ca8568457789132a7c Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Sat, 24 Mar 2018 17:04:32 +0100 Subject: [PATCH 044/143] defaultSeedDisconnectWaitPeriod should be at least as long as we expect it to take for a peer to become MarkGood Refs #1130 --- p2p/pex/pex_reactor.go | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/p2p/pex/pex_reactor.go b/p2p/pex/pex_reactor.go index 45f689c05..b6378fd0c 100644 --- a/p2p/pex/pex_reactor.go +++ b/p2p/pex/pex_reactor.go @@ -30,15 +30,17 @@ const ( defaultMinNumOutboundPeers = 10 // Seed/Crawler constants - // TODO: - // We want seeds to only advertise good peers. - // Peers are marked by external mechanisms. - // We need a config value that can be set to be - // on the order of how long it would take before a good - // peer is marked good. - defaultSeedDisconnectWaitPeriod = 2 * time.Minute // disconnect after this - defaultCrawlPeerInterval = 2 * time.Minute // dont redial for this. TODO: back-off - defaultCrawlPeersPeriod = 30 * time.Second // check some peers every this + + // We want seeds to only advertise good peers. Therefore they should wait at + // least as long as we expect it to take for a peer to become good before + // disconnecting. + // see consensus/reactor.go: blocksToContributeToBecomeGoodPeer + // 10000 blocks assuming 1s blocks ~ 2.7 hours. + defaultSeedDisconnectWaitPeriod = 3 * time.Hour + + defaultCrawlPeerInterval = 2 * time.Minute // don't redial for this. TODO: back-off. what for? + + defaultCrawlPeersPeriod = 30 * time.Second // check some peers every this maxAttemptsToDial = 16 // ~ 35h in total (last attempt - 18h) ) @@ -578,8 +580,7 @@ func (r *PEXReactor) crawlPeers() { // attemptDisconnects checks if we've been with each peer long enough to disconnect func (r *PEXReactor) attemptDisconnects() { for _, peer := range r.Switch.Peers().List() { - status := peer.Status() - if status.Duration < defaultSeedDisconnectWaitPeriod { + if peer.Status().Duration < defaultSeedDisconnectWaitPeriod { continue } if peer.IsPersistent() { From 1585152341cabf966381dcb5cd984e3a1c579633 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Sat, 24 Mar 2018 17:04:59 +0100 Subject: [PATCH 045/143] https://github.com/tendermint/tendermint/pull/1128#discussion_r162799294 Refs #1130 --- p2p/pex/pex_reactor.go | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/p2p/pex/pex_reactor.go b/p2p/pex/pex_reactor.go index b6378fd0c..c56181de3 100644 --- a/p2p/pex/pex_reactor.go +++ b/p2p/pex/pex_reactor.go @@ -564,16 +564,9 @@ func (r *PEXReactor) crawlPeers() { r.book.MarkAttempt(pi.Addr) continue } - } - // Crawl the connected peers asking for more addresses - for _, pi := range peerInfos { - // We will wait a minimum period of time before crawling peers again - if now.Sub(pi.LastAttempt) >= defaultCrawlPeerInterval { - peer := r.Switch.Peers().Get(pi.Addr.ID) - if peer != nil { - r.RequestAddrs(peer) - } - } + // Ask for more addresses + peer := r.Switch.Peers().Get(pi.Addr.ID) + r.RequestAddrs(peer) } } From cee7b5cb5411b3ee94f82d7d0be5fde42df5bf36 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Sat, 24 Mar 2018 17:49:42 +0100 Subject: [PATCH 046/143] GetSelectionWithBias Refs #1130 --- CHANGELOG.md | 1 + p2p/pex/addrbook.go | 124 ++++++++++++++++++++++++++++++++++++--- p2p/pex/addrbook_test.go | 109 ++++++++++++++++++++++++++++++++++ p2p/pex/pex_reactor.go | 8 ++- 4 files changed, 231 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf01bdbb1..d88f3ff9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ BUG FIXES: - [rpc] fix subscribing using an abci.ResponseDeliverTx tag IMPROVEMENTS: +- [p2p] seeds respond with a bias towards good peers - [rpc] `/tx` and `/tx_search` responses now include the transaction hash - [rpc] include validator power in `/status` diff --git a/p2p/pex/addrbook.go b/p2p/pex/addrbook.go index 3a3920486..efb48f6da 100644 --- a/p2p/pex/addrbook.go +++ b/p2p/pex/addrbook.go @@ -42,15 +42,19 @@ type AddrBook interface { NeedMoreAddrs() bool // Pick an address to dial - PickAddress(newBias int) *p2p.NetAddress + PickAddress(biasTowardsNewAddrs int) *p2p.NetAddress // Mark address MarkGood(*p2p.NetAddress) MarkAttempt(*p2p.NetAddress) MarkBad(*p2p.NetAddress) + IsGood(*p2p.NetAddress) bool + // Send a selection of addresses to peers GetSelection() []*p2p.NetAddress + // Send a selection of addresses with bias + GetSelectionWithBias(biasTowardsNewAddrs int) []*p2p.NetAddress // TODO: remove ListOfKnownAddresses() []*knownAddress @@ -173,6 +177,14 @@ func (a *addrBook) RemoveAddress(addr *p2p.NetAddress) { a.removeFromAllBuckets(ka) } +// IsGood returns true if peer was ever marked as good and haven't +// done anything wrong since then. +func (a *addrBook) IsGood(addr *p2p.NetAddress) bool { + a.mtx.Lock() + defer a.mtx.Unlock() + return a.addrLookup[addr.ID].isOld() +} + // NeedMoreAddrs implements AddrBook - returns true if there are not have enough addresses in the book. func (a *addrBook) NeedMoreAddrs() bool { return a.Size() < needAddressThreshold @@ -180,27 +192,27 @@ func (a *addrBook) NeedMoreAddrs() bool { // PickAddress implements AddrBook. It picks an address to connect to. // The address is picked randomly from an old or new bucket according -// to the newBias argument, which must be between [0, 100] (or else is truncated to that range) +// to the biasTowardsNewAddrs argument, which must be between [0, 100] (or else is truncated to that range) // and determines how biased we are to pick an address from a new bucket. // PickAddress returns nil if the AddrBook is empty or if we try to pick // from an empty bucket. -func (a *addrBook) PickAddress(newBias int) *p2p.NetAddress { +func (a *addrBook) PickAddress(biasTowardsNewAddrs int) *p2p.NetAddress { a.mtx.Lock() defer a.mtx.Unlock() if a.size() == 0 { return nil } - if newBias > 100 { - newBias = 100 + if biasTowardsNewAddrs > 100 { + biasTowardsNewAddrs = 100 } - if newBias < 0 { - newBias = 0 + if biasTowardsNewAddrs < 0 { + biasTowardsNewAddrs = 0 } // Bias between new and old addresses. - oldCorrelation := math.Sqrt(float64(a.nOld)) * (100.0 - float64(newBias)) - newCorrelation := math.Sqrt(float64(a.nNew)) * float64(newBias) + oldCorrelation := math.Sqrt(float64(a.nOld)) * (100.0 - float64(biasTowardsNewAddrs)) + newCorrelation := math.Sqrt(float64(a.nNew)) * float64(biasTowardsNewAddrs) // pick a random peer from a random bucket var bucket map[string]*knownAddress @@ -295,6 +307,100 @@ func (a *addrBook) GetSelection() []*p2p.NetAddress { return allAddr[:numAddresses] } +// GetSelectionWithBias implements AddrBook. +// It randomly selects some addresses (old & new). Suitable for peer-exchange protocols. +// +// Each address is picked randomly from an old or new bucket according to the +// biasTowardsNewAddrs argument, which must be between [0, 100] (or else is truncated to +// that range) and determines how biased we are to pick an address from a new +// bucket. +func (a *addrBook) GetSelectionWithBias(biasTowardsNewAddrs int) []*p2p.NetAddress { + a.mtx.Lock() + defer a.mtx.Unlock() + + if a.size() == 0 { + return nil + } + + if biasTowardsNewAddrs > 100 { + biasTowardsNewAddrs = 100 + } + if biasTowardsNewAddrs < 0 { + biasTowardsNewAddrs = 0 + } + + numAddresses := cmn.MaxInt( + cmn.MinInt(minGetSelection, a.size()), + a.size()*getSelectionPercent/100) + numAddresses = cmn.MinInt(maxGetSelection, numAddresses) + + selection := make([]*p2p.NetAddress, numAddresses) + + oldBucketToAddrsMap := make(map[int]map[string]struct{}) + var oldIndex int + newBucketToAddrsMap := make(map[int]map[string]struct{}) + var newIndex int + + selectionIndex := 0 +ADDRS_LOOP: + for selectionIndex < numAddresses { + pickFromOldBucket := int((float64(selectionIndex)/float64(numAddresses))*100) >= biasTowardsNewAddrs + pickFromOldBucket = (pickFromOldBucket && a.nOld > 0) || a.nNew == 0 + bucket := make(map[string]*knownAddress) + + // loop until we pick a random non-empty bucket + for len(bucket) == 0 { + if pickFromOldBucket { + oldIndex = a.rand.Intn(len(a.bucketsOld)) + bucket = a.bucketsOld[oldIndex] + } else { + newIndex = a.rand.Intn(len(a.bucketsNew)) + bucket = a.bucketsNew[newIndex] + } + } + + // pick a random index + randIndex := a.rand.Intn(len(bucket)) + + // loop over the map to return that index + var selectedAddr *p2p.NetAddress + for _, ka := range bucket { + if randIndex == 0 { + selectedAddr = ka.Addr + break + } + randIndex-- + } + + // if we have selected the address before, restart the loop + // otherwise, record it and continue + if pickFromOldBucket { + if addrsMap, ok := oldBucketToAddrsMap[oldIndex]; ok { + if _, ok = addrsMap[selectedAddr.String()]; ok { + continue ADDRS_LOOP + } + } else { + oldBucketToAddrsMap[oldIndex] = make(map[string]struct{}) + } + oldBucketToAddrsMap[oldIndex][selectedAddr.String()] = struct{}{} + } else { + if addrsMap, ok := newBucketToAddrsMap[newIndex]; ok { + if _, ok = addrsMap[selectedAddr.String()]; ok { + continue ADDRS_LOOP + } + } else { + newBucketToAddrsMap[newIndex] = make(map[string]struct{}) + } + newBucketToAddrsMap[newIndex][selectedAddr.String()] = struct{}{} + } + + selection[selectionIndex] = selectedAddr + selectionIndex++ + } + + return selection +} + // ListOfKnownAddresses returns the new and old addresses. func (a *addrBook) ListOfKnownAddresses() []*knownAddress { a.mtx.Lock() diff --git a/p2p/pex/addrbook_test.go b/p2p/pex/addrbook_test.go index 4a8df7166..68edb188a 100644 --- a/p2p/pex/addrbook_test.go +++ b/p2p/pex/addrbook_test.go @@ -157,6 +157,13 @@ func TestAddrBookPromoteToOld(t *testing.T) { t.Errorf("selection could not be bigger than the book") } + selection = book.GetSelectionWithBias(30) + t.Logf("selection: %v", selection) + + if len(selection) > book.Size() { + t.Errorf("selection with bias could not be bigger than the book") + } + assert.Equal(t, book.Size(), 100, "expecting book size to be 100") } @@ -229,3 +236,105 @@ func TestAddrBookRemoveAddress(t *testing.T) { book.RemoveAddress(nonExistingAddr) assert.Equal(t, 0, book.Size()) } + +func TestAddrBookGetSelection(t *testing.T) { + fname := createTempFileName("addrbook_test") + defer deleteTempFile(fname) + + book := NewAddrBook(fname, true) + book.SetLogger(log.TestingLogger()) + + // 1) empty book + assert.Empty(t, book.GetSelection()) + + // 2) add one address + addr := randIPv4Address(t) + book.AddAddress(addr, addr) + + assert.Equal(t, 1, len(book.GetSelection())) + assert.Equal(t, addr, book.GetSelection()[0]) + + // 3) add a bunch of addresses + randAddrs := randNetAddressPairs(t, 100) + for _, addrSrc := range randAddrs { + book.AddAddress(addrSrc.addr, addrSrc.src) + } + + // check there is no duplicates + addrs := make(map[string]*p2p.NetAddress) + selection := book.GetSelection() + for _, addr := range selection { + if dup, ok := addrs[addr.String()]; ok { + t.Fatalf("selection %v contains duplicates %v", selection, dup) + } + addrs[addr.String()] = addr + } + + if len(selection) > book.Size() { + t.Errorf("selection %v could not be bigger than the book", selection) + } +} + +func TestAddrBookGetSelectionWithBias(t *testing.T) { + const biasTowardsNewAddrs = 30 + + fname := createTempFileName("addrbook_test") + defer deleteTempFile(fname) + + book := NewAddrBook(fname, true) + book.SetLogger(log.TestingLogger()) + + // 1) empty book + selection := book.GetSelectionWithBias(biasTowardsNewAddrs) + assert.Empty(t, selection) + + // 2) add one address + addr := randIPv4Address(t) + book.AddAddress(addr, addr) + + selection = book.GetSelectionWithBias(biasTowardsNewAddrs) + assert.Equal(t, 1, len(selection)) + assert.Equal(t, addr, selection[0]) + + // 3) add a bunch of addresses + randAddrs := randNetAddressPairs(t, 100) + for _, addrSrc := range randAddrs { + book.AddAddress(addrSrc.addr, addrSrc.src) + } + + // check there is no duplicates + addrs := make(map[string]*p2p.NetAddress) + selection = book.GetSelectionWithBias(biasTowardsNewAddrs) + for _, addr := range selection { + if dup, ok := addrs[addr.String()]; ok { + t.Fatalf("selection %v contains duplicates %v", selection, dup) + } + addrs[addr.String()] = addr + } + + if len(selection) > book.Size() { + t.Fatalf("selection %v could not be bigger than the book", selection) + } + + // 4) mark 80% of the addresses as good + randAddrsLen := len(randAddrs) + for i, addrSrc := range randAddrs { + if int((float64(i)/float64(randAddrsLen))*100) >= 20 { + book.MarkGood(addrSrc.addr) + } + } + + selection = book.GetSelectionWithBias(biasTowardsNewAddrs) + + // check that ~70% of addresses returned are good + good := 0 + for _, addr := range selection { + if book.IsGood(addr) { + good++ + } + } + got, expected := int((float64(good)/float64(len(selection)))*100), (100 - biasTowardsNewAddrs) + if got >= expected { + t.Fatalf("expected more good peers (%% got: %d, %% expected: %d, number of good addrs: %d, total: %d)", got, expected, good, len(selection)) + } +} diff --git a/p2p/pex/pex_reactor.go b/p2p/pex/pex_reactor.go index c56181de3..8c6ad5a8b 100644 --- a/p2p/pex/pex_reactor.go +++ b/p2p/pex/pex_reactor.go @@ -43,6 +43,11 @@ const ( defaultCrawlPeersPeriod = 30 * time.Second // check some peers every this maxAttemptsToDial = 16 // ~ 35h in total (last attempt - 18h) + + // if node connects to seed, it does not have any trusted peers. + // Especially in the beginning, node should have more trusted peers than + // untrusted. + biasToSelectNewPeers = 30 // 70 to select good peers ) // PEXReactor handles PEX (peer exchange) and ensures that an @@ -191,8 +196,7 @@ func (r *PEXReactor) Receive(chID byte, src Peer, msgBytes []byte) { // Seeds disconnect after sending a batch of addrs if r.config.SeedMode { - // TODO: should we be more selective ? - r.SendAddrs(src, r.book.GetSelection()) + r.SendAddrs(src, r.book.GetSelectionWithBias(biasToSelectNewPeers)) r.Switch.StopPeerGracefully(src) } else { r.SendAddrs(src, r.book.GetSelection()) From 1b9323f1056cbbf5fc5083cc8e321ceec2e7d6bf Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Thu, 5 Apr 2018 05:17:43 -0700 Subject: [PATCH 047/143] Fix blockchain tests --- Gopkg.lock | 10 +++++----- Gopkg.toml | 4 ++-- blockchain/reactor_test.go | 8 ++++++-- blockchain/store.go | 32 +++++++++++++++++++++++++------- blockchain/store_test.go | 9 ++++++--- blockchain/wire.go | 2 ++ types/block.go | 6 +++--- 7 files changed, 49 insertions(+), 22 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 7119588b1..19a15cc5d 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -254,8 +254,8 @@ [[projects]] name = "github.com/tendermint/go-amino" packages = ["."] - revision = "f55c3351f30e5987500020631f00e87a035ed415" - version = "0.9.3" + revision = "26718ab6738f938d4b33d593543cee7681f2a6a6" + version = "0.9.5" [[projects]] name = "github.com/tendermint/go-crypto" @@ -288,8 +288,8 @@ "pubsub/query", "test" ] - revision = "e9cf47606cfcbdc28a7c16671b4a70b459e9d4cc" - version = "v0.8.0-dev" + revision = "2e24b64fc121dcdf1cabceab8dc2f7257675483c" + version = "0.8.1" [[projects]] branch = "master" @@ -386,6 +386,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "9aace7a292e5402f97327860531e13b77f036c19843b698566ee958773404f48" + inputs-digest = "8cd32c0a5faec4d8cbca3d9ea5d97b0e4e90bfc2a9f0e6b71a4846cb6030be94" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index aa523e119..73f73bebc 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -79,11 +79,11 @@ [[constraint]] name = "github.com/tendermint/go-amino" - version = "0.9.3" + version = "0.9.5" [[constraint]] name = "github.com/tendermint/tmlibs" - version = "0.8.0-dev" + version = "0.8.1" [[constraint]] name = "google.golang.org/grpc" diff --git a/blockchain/reactor_test.go b/blockchain/reactor_test.go index d72ff8c30..63e3c72bb 100644 --- a/blockchain/reactor_test.go +++ b/blockchain/reactor_test.go @@ -16,8 +16,12 @@ import ( func makeStateAndBlockStore(logger log.Logger) (sm.State, *BlockStore) { config := cfg.ResetTestRoot("blockchain_reactor_test") - blockStore := NewBlockStore(dbm.NewMemDB()) - state, err := sm.LoadStateFromDBOrGenesisFile(dbm.NewMemDB(), config.GenesisFile()) + // blockDB := dbm.NewDebugDB("blockDB", dbm.NewMemDB()) + // stateDB := dbm.NewDebugDB("stateDB", dbm.NewMemDB()) + blockDB := dbm.NewMemDB() + stateDB := dbm.NewMemDB() + blockStore := NewBlockStore(blockDB) + state, err := sm.LoadStateFromDBOrGenesisFile(stateDB, config.GenesisFile()) if err != nil { panic(cmn.ErrorWrap(err, "error constructing state from genesis file")) } diff --git a/blockchain/store.go b/blockchain/store.go index f699c1c7c..8210f1439 100644 --- a/blockchain/store.go +++ b/blockchain/store.go @@ -53,14 +53,17 @@ func (bs *BlockStore) Height() int64 { // LoadBlock returns the block with the given height. // If no block is found for that height, it returns nil. func (bs *BlockStore) LoadBlock(height int64) *types.Block { - var blockMeta *types.BlockMeta + var blockMeta = new(types.BlockMeta) bz := bs.db.Get(calcBlockMetaKey(height)) + if len(bz) == 0 { + return nil + } err := cdc.UnmarshalBinaryBare(bz, blockMeta) if err != nil { panic(cmn.ErrorWrap(err, "Error reading block meta")) } - var block *types.Block + var block = new(types.Block) buf := []byte{} for i := 0; i < blockMeta.BlockID.PartsHeader.Total; i++ { part := bs.LoadBlockPart(height, i) @@ -68,6 +71,8 @@ func (bs *BlockStore) LoadBlock(height int64) *types.Block { } err = cdc.UnmarshalBinaryBare(buf, block) if err != nil { + // NOTE: The existence of meta should imply the existence of the + // block. So, make sure meta is only saved after blocks are saved. panic(cmn.ErrorWrap(err, "Error reading block")) } return block @@ -77,8 +82,11 @@ func (bs *BlockStore) LoadBlock(height int64) *types.Block { // from the block at the given height. // If no part is found for the given height and index, it returns nil. func (bs *BlockStore) LoadBlockPart(height int64, index int) *types.Part { - var part *types.Part + var part = new(types.Part) bz := bs.db.Get(calcBlockPartKey(height, index)) + if len(bz) == 0 { + return nil + } err := cdc.UnmarshalBinaryBare(bz, part) if err != nil { panic(cmn.ErrorWrap(err, "Error reading block part")) @@ -89,8 +97,11 @@ func (bs *BlockStore) LoadBlockPart(height int64, index int) *types.Part { // LoadBlockMeta returns the BlockMeta for the given height. // If no block is found for the given height, it returns nil. func (bs *BlockStore) LoadBlockMeta(height int64) *types.BlockMeta { - var blockMeta *types.BlockMeta + var blockMeta = new(types.BlockMeta) bz := bs.db.Get(calcBlockMetaKey(height)) + if len(bz) == 0 { + return nil + } err := cdc.UnmarshalBinaryBare(bz, blockMeta) if err != nil { panic(cmn.ErrorWrap(err, "Error reading block meta")) @@ -103,8 +114,11 @@ func (bs *BlockStore) LoadBlockMeta(height int64) *types.BlockMeta { // and it comes from the block.LastCommit for `height+1`. // If no commit is found for the given height, it returns nil. func (bs *BlockStore) LoadBlockCommit(height int64) *types.Commit { - var commit *types.Commit + var commit = new(types.Commit) bz := bs.db.Get(calcBlockCommitKey(height)) + if len(bz) == 0 { + return nil + } err := cdc.UnmarshalBinaryBare(bz, commit) if err != nil { panic(cmn.ErrorWrap(err, "Error reading block commit")) @@ -116,8 +130,11 @@ func (bs *BlockStore) LoadBlockCommit(height int64) *types.Commit { // This is useful when we've seen a commit, but there has not yet been // a new block at `height + 1` that includes this commit in its block.LastCommit. func (bs *BlockStore) LoadSeenCommit(height int64) *types.Commit { - var commit *types.Commit + var commit = new(types.Commit) bz := bs.db.Get(calcSeenCommitKey(height)) + if len(bz) == 0 { + return nil + } err := cdc.UnmarshalBinaryBare(bz, commit) if err != nil { panic(cmn.ErrorWrap(err, "Error reading block commit")) @@ -150,7 +167,8 @@ func (bs *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, s // Save block parts for i := 0; i < blockParts.Total(); i++ { - bs.saveBlockPart(height, i, blockParts.GetPart(i)) + part := blockParts.GetPart(i) + bs.saveBlockPart(height, i, part) } // Save block commit (duplicate and separate from the Block) diff --git a/blockchain/store_test.go b/blockchain/store_test.go index ac0f1a430..b2188043e 100644 --- a/blockchain/store_test.go +++ b/blockchain/store_test.go @@ -153,14 +153,14 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) { parts: validPartSet, seenCommit: seenCommit1, corruptCommitInDB: true, // Corrupt the DB's commit entry - wantPanic: "rror reading commit", + wantPanic: "Error reading block commit", }, { block: newBlock(&header1, commitAtH10), parts: validPartSet, seenCommit: seenCommit1, - wantPanic: "rror reading block", + wantPanic: "Error reading block", corruptBlockInDB: true, // Corrupt the DB's block entry }, @@ -179,7 +179,7 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) { seenCommit: seenCommit1, corruptSeenCommitInDB: true, - wantPanic: "rror reading commit", + wantPanic: "Error reading block commit", }, { @@ -341,6 +341,9 @@ func TestBlockFetchAtHeight(t *testing.T) { require.Equal(t, bs.Height(), block.Header.Height, "expecting the new height to be changed") blockAtHeight := bs.LoadBlock(bs.Height()) + bz1 := cdc.MustMarshalBinaryBare(block) + bz2 := cdc.MustMarshalBinaryBare(blockAtHeight) + require.Equal(t, bz1, bz2) require.Equal(t, block.Hash(), blockAtHeight.Hash(), "expecting a successful load of the last saved block") diff --git a/blockchain/wire.go b/blockchain/wire.go index 72431db82..55b4e60ae 100644 --- a/blockchain/wire.go +++ b/blockchain/wire.go @@ -2,10 +2,12 @@ package blockchain import ( "github.com/tendermint/go-amino" + "github.com/tendermint/go-crypto" ) var cdc = amino.NewCodec() func init() { RegisterBlockchainMessages(cdc) + crypto.RegisterAmino(cdc) } diff --git a/types/block.go b/types/block.go index 44e616e68..1a9b47803 100644 --- a/types/block.go +++ b/types/block.go @@ -94,7 +94,7 @@ func (b *Block) Hash() cmn.HexBytes { // MakePartSet returns a PartSet containing parts of a serialized block. // This is the form in which the block is gossipped to peers. func (b *Block) MakePartSet(partSize int) *PartSet { - bz, err := cdc.MarshalBinary(b) + bz, err := cdc.MarshalBinaryBare(b) if err != nil { panic(err) } @@ -498,7 +498,7 @@ func (blockID BlockID) Equals(other BlockID) bool { // Key returns a machine-readable string representation of the BlockID func (blockID BlockID) Key() string { - bz, err := cdc.MarshalBinary(blockID.PartsHeader) + bz, err := cdc.MarshalBinaryBare(blockID.PartsHeader) if err != nil { panic(err) } @@ -518,7 +518,7 @@ type hasher struct { func (h hasher) Hash() []byte { hasher := ripemd160.New() - if h.item != nil && !cmn.IsTypedNil(h.item) { + if h.item != nil && !cmn.IsTypedNil(h.item) && !cmn.IsEmpty(h.item) { bz, err := cdc.MarshalBinaryBare(h.item) if err != nil { panic(err) From 5d1c758730f7e2b86f3453a95b7bd0b2900c0ff2 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Thu, 5 Apr 2018 05:43:23 -0700 Subject: [PATCH 048/143] Fix evidence --- evidence/reactor.go | 34 +++++++++++++--------------------- evidence/store.go | 20 ++++++++++++++------ evidence/store_test.go | 13 ------------- evidence/wire.go | 25 +++++++++++++++++++++++++ p2p/pex/pex_reactor_test.go | 2 +- types/evidence.go | 4 ++-- 6 files changed, 55 insertions(+), 43 deletions(-) create mode 100644 evidence/wire.go diff --git a/evidence/reactor.go b/evidence/reactor.go index 6647db969..aeab4d831 100644 --- a/evidence/reactor.go +++ b/evidence/reactor.go @@ -1,14 +1,12 @@ package evidence import ( - "bytes" "fmt" + "github.com/tendermint/go-amino" + "github.com/tendermint/tmlibs/log" "reflect" "time" - wire "github.com/tendermint/go-wire" - "github.com/tendermint/tmlibs/log" - "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/types" ) @@ -68,7 +66,7 @@ func (evR *EvidenceReactor) AddPeer(peer p2p.Peer) { // the rest will be sent by the broadcastRoutine evidences := evR.evpool.PriorityEvidence() msg := &EvidenceListMessage{evidences} - success := peer.Send(EvidenceChannel, struct{ EvidenceMessage }{msg}) + success := peer.Send(EvidenceChannel, cdc.MustMarshalBinaryBare(msg)) if !success { // TODO: remove peer ? } @@ -82,7 +80,7 @@ func (evR *EvidenceReactor) RemovePeer(peer p2p.Peer, reason interface{}) { // Receive implements Reactor. // It adds any received evidence to the evpool. func (evR *EvidenceReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) { - _, msg, err := DecodeMessage(msgBytes) + msg, err := DecodeMessage(msgBytes) if err != nil { evR.Logger.Error("Error decoding message", "src", src, "chId", chID, "msg", msg, "err", err, "bytes", msgBytes) evR.Switch.StopPeerForError(src, err) @@ -119,7 +117,7 @@ func (evR *EvidenceReactor) broadcastRoutine() { case evidence := <-evR.evpool.EvidenceChan(): // broadcast some new evidence msg := &EvidenceListMessage{[]types.Evidence{evidence}} - evR.Switch.Broadcast(EvidenceChannel, struct{ EvidenceMessage }{msg}) + evR.Switch.Broadcast(EvidenceChannel, cdc.MustMarshalBinaryBare(msg)) // TODO: Broadcast runs asynchronously, so this should wait on the successChan // in another routine before marking to be proper. @@ -127,7 +125,7 @@ func (evR *EvidenceReactor) broadcastRoutine() { case <-ticker.C: // broadcast all pending evidence msg := &EvidenceListMessage{evR.evpool.PendingEvidence()} - evR.Switch.Broadcast(EvidenceChannel, struct{ EvidenceMessage }{msg}) + evR.Switch.Broadcast(EvidenceChannel, cdc.MustMarshalBinaryBare(msg)) case <-evR.Quit(): return } @@ -137,24 +135,18 @@ func (evR *EvidenceReactor) broadcastRoutine() { //----------------------------------------------------------------------------- // Messages -const ( - msgTypeEvidence = byte(0x01) -) - // EvidenceMessage is a message sent or received by the EvidenceReactor. type EvidenceMessage interface{} -var _ = wire.RegisterInterface( - struct{ EvidenceMessage }{}, - wire.ConcreteType{&EvidenceListMessage{}, msgTypeEvidence}, -) +func RegisterEvidenceMessages(cdc *amino.Codec) { + cdc.RegisterInterface((*EvidenceMessage)(nil), nil) + cdc.RegisterConcrete(&EvidenceListMessage{}, + "tendermint/evidence/EvidenceListMessagE", nil) +} // DecodeMessage decodes a byte-array into a EvidenceMessage. -func DecodeMessage(bz []byte) (msgType byte, msg EvidenceMessage, err error) { - msgType = bz[0] - n := new(int) - r := bytes.NewReader(bz) - msg = wire.ReadBinary(struct{ EvidenceMessage }{}, r, maxEvidenceMessageSize, n, &err).(struct{ EvidenceMessage }).EvidenceMessage +func DecodeMessage(bz []byte) (msg EvidenceMessage, err error) { + err = cdc.UnmarshalBinaryBare(bz, &msg) return } diff --git a/evidence/store.go b/evidence/store.go index 7c8becd02..081715e3c 100644 --- a/evidence/store.go +++ b/evidence/store.go @@ -3,7 +3,6 @@ package evidence import ( "fmt" - wire "github.com/tendermint/go-wire" "github.com/tendermint/tendermint/types" dbm "github.com/tendermint/tmlibs/db" ) @@ -104,7 +103,10 @@ func (store *EvidenceStore) ListEvidence(prefixKey string) (evidence []types.Evi val := iter.Value() var ei EvidenceInfo - wire.ReadBinaryBytes(val, &ei) + err := cdc.UnmarshalBinaryBare(val, &ei) + if err != nil { + panic(err) + } evidence = append(evidence, ei.Evidence) } return evidence @@ -119,7 +121,10 @@ func (store *EvidenceStore) GetEvidence(height int64, hash []byte) *EvidenceInfo return nil } var ei EvidenceInfo - wire.ReadBinaryBytes(val, &ei) + err := cdc.UnmarshalBinaryBare(val, &ei) + if err != nil { + panic(err) + } return &ei } @@ -137,7 +142,7 @@ func (store *EvidenceStore) AddNewEvidence(evidence types.Evidence, priority int Priority: priority, Evidence: evidence, } - eiBytes := wire.BinaryBytes(ei) + eiBytes := cdc.MustMarshalBinaryBare(ei) // add it to the store key := keyOutqueue(evidence, priority) @@ -171,7 +176,7 @@ func (store *EvidenceStore) MarkEvidenceAsCommitted(evidence types.Evidence) { ei.Committed = true lookupKey := keyLookup(evidence) - store.db.SetSync(lookupKey, wire.BinaryBytes(ei)) + store.db.SetSync(lookupKey, cdc.MustMarshalBinaryBare(ei)) } //--------------------------------------------------- @@ -181,6 +186,9 @@ func (store *EvidenceStore) getEvidenceInfo(evidence types.Evidence) EvidenceInf key := keyLookup(evidence) var ei EvidenceInfo b := store.db.Get(key) - wire.ReadBinaryBytes(b, &ei) + err := cdc.UnmarshalBinaryBare(b, &ei) + if err != nil { + panic(err) + } return ei } diff --git a/evidence/store_test.go b/evidence/store_test.go index 180bee58f..3fdb3ba6e 100644 --- a/evidence/store_test.go +++ b/evidence/store_test.go @@ -4,7 +4,6 @@ import ( "testing" "github.com/stretchr/testify/assert" - wire "github.com/tendermint/go-wire" "github.com/tendermint/tendermint/types" dbm "github.com/tendermint/tmlibs/db" ) @@ -108,15 +107,3 @@ func TestStorePriority(t *testing.T) { assert.Equal(ev, cases[i].ev) } } - -//------------------------------------------- -const ( - evidenceTypeMockGood = byte(0x01) - evidenceTypeMockBad = byte(0x02) -) - -var _ = wire.RegisterInterface( - struct{ types.Evidence }{}, - wire.ConcreteType{types.MockGoodEvidence{}, evidenceTypeMockGood}, - wire.ConcreteType{types.MockBadEvidence{}, evidenceTypeMockBad}, -) diff --git a/evidence/wire.go b/evidence/wire.go new file mode 100644 index 000000000..842e0707a --- /dev/null +++ b/evidence/wire.go @@ -0,0 +1,25 @@ +package evidence + +import ( + "github.com/tendermint/go-amino" + "github.com/tendermint/go-crypto" + "github.com/tendermint/tendermint/types" +) + +var cdc = amino.NewCodec() + +func init() { + RegisterEvidenceMessages(cdc) + crypto.RegisterAmino(cdc) + types.RegisterEvidences(cdc) + RegisterMockEvidences(cdc) // For testing +} + +//------------------------------------------- + +func RegisterMockEvidences(cdc *amino.Codec) { + cdc.RegisterConcrete(types.MockGoodEvidence{}, + "tendermint/MockGoodEvidence", nil) + cdc.RegisterConcrete(types.MockBadEvidence{}, + "tendermint/MockBadEvidence", nil) +} diff --git a/p2p/pex/pex_reactor_test.go b/p2p/pex/pex_reactor_test.go index 00345758a..0938fce9b 100644 --- a/p2p/pex/pex_reactor_test.go +++ b/p2p/pex/pex_reactor_test.go @@ -238,7 +238,7 @@ func TestPEXReactorUsesSeedsIfNeeded(t *testing.T) { defer peer.Stop() // 3. check that the peer connects to seed immediately - assertPeersWithTimeout(t, []*p2p.Switch{peer}, 10*time.Millisecond, 1*time.Second, 1) + assertPeersWithTimeout(t, []*p2p.Switch{peer}, 10*time.Millisecond, 3*time.Second, 1) } func TestPEXReactorCrawlStatus(t *testing.T) { diff --git a/types/evidence.go b/types/evidence.go index 658c40227..371b3159f 100644 --- a/types/evidence.go +++ b/types/evidence.go @@ -38,9 +38,9 @@ type Evidence interface { String() string } -func RegisterEvidence(cdc *amino.Codec) { +func RegisterEvidences(cdc *amino.Codec) { cdc.RegisterInterface((*Evidence)(nil), nil) - cdc.RegisterConcrete(&DuplicateVoteEvidence{}, "tendermint/Evidence", nil) + cdc.RegisterConcrete(&DuplicateVoteEvidence{}, "tendermint/DuplicateVoteEvidence", nil) } //------------------------------------------- From 59ca9bf480b167ba3a17b4e6e3ed3f3b52b14a8a Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 5 Apr 2018 16:16:36 +0300 Subject: [PATCH 049/143] update to tmlibs v0.8.1 --- Gopkg.lock | 30 +++++++++++++++--------------- Gopkg.toml | 4 ++-- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 09dab7a5b..9461b319b 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -105,7 +105,7 @@ "json/scanner", "json/token" ] - revision = "f40e974e75af4e271d97ce0fc917af5898ae7bda" + revision = "ef8a98b0bbce4a65b5aa4c368430a80ddc533168" [[projects]] name = "github.com/inconshreveable/mousetrap" @@ -167,8 +167,8 @@ ".", "mem" ] - revision = "bb8f1927f2a9d3ab41c9340aa034f6b803f4359c" - version = "v1.0.2" + revision = "63644898a8da0bc22138abf860edaf5277b6102e" + version = "v1.1.0" [[projects]] name = "github.com/spf13/cast" @@ -179,8 +179,8 @@ [[projects]] name = "github.com/spf13/cobra" packages = ["."] - revision = "7b2c5ac9fc04fc5efafb60700713d4fa609b777b" - version = "v0.0.1" + revision = "a1f051bc3eba734da4772d60e2d677f47cf93ef4" + version = "v0.0.2" [[projects]] branch = "master" @@ -226,7 +226,7 @@ "leveldb/table", "leveldb/util" ] - revision = "169b1b37be738edb2813dab48c97a549bcf99bb5" + revision = "714f901b98fdb3aa954b4193d8cbd64a28d80cad" [[projects]] name = "github.com/tendermint/abci" @@ -268,7 +268,6 @@ version = "v0.7.3" [[projects]] - branch = "develop" name = "github.com/tendermint/tmlibs" packages = [ "autofile", @@ -284,7 +283,8 @@ "pubsub/query", "test" ] - revision = "0f2811441f4cf44b414df16ceae3c0931c74662e" + revision = "2e24b64fc121dcdf1cabceab8dc2f7257675483c" + version = "0.8.1" [[projects]] branch = "master" @@ -299,7 +299,7 @@ "ripemd160", "salsa20/salsa" ] - revision = "88942b9c40a4c9d203b82b3731787b672d6e809b" + revision = "b2aa35443fbc700ab74c586ae79b81c171851023" [[projects]] branch = "master" @@ -313,13 +313,13 @@ "lex/httplex", "trace" ] - revision = "6078986fec03a1dcc236c34816c71b0e05018fda" + revision = "b3c676e531a6dc479fa1b35ac961c13f5e2b4d2e" [[projects]] branch = "master" name = "golang.org/x/sys" packages = ["unix"] - revision = "91ee8cde435411ca3f1cd365e8f20131aed4d0a1" + revision = "1d206c9fa8975fb4cf00df1dc8bf3283dc24ba0e" [[projects]] name = "golang.org/x/text" @@ -346,7 +346,7 @@ branch = "master" name = "google.golang.org/genproto" packages = ["googleapis/rpc/status"] - revision = "f8c8703595236ae70fdf8789ecb656ea0bcdcf46" + revision = "35de2414665fc36f56b72d982c5af480d86de5ab" [[projects]] name = "google.golang.org/grpc" @@ -375,12 +375,12 @@ [[projects]] name = "gopkg.in/yaml.v2" packages = ["."] - revision = "7f97868eec74b32b0982dd158a51a446d1da7eb5" - version = "v2.1.1" + revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183" + version = "v2.2.1" [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "88f227f33e785669706f12e131610a86fd3c00f5dfd6f22c62d46e0688049726" + inputs-digest = "b7c02a311569ec5fe2197614444fb231ea60f3e65a11a20e318421f1752054d7" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 8de0de9c2..f6846793d 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -85,8 +85,8 @@ [[override]] # [[constraint]] name = "github.com/tendermint/tmlibs" - # version = "~0.7.1" - branch = "develop" + version = "~0.8.1" + # branch = "develop" [[constraint]] name = "google.golang.org/grpc" From 9a57ef9cbf671dbbeafa7aad268a44bade6117e6 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 21 Mar 2018 19:41:56 +0100 Subject: [PATCH 050/143] do not dial ourselves (ok, maybe just once) Refs #1275 --- p2p/node_info.go | 8 ++++++++ p2p/switch.go | 11 +++++++++++ 2 files changed, 19 insertions(+) diff --git a/p2p/node_info.go b/p2p/node_info.go index 6f44b49cd..46c9a1629 100644 --- a/p2p/node_info.go +++ b/p2p/node_info.go @@ -2,6 +2,7 @@ package p2p import ( "fmt" + "net" "strings" crypto "github.com/tendermint/go-crypto" @@ -107,10 +108,17 @@ OUTER_LOOP: return nil } +// ID returns node's ID. func (info NodeInfo) ID() ID { return PubKeyToID(info.PubKey) } +// IP returns node listen addr's IP. +func (info NodeInfo) IP() net.IP { + hostPort := strings.SplitN(info.ListenAddr, ":", 2) + return net.ParseIP(hostPort[0]) +} + // NetAddress returns a NetAddress derived from the NodeInfo - // it includes the authenticated peer ID and the self-reported // ListenAddr. Note that the ListenAddr is not authenticated and diff --git a/p2p/switch.go b/p2p/switch.go index fa037a9b8..f50cab802 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -58,6 +58,7 @@ type Switch struct { dialing *cmn.CMap nodeInfo NodeInfo // our node info nodeKey *NodeKey // our node privkey + nodeIP net.IP // our IP addrBook AddrBook filterConnByAddr func(net.Addr) error @@ -148,6 +149,7 @@ func (sw *Switch) IsListening() bool { // NOTE: Not goroutine safe. func (sw *Switch) SetNodeInfo(nodeInfo NodeInfo) { sw.nodeInfo = nodeInfo + sw.nodeIP = nodeInfo.IP() } // NodeInfo returns the switch's NodeInfo. @@ -381,6 +383,11 @@ func (sw *Switch) DialPeersAsync(addrBook AddrBook, peers []string, persistent b // DialPeerWithAddress dials the given peer and runs sw.addPeer if it connects and authenticates successfully. // If `persistent == true`, the switch will always try to reconnect to this peer if the connection ever fails. func (sw *Switch) DialPeerWithAddress(addr *NetAddress, persistent bool) error { + // do not dial ourselves + if addr.IP == sw.nodeIP { + return ErrSwitchConnectToSelf + } + sw.dialing.Set(string(addr.ID), addr) defer sw.dialing.Delete(string(addr.ID)) return sw.addOutboundPeerWithConfig(addr, sw.peerConfig, persistent) @@ -522,6 +529,10 @@ func (sw *Switch) addPeer(pc peerConn) error { // Avoid self if sw.nodeKey.ID() == peerID { + // overwrite current IP to avoid dialing ourselves again + // it means original nodeIP was different from public one + sw.nodeIP = peerNodeInfo.IP() + return ErrSwitchConnectToSelf } From 5a2fa71b033c576177060845c49b995af820668a Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 21 Mar 2018 20:04:22 +0100 Subject: [PATCH 051/143] use combination of IP and port, not just IP --- p2p/node_info.go | 7 ------- p2p/switch.go | 34 +++++++++++++++++----------------- 2 files changed, 17 insertions(+), 24 deletions(-) diff --git a/p2p/node_info.go b/p2p/node_info.go index 46c9a1629..346de37d3 100644 --- a/p2p/node_info.go +++ b/p2p/node_info.go @@ -2,7 +2,6 @@ package p2p import ( "fmt" - "net" "strings" crypto "github.com/tendermint/go-crypto" @@ -113,12 +112,6 @@ func (info NodeInfo) ID() ID { return PubKeyToID(info.PubKey) } -// IP returns node listen addr's IP. -func (info NodeInfo) IP() net.IP { - hostPort := strings.SplitN(info.ListenAddr, ":", 2) - return net.ParseIP(hostPort[0]) -} - // NetAddress returns a NetAddress derived from the NodeInfo - // it includes the authenticated peer ID and the self-reported // ListenAddr. Note that the ListenAddr is not authenticated and diff --git a/p2p/switch.go b/p2p/switch.go index f50cab802..d0e6df721 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -48,18 +48,18 @@ type AddrBook interface { type Switch struct { cmn.BaseService - config *cfg.P2PConfig - peerConfig *PeerConfig - listeners []Listener - reactors map[string]Reactor - chDescs []*conn.ChannelDescriptor - reactorsByCh map[byte]Reactor - peers *PeerSet - dialing *cmn.CMap - nodeInfo NodeInfo // our node info - nodeKey *NodeKey // our node privkey - nodeIP net.IP // our IP - addrBook AddrBook + config *cfg.P2PConfig + peerConfig *PeerConfig + listeners []Listener + reactors map[string]Reactor + chDescs []*conn.ChannelDescriptor + reactorsByCh map[byte]Reactor + peers *PeerSet + dialing *cmn.CMap + nodeInfo NodeInfo // our node info + nodeKey *NodeKey // our node privkey + nodePublicAddr *NetAddress + addrBook AddrBook filterConnByAddr func(net.Addr) error filterConnByID func(ID) error @@ -149,7 +149,7 @@ func (sw *Switch) IsListening() bool { // NOTE: Not goroutine safe. func (sw *Switch) SetNodeInfo(nodeInfo NodeInfo) { sw.nodeInfo = nodeInfo - sw.nodeIP = nodeInfo.IP() + sw.nodePublicAddr = nodeInfo.NetAddress() } // NodeInfo returns the switch's NodeInfo. @@ -384,7 +384,7 @@ func (sw *Switch) DialPeersAsync(addrBook AddrBook, peers []string, persistent b // If `persistent == true`, the switch will always try to reconnect to this peer if the connection ever fails. func (sw *Switch) DialPeerWithAddress(addr *NetAddress, persistent bool) error { // do not dial ourselves - if addr.IP == sw.nodeIP { + if addr.Same(sw.nodePublicAddr) { return ErrSwitchConnectToSelf } @@ -529,9 +529,9 @@ func (sw *Switch) addPeer(pc peerConn) error { // Avoid self if sw.nodeKey.ID() == peerID { - // overwrite current IP to avoid dialing ourselves again - // it means original nodeIP was different from public one - sw.nodeIP = peerNodeInfo.IP() + // overwrite current addr to avoid dialing ourselves again + // it means original nodePublicAddr was different from public one + sw.nodePublicAddr = peerNodeInfo.NetAddress() return ErrSwitchConnectToSelf } From 4b8e342309e25c801b6295313bb234c2a9c356d2 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 21 Mar 2018 20:08:43 +0100 Subject: [PATCH 052/143] fix panic: lookup testing on 10.0.2.3:53: no such host --- p2p/switch.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/p2p/switch.go b/p2p/switch.go index d0e6df721..fec448265 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -58,7 +58,7 @@ type Switch struct { dialing *cmn.CMap nodeInfo NodeInfo // our node info nodeKey *NodeKey // our node privkey - nodePublicAddr *NetAddress + nodePublicAddr string addrBook AddrBook filterConnByAddr func(net.Addr) error @@ -149,7 +149,7 @@ func (sw *Switch) IsListening() bool { // NOTE: Not goroutine safe. func (sw *Switch) SetNodeInfo(nodeInfo NodeInfo) { sw.nodeInfo = nodeInfo - sw.nodePublicAddr = nodeInfo.NetAddress() + sw.nodePublicAddr = nodeInfo.ListenAddr } // NodeInfo returns the switch's NodeInfo. @@ -384,7 +384,7 @@ func (sw *Switch) DialPeersAsync(addrBook AddrBook, peers []string, persistent b // If `persistent == true`, the switch will always try to reconnect to this peer if the connection ever fails. func (sw *Switch) DialPeerWithAddress(addr *NetAddress, persistent bool) error { // do not dial ourselves - if addr.Same(sw.nodePublicAddr) { + if addr.DialString() == sw.nodePublicAddr { return ErrSwitchConnectToSelf } @@ -531,7 +531,7 @@ func (sw *Switch) addPeer(pc peerConn) error { if sw.nodeKey.ID() == peerID { // overwrite current addr to avoid dialing ourselves again // it means original nodePublicAddr was different from public one - sw.nodePublicAddr = peerNodeInfo.NetAddress() + sw.nodePublicAddr = peerNodeInfo.ListenAddr return ErrSwitchConnectToSelf } From 3a672cb2a904a491a2a257d0eb0b3bcab834d5f7 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 21 Mar 2018 20:23:22 +0100 Subject: [PATCH 053/143] update changelog [ci skip] --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79e866a2c..69c98bf7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ IMPROVEMENTS: - [p2p] seeds respond with a bias towards good peers - [rpc] `/tx` and `/tx_search` responses now include the transaction hash - [rpc] include validator power in `/status` +- [p2p] do not try to connect to ourselves (ok, maybe only once) BUG FIXES: - [rpc] fix subscribing using an abci.ResponseDeliverTx tag From fc9ffee2e3c0bb650b424e77462fa572734f6b89 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 22 Mar 2018 09:54:32 +0100 Subject: [PATCH 054/143] remove unused tracking because it leads to memory leaks in tests see https://blog.cosmos.network/debugging-the-memory-leak-in-tendermint-210186711420 --- p2p/switch_test.go | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/p2p/switch_test.go b/p2p/switch_test.go index 06e8b642e..de0c8d204 100644 --- a/p2p/switch_test.go +++ b/p2p/switch_test.go @@ -39,8 +39,6 @@ type TestReactor struct { mtx sync.Mutex channels []*conn.ChannelDescriptor - peersAdded []Peer - peersRemoved []Peer logMessages bool msgsCounter int msgsReceived map[byte][]PeerMessage @@ -61,17 +59,9 @@ func (tr *TestReactor) GetChannels() []*conn.ChannelDescriptor { return tr.channels } -func (tr *TestReactor) AddPeer(peer Peer) { - tr.mtx.Lock() - defer tr.mtx.Unlock() - tr.peersAdded = append(tr.peersAdded, peer) -} +func (tr *TestReactor) AddPeer(peer Peer) {} -func (tr *TestReactor) RemovePeer(peer Peer, reason interface{}) { - tr.mtx.Lock() - defer tr.mtx.Unlock() - tr.peersRemoved = append(tr.peersRemoved, peer) -} +func (tr *TestReactor) RemovePeer(peer Peer, reason interface{}) {} func (tr *TestReactor) Receive(chID byte, peer Peer, msgBytes []byte) { if tr.logMessages { From 3284a13feed2ee3cda84e6ddbabc2366014449f7 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 22 Mar 2018 09:55:21 +0100 Subject: [PATCH 055/143] add test Refs #1275 --- p2p/switch_test.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/p2p/switch_test.go b/p2p/switch_test.go index de0c8d204..7ac72395c 100644 --- a/p2p/switch_test.go +++ b/p2p/switch_test.go @@ -175,6 +175,37 @@ func TestConnAddrFilter(t *testing.T) { assertNoPeersAfterTimeout(t, s2, 400*time.Millisecond) } +func TestSwitchFiltersOutItself(t *testing.T) { + s1 := MakeSwitch(config, 1, "127.0.0.2", "123.123.123", initSwitchFunc) + + // addr should be rejected immediately because of the same IP & port + addr := s1.NodeInfo().NetAddress() + err := s1.DialPeerWithAddress(addr, false) + if assert.Error(t, err) { + assert.Equal(t, ErrSwitchConnectToSelf, err) + } + + // simulate s1 having a public IP by creating a remote peer with the same ID + rp := &remotePeer{PrivKey: s1.nodeKey.PrivKey, Config: DefaultPeerConfig()} + rp.Start() + + // addr should be rejected in addPeer based on the same ID + err = s1.DialPeerWithAddress(rp.Addr(), false) + if assert.Error(t, err) { + assert.Equal(t, ErrSwitchConnectToSelf, err) + } + + // addr should be rejected immediately because during previous step we changed node's public IP + err = s1.DialPeerWithAddress(rp.Addr(), false) + if assert.Error(t, err) { + assert.Equal(t, ErrSwitchConnectToSelf, err) + } + + rp.Stop() + + assertNoPeersAfterTimeout(t, s1, 100*time.Millisecond) +} + func assertNoPeersAfterTimeout(t *testing.T, sw *Switch, timeout time.Duration) { time.Sleep(timeout) if sw.Peers().Size() != 0 { From 3b3f45d49b6ba255ed32152019864f95a7755e82 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 22 Mar 2018 16:58:57 +0100 Subject: [PATCH 056/143] use addrbook#AddOurAddress to store our address --- node/node.go | 9 ++++-- p2p/pex/addrbook.go | 17 +++++++++-- p2p/pex/pex_reactor_test.go | 57 +++++++++++++++++++++++++------------ p2p/switch.go | 34 +++++++++++----------- p2p/switch_test.go | 23 ++++++++++++++- 5 files changed, 99 insertions(+), 41 deletions(-) diff --git a/node/node.go b/node/node.go index 3a59d2f06..8b7ca1032 100644 --- a/node/node.go +++ b/node/node.go @@ -414,9 +414,14 @@ func (n *Node) OnStart() error { } n.Logger.Info("P2P Node ID", "ID", nodeKey.ID(), "file", n.config.NodeKeyFile()) - // Start the switch - n.sw.SetNodeInfo(n.makeNodeInfo(nodeKey.PubKey())) + nodeInfo := n.makeNodeInfo(nodeKey.PubKey()) + n.sw.SetNodeInfo(nodeInfo) n.sw.SetNodeKey(nodeKey) + + // Add ourselves to addrbook to prevent dialing ourselves + n.addrBook.AddOurAddress(nodeInfo.NetAddress()) + + // Start the switch err = n.sw.Start() if err != nil { return err diff --git a/p2p/pex/addrbook.go b/p2p/pex/addrbook.go index efb48f6da..adaba2fd6 100644 --- a/p2p/pex/addrbook.go +++ b/p2p/pex/addrbook.go @@ -34,6 +34,9 @@ type AddrBook interface { // Add our own addresses so we don't later add ourselves AddOurAddress(*p2p.NetAddress) + // Check if it is our address + OurAddress(*p2p.NetAddress) bool + // Add and remove an address AddAddress(addr *p2p.NetAddress, src *p2p.NetAddress) error RemoveAddress(addr *p2p.NetAddress) @@ -78,7 +81,7 @@ type addrBook struct { // accessed concurrently mtx sync.Mutex rand *rand.Rand - ourAddrs map[string]*p2p.NetAddress + ourAddrs map[string]struct{} addrLookup map[p2p.ID]*knownAddress // new & old bucketsOld []map[string]*knownAddress bucketsNew []map[string]*knownAddress @@ -93,7 +96,7 @@ type addrBook struct { func NewAddrBook(filePath string, routabilityStrict bool) *addrBook { am := &addrBook{ rand: rand.New(rand.NewSource(time.Now().UnixNano())), // TODO: seed from outside - ourAddrs: make(map[string]*p2p.NetAddress), + ourAddrs: make(map[string]struct{}), addrLookup: make(map[p2p.ID]*knownAddress), filePath: filePath, routabilityStrict: routabilityStrict, @@ -154,7 +157,15 @@ func (a *addrBook) AddOurAddress(addr *p2p.NetAddress) { a.mtx.Lock() defer a.mtx.Unlock() a.Logger.Info("Add our address to book", "addr", addr) - a.ourAddrs[addr.String()] = addr + a.ourAddrs[addr.String()] = struct{}{} +} + +// OurAddress returns true if it is our address. +func (a *addrBook) OurAddress(addr *p2p.NetAddress) bool { + a.mtx.Lock() + _, ok := a.ourAddrs[addr.String()] + a.mtx.Unlock() + return ok } // AddAddress implements AddrBook - adds the given address as received from the given source. diff --git a/p2p/pex/pex_reactor_test.go b/p2p/pex/pex_reactor_test.go index 20276ec86..329d17f37 100644 --- a/p2p/pex/pex_reactor_test.go +++ b/p2p/pex/pex_reactor_test.go @@ -63,35 +63,45 @@ func TestPEXReactorRunning(t *testing.T) { N := 3 switches := make([]*p2p.Switch, N) + // directory to store address books dir, err := ioutil.TempDir("", "pex_reactor") require.Nil(t, err) defer os.RemoveAll(dir) // nolint: errcheck - book := NewAddrBook(filepath.Join(dir, "addrbook.json"), false) - book.SetLogger(log.TestingLogger()) + + books := make([]*addrBook, N) + logger := log.TestingLogger() // create switches for i := 0; i < N; i++ { switches[i] = p2p.MakeSwitch(config, i, "127.0.0.1", "123.123.123", func(i int, sw *p2p.Switch) *p2p.Switch { - sw.SetLogger(log.TestingLogger().With("switch", i)) + books[i] = NewAddrBook(filepath.Join(dir, fmt.Sprintf("addrbook%d.json", i)), false) + books[i].SetLogger(logger.With("pex", i)) + sw.SetAddrBook(books[i]) - r := NewPEXReactor(book, &PEXReactorConfig{}) - r.SetLogger(log.TestingLogger()) + sw.SetLogger(logger.With("pex", i)) + + r := NewPEXReactor(books[i], &PEXReactorConfig{}) + r.SetLogger(logger.With("pex", i)) r.SetEnsurePeersPeriod(250 * time.Millisecond) sw.AddReactor("pex", r) + return sw }) } - // fill the address book and add listeners - for _, s := range switches { - addr := s.NodeInfo().NetAddress() - book.AddAddress(addr, addr) - s.AddListener(p2p.NewDefaultListener("tcp", s.NodeInfo().ListenAddr, true, log.TestingLogger())) + addOtherNodeAddrToAddrBook := func(switchIndex, otherSwitchIndex int) { + addr := switches[otherSwitchIndex].NodeInfo().NetAddress() + books[switchIndex].AddAddress(addr, addr) } - // start switches - for _, s := range switches { - err := s.Start() // start switch and reactors + addOtherNodeAddrToAddrBook(0, 1) + addOtherNodeAddrToAddrBook(1, 0) + addOtherNodeAddrToAddrBook(2, 1) + + for i, sw := range switches { + sw.AddListener(p2p.NewDefaultListener("tcp", sw.NodeInfo().ListenAddr, true, logger.With("pex", i))) + + err := sw.Start() // start switch and reactors require.Nil(t, err) } @@ -127,6 +137,7 @@ func TestPEXReactorRequestMessageAbuse(t *testing.T) { defer teardownReactor(book) sw := createSwitchAndAddReactors(r) + sw.SetAddrBook(book) peer := newMockPeer() p2p.AddPeerToSwitch(sw, peer) @@ -156,6 +167,7 @@ func TestPEXReactorAddrsMessageAbuse(t *testing.T) { defer teardownReactor(book) sw := createSwitchAndAddReactors(r) + sw.SetAddrBook(book) peer := newMockPeer() p2p.AddPeerToSwitch(sw, peer) @@ -182,13 +194,11 @@ func TestPEXReactorAddrsMessageAbuse(t *testing.T) { } func TestPEXReactorUsesSeedsIfNeeded(t *testing.T) { + // directory to store address books dir, err := ioutil.TempDir("", "pex_reactor") require.Nil(t, err) defer os.RemoveAll(dir) // nolint: errcheck - book := NewAddrBook(filepath.Join(dir, "addrbook.json"), false) - book.SetLogger(log.TestingLogger()) - // 1. create seed seed := p2p.MakeSwitch( config, @@ -196,6 +206,10 @@ func TestPEXReactorUsesSeedsIfNeeded(t *testing.T) { "127.0.0.1", "123.123.123", func(i int, sw *p2p.Switch) *p2p.Switch { + book := NewAddrBook(filepath.Join(dir, "addrbook0.json"), false) + book.SetLogger(log.TestingLogger()) + sw.SetAddrBook(book) + sw.SetLogger(log.TestingLogger()) r := NewPEXReactor(book, &PEXReactorConfig{}) @@ -222,6 +236,10 @@ func TestPEXReactorUsesSeedsIfNeeded(t *testing.T) { "127.0.0.1", "123.123.123", func(i int, sw *p2p.Switch) *p2p.Switch { + book := NewAddrBook(filepath.Join(dir, "addrbook1.json"), false) + book.SetLogger(log.TestingLogger()) + sw.SetAddrBook(book) + sw.SetLogger(log.TestingLogger()) r := NewPEXReactor( @@ -247,7 +265,8 @@ func TestPEXReactorCrawlStatus(t *testing.T) { defer teardownReactor(book) // Seed/Crawler mode uses data from the Switch - _ = createSwitchAndAddReactors(pexR) + sw := createSwitchAndAddReactors(pexR) + sw.SetAddrBook(book) // Create a peer, add it to the peer set and the addrbook. peer := p2p.CreateRandomPeer(false) @@ -291,7 +310,8 @@ func TestPEXReactorDialPeer(t *testing.T) { pexR, book := createReactor(&PEXReactorConfig{}) defer teardownReactor(book) - _ = createSwitchAndAddReactors(pexR) + sw := createSwitchAndAddReactors(pexR) + sw.SetAddrBook(book) peer := newMockPeer() addr := peer.NodeInfo().NetAddress() @@ -397,6 +417,7 @@ func assertPeersWithTimeout( } func createReactor(config *PEXReactorConfig) (r *PEXReactor, book *addrBook) { + // directory to store address book dir, err := ioutil.TempDir("", "pex_reactor") if err != nil { panic(err) diff --git a/p2p/switch.go b/p2p/switch.go index fec448265..a12052953 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -35,6 +35,8 @@ const ( type AddrBook interface { AddAddress(addr *NetAddress, src *NetAddress) error + AddOurAddress(*NetAddress) + OurAddress(*NetAddress) bool MarkGood(*NetAddress) Save() } @@ -48,18 +50,17 @@ type AddrBook interface { type Switch struct { cmn.BaseService - config *cfg.P2PConfig - peerConfig *PeerConfig - listeners []Listener - reactors map[string]Reactor - chDescs []*conn.ChannelDescriptor - reactorsByCh map[byte]Reactor - peers *PeerSet - dialing *cmn.CMap - nodeInfo NodeInfo // our node info - nodeKey *NodeKey // our node privkey - nodePublicAddr string - addrBook AddrBook + config *cfg.P2PConfig + peerConfig *PeerConfig + listeners []Listener + reactors map[string]Reactor + chDescs []*conn.ChannelDescriptor + reactorsByCh map[byte]Reactor + peers *PeerSet + dialing *cmn.CMap + nodeInfo NodeInfo // our node info + nodeKey *NodeKey // our node privkey + addrBook AddrBook filterConnByAddr func(net.Addr) error filterConnByID func(ID) error @@ -149,7 +150,6 @@ func (sw *Switch) IsListening() bool { // NOTE: Not goroutine safe. func (sw *Switch) SetNodeInfo(nodeInfo NodeInfo) { sw.nodeInfo = nodeInfo - sw.nodePublicAddr = nodeInfo.ListenAddr } // NodeInfo returns the switch's NodeInfo. @@ -384,7 +384,7 @@ func (sw *Switch) DialPeersAsync(addrBook AddrBook, peers []string, persistent b // If `persistent == true`, the switch will always try to reconnect to this peer if the connection ever fails. func (sw *Switch) DialPeerWithAddress(addr *NetAddress, persistent bool) error { // do not dial ourselves - if addr.DialString() == sw.nodePublicAddr { + if sw.addrBook.OurAddress(addr) { return ErrSwitchConnectToSelf } @@ -529,9 +529,9 @@ func (sw *Switch) addPeer(pc peerConn) error { // Avoid self if sw.nodeKey.ID() == peerID { - // overwrite current addr to avoid dialing ourselves again - // it means original nodePublicAddr was different from public one - sw.nodePublicAddr = peerNodeInfo.ListenAddr + // add given address to the address book to avoid dialing ourselves again + // this is our public address + sw.addrBook.AddOurAddress(peerNodeInfo.NetAddress()) return ErrSwitchConnectToSelf } diff --git a/p2p/switch_test.go b/p2p/switch_test.go index 7ac72395c..4995ad3f2 100644 --- a/p2p/switch_test.go +++ b/p2p/switch_test.go @@ -90,6 +90,8 @@ func MakeSwitchPair(t testing.TB, initSwitch func(int, *Switch) *Switch) (*Switc } func initSwitchFunc(i int, sw *Switch) *Switch { + sw.SetAddrBook(&addrBookMock{ourAddrs: make(map[string]struct{})}) + // Make two reactors of two channels each sw.AddReactor("foo", NewTestReactor([]*conn.ChannelDescriptor{ {ID: byte(0x00), Priority: 10}, @@ -99,6 +101,7 @@ func initSwitchFunc(i int, sw *Switch) *Switch { {ID: byte(0x02), Priority: 10}, {ID: byte(0x03), Priority: 10}, }, true)) + return sw } @@ -177,9 +180,12 @@ func TestConnAddrFilter(t *testing.T) { func TestSwitchFiltersOutItself(t *testing.T) { s1 := MakeSwitch(config, 1, "127.0.0.2", "123.123.123", initSwitchFunc) + addr := s1.NodeInfo().NetAddress() + + // add ourselves like we do in node.go#427 + s1.addrBook.AddOurAddress(addr) // addr should be rejected immediately because of the same IP & port - addr := s1.NodeInfo().NetAddress() err := s1.DialPeerWithAddress(addr, false) if assert.Error(t, err) { assert.Equal(t, ErrSwitchConnectToSelf, err) @@ -371,3 +377,18 @@ func BenchmarkSwitchBroadcast(b *testing.B) { b.Logf("success: %v, failure: %v", numSuccess, numFailure) } + +type addrBookMock struct { + ourAddrs map[string]struct{} +} + +var _ AddrBook = (*addrBookMock)(nil) + +func (book *addrBookMock) AddAddress(addr *NetAddress, src *NetAddress) error { return nil } +func (book *addrBookMock) AddOurAddress(addr *NetAddress) { book.ourAddrs[addr.String()] = struct{}{} } +func (book *addrBookMock) OurAddress(addr *NetAddress) bool { + _, ok := book.ourAddrs[addr.String()] + return ok +} +func (book *addrBookMock) MarkGood(*NetAddress) {} +func (book *addrBookMock) Save() {} From 34b77fcad452cf0d797967a46359f05e4841c574 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 5 Apr 2018 15:19:15 +0200 Subject: [PATCH 057/143] log error when we fail to add new address --- p2p/pex/pex_reactor.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/p2p/pex/pex_reactor.go b/p2p/pex/pex_reactor.go index 8c6ad5a8b..1bcc493dd 100644 --- a/p2p/pex/pex_reactor.go +++ b/p2p/pex/pex_reactor.go @@ -164,7 +164,10 @@ func (r *PEXReactor) AddPeer(p Peer) { // peers when we need - we don't trust inbound peers as much. addr := p.NodeInfo().NetAddress() if !isAddrPrivate(addr, r.config.PrivatePeerIDs) { - r.book.AddAddress(addr, addr) + err := r.book.AddAddress(addr, addr) + if err != nil { + r.Logger.Error("Failed to add new address", "err", err) + } } } } @@ -265,7 +268,10 @@ func (r *PEXReactor) ReceiveAddrs(addrs []*p2p.NetAddress, src Peer) error { srcAddr := src.NodeInfo().NetAddress() for _, netAddr := range addrs { if netAddr != nil && !isAddrPrivate(netAddr, r.config.PrivatePeerIDs) { - r.book.AddAddress(netAddr, srcAddr) + err := r.book.AddAddress(netAddr, srcAddr) + if err != nil { + r.Logger.Error("Failed to add new address", "err", err) + } } } return nil From 7f6ee7a46b69cb86bebcacbcd7bd86a122130698 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 5 Apr 2018 15:20:53 +0200 Subject: [PATCH 058/143] add a comment for NewSwitch --- p2p/switch.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/p2p/switch.go b/p2p/switch.go index a12052953..1abc6b14e 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -33,6 +33,8 @@ const ( //----------------------------------------------------------------------------- +// An AddrBook represents an address book from the pex package, which is used +// to store peer addresses. type AddrBook interface { AddAddress(addr *NetAddress, src *NetAddress) error AddOurAddress(*NetAddress) @@ -43,7 +45,7 @@ type AddrBook interface { //----------------------------------------------------------------------------- -// `Switch` handles peer connections and exposes an API to receive incoming messages +// Switch handles peer connections and exposes an API to receive incoming messages // on `Reactors`. Each `Reactor` is responsible for handling incoming messages of one // or more `Channels`. So while sending outgoing messages is typically performed on the peer, // incoming messages are received on the reactor. @@ -68,6 +70,7 @@ type Switch struct { rng *rand.Rand // seed for randomizing dial times and orders } +// NewSwitch creates a new Switch with the given config. func NewSwitch(config *cfg.P2PConfig) *Switch { sw := &Switch{ config: config, From d38a6cc7ea2bc536921c2cf8a5538004ede119e8 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 29 Mar 2018 12:06:15 +0200 Subject: [PATCH 059/143] trim whitespace from elements of lists (like `persistent_peers`) Refs #1380 --- CHANGELOG.md | 1 + node/node.go | 19 +++++-------------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79e866a2c..7f2b00ed1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ BREAKING: IMPROVEMENTS: - [p2p] seeds respond with a bias towards good peers +- [config] trim whitespace from elements of lists (like `persistent_peers`) - [rpc] `/tx` and `/tx_search` responses now include the transaction hash - [rpc] include validator power in `/status` diff --git a/node/node.go b/node/node.go index 3a59d2f06..dbda514b7 100644 --- a/node/node.go +++ b/node/node.go @@ -7,7 +7,6 @@ import ( "fmt" "net" "net/http" - "strings" abci "github.com/tendermint/abci/types" crypto "github.com/tendermint/go-crypto" @@ -277,19 +276,11 @@ func NewNode(config *cfg.Config, trustMetricStore = trust.NewTrustMetricStore(trustHistoryDB, trust.DefaultConfig()) trustMetricStore.SetLogger(p2pLogger) - var seeds []string - if config.P2P.Seeds != "" { - seeds = strings.Split(config.P2P.Seeds, ",") - } - var privatePeerIDs []string - if config.P2P.PrivatePeerIDs != "" { - privatePeerIDs = strings.Split(config.P2P.PrivatePeerIDs, ",") - } pexReactor := pex.NewPEXReactor(addrBook, &pex.PEXReactorConfig{ - Seeds: seeds, + Seeds: cmn.SplitAndTrim(config.P2P.Seeds, ",", " "), SeedMode: config.P2P.SeedMode, - PrivatePeerIDs: privatePeerIDs}) + PrivatePeerIDs: cmn.SplitAndTrim(config.P2P.PrivatePeerIDs, ",", " ")}) pexReactor.SetLogger(p2pLogger) sw.AddReactor("PEX", pexReactor) } @@ -339,7 +330,7 @@ func NewNode(config *cfg.Config, return nil, err } if config.TxIndex.IndexTags != "" { - txIndexer = kv.NewTxIndex(store, kv.IndexTags(strings.Split(config.TxIndex.IndexTags, ","))) + txIndexer = kv.NewTxIndex(store, kv.IndexTags(cmn.SplitAndTrim(config.TxIndex.IndexTags, ",", " "))) } else if config.TxIndex.IndexAllTags { txIndexer = kv.NewTxIndex(store, kv.IndexAllTags()) } else { @@ -424,7 +415,7 @@ func (n *Node) OnStart() error { // Always connect to persistent peers if n.config.P2P.PersistentPeers != "" { - err = n.sw.DialPeersAsync(n.addrBook, strings.Split(n.config.P2P.PersistentPeers, ","), true) + err = n.sw.DialPeersAsync(n.addrBook, cmn.SplitAndTrim(n.config.P2P.PersistentPeers, ",", " "), true) if err != nil { return err } @@ -495,7 +486,7 @@ func (n *Node) ConfigureRPC() { func (n *Node) startRPC() ([]net.Listener, error) { n.ConfigureRPC() - listenAddrs := strings.Split(n.config.RPC.ListenAddress, ",") + listenAddrs := cmn.SplitAndTrim(n.config.RPC.ListenAddress, ",", " ") if n.config.RPC.Unsafe { rpccore.AddUnsafeRoutes() From 6e39ec6e262790c9dc0080b043fa3c55ca76f364 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 5 Apr 2018 15:21:11 +0200 Subject: [PATCH 060/143] do not even try to dial ourselves also, remove address from the book (plus mark it as our address) and return an error if we fail to parse peers list --- p2p/pex/addrbook.go | 14 +++++++++++-- p2p/pex/addrbook_test.go | 16 +++++++++++++++ p2p/switch.go | 39 +++++++++++++++++++++++------------- p2p/switch_test.go | 43 ++++++++++++++++++++++------------------ 4 files changed, 77 insertions(+), 35 deletions(-) diff --git a/p2p/pex/addrbook.go b/p2p/pex/addrbook.go index adaba2fd6..a8462f375 100644 --- a/p2p/pex/addrbook.go +++ b/p2p/pex/addrbook.go @@ -33,13 +33,15 @@ type AddrBook interface { // Add our own addresses so we don't later add ourselves AddOurAddress(*p2p.NetAddress) - // Check if it is our address OurAddress(*p2p.NetAddress) bool // Add and remove an address AddAddress(addr *p2p.NetAddress, src *p2p.NetAddress) error - RemoveAddress(addr *p2p.NetAddress) + RemoveAddress(*p2p.NetAddress) + + // Check if the address is in the book + HasAddress(*p2p.NetAddress) bool // Do we need more peers? NeedMoreAddrs() bool @@ -196,6 +198,14 @@ func (a *addrBook) IsGood(addr *p2p.NetAddress) bool { return a.addrLookup[addr.ID].isOld() } +// HasAddress returns true if the address is in the book. +func (a *addrBook) HasAddress(addr *p2p.NetAddress) bool { + a.mtx.Lock() + defer a.mtx.Unlock() + ka := a.addrLookup[addr.ID] + return ka != nil +} + // NeedMoreAddrs implements AddrBook - returns true if there are not have enough addresses in the book. func (a *addrBook) NeedMoreAddrs() bool { return a.Size() < needAddressThreshold diff --git a/p2p/pex/addrbook_test.go b/p2p/pex/addrbook_test.go index 68edb188a..2e2604286 100644 --- a/p2p/pex/addrbook_test.go +++ b/p2p/pex/addrbook_test.go @@ -338,3 +338,19 @@ func TestAddrBookGetSelectionWithBias(t *testing.T) { t.Fatalf("expected more good peers (%% got: %d, %% expected: %d, number of good addrs: %d, total: %d)", got, expected, good, len(selection)) } } + +func TestAddrBookHasAddress(t *testing.T) { + fname := createTempFileName("addrbook_test") + defer deleteTempFile(fname) + + book := NewAddrBook(fname, true) + book.SetLogger(log.TestingLogger()) + addr := randIPv4Address(t) + book.AddAddress(addr, addr) + + assert.True(t, book.HasAddress(addr)) + + book.RemoveAddress(addr) + + assert.False(t, book.HasAddress(addr)) +} diff --git a/p2p/switch.go b/p2p/switch.go index 1abc6b14e..9f65a85e0 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -40,6 +40,8 @@ type AddrBook interface { AddOurAddress(*NetAddress) OurAddress(*NetAddress) bool MarkGood(*NetAddress) + RemoveAddress(*NetAddress) + HasAddress(*NetAddress) bool Save() } @@ -351,17 +353,20 @@ func (sw *Switch) DialPeersAsync(addrBook AddrBook, peers []string, persistent b for _, err := range errs { sw.Logger.Error("Error in peer's address", "err", err) } + if len(errs) > 0 { + return errors.New("Errors in peer addresses (see errors above)") + } + ourAddr := sw.nodeInfo.NetAddress() + + // TODO: move this out of here ? if addrBook != nil { // add peers to `addrBook` - ourAddr := sw.nodeInfo.NetAddress() for _, netAddr := range netAddrs { // do not add our address or ID - if netAddr.Same(ourAddr) { - continue + if !netAddr.Same(ourAddr) { + addrBook.AddAddress(netAddr, ourAddr) } - // TODO: move this out of here ? - addrBook.AddAddress(netAddr, ourAddr) } // Persist some peers to disk right away. // NOTE: integration tests depend on this @@ -372,8 +377,14 @@ func (sw *Switch) DialPeersAsync(addrBook AddrBook, peers []string, persistent b perm := sw.rng.Perm(len(netAddrs)) for i := 0; i < len(perm); i++ { go func(i int) { - sw.randomSleep(0) j := perm[i] + + // do not dial ourselves + if netAddrs[j].Same(ourAddr) { + return + } + + sw.randomSleep(0) err := sw.DialPeerWithAddress(netAddrs[j], persistent) if err != nil { sw.Logger.Error("Error dialing peer", "err", err) @@ -386,11 +397,6 @@ func (sw *Switch) DialPeersAsync(addrBook AddrBook, peers []string, persistent b // DialPeerWithAddress dials the given peer and runs sw.addPeer if it connects and authenticates successfully. // If `persistent == true`, the switch will always try to reconnect to this peer if the connection ever fails. func (sw *Switch) DialPeerWithAddress(addr *NetAddress, persistent bool) error { - // do not dial ourselves - if sw.addrBook.OurAddress(addr) { - return ErrSwitchConnectToSelf - } - sw.dialing.Set(string(addr.ID), addr) defer sw.dialing.Delete(string(addr.ID)) return sw.addOutboundPeerWithConfig(addr, sw.peerConfig, persistent) @@ -532,9 +538,14 @@ func (sw *Switch) addPeer(pc peerConn) error { // Avoid self if sw.nodeKey.ID() == peerID { - // add given address to the address book to avoid dialing ourselves again - // this is our public address - sw.addrBook.AddOurAddress(peerNodeInfo.NetAddress()) + addr := peerNodeInfo.NetAddress() + + // remove the given address from the address book if we're added it earlier + sw.addrBook.RemoveAddress(addr) + + // add the given address to the address book to avoid dialing ourselves + // again this is our public address + sw.addrBook.AddOurAddress(addr) return ErrSwitchConnectToSelf } diff --git a/p2p/switch_test.go b/p2p/switch_test.go index 4995ad3f2..8fec3e7ab 100644 --- a/p2p/switch_test.go +++ b/p2p/switch_test.go @@ -90,7 +90,9 @@ func MakeSwitchPair(t testing.TB, initSwitch func(int, *Switch) *Switch) (*Switc } func initSwitchFunc(i int, sw *Switch) *Switch { - sw.SetAddrBook(&addrBookMock{ourAddrs: make(map[string]struct{})}) + sw.SetAddrBook(&addrBookMock{ + addrs: make(map[string]struct{}, 0), + ourAddrs: make(map[string]struct{}, 0)}) // Make two reactors of two channels each sw.AddReactor("foo", NewTestReactor([]*conn.ChannelDescriptor{ @@ -180,32 +182,24 @@ func TestConnAddrFilter(t *testing.T) { func TestSwitchFiltersOutItself(t *testing.T) { s1 := MakeSwitch(config, 1, "127.0.0.2", "123.123.123", initSwitchFunc) - addr := s1.NodeInfo().NetAddress() + // addr := s1.NodeInfo().NetAddress() - // add ourselves like we do in node.go#427 - s1.addrBook.AddOurAddress(addr) - - // addr should be rejected immediately because of the same IP & port - err := s1.DialPeerWithAddress(addr, false) - if assert.Error(t, err) { - assert.Equal(t, ErrSwitchConnectToSelf, err) - } + // // add ourselves like we do in node.go#427 + // s1.addrBook.AddOurAddress(addr) // simulate s1 having a public IP by creating a remote peer with the same ID rp := &remotePeer{PrivKey: s1.nodeKey.PrivKey, Config: DefaultPeerConfig()} rp.Start() // addr should be rejected in addPeer based on the same ID - err = s1.DialPeerWithAddress(rp.Addr(), false) + err := s1.DialPeerWithAddress(rp.Addr(), false) if assert.Error(t, err) { assert.Equal(t, ErrSwitchConnectToSelf, err) } - // addr should be rejected immediately because during previous step we changed node's public IP - err = s1.DialPeerWithAddress(rp.Addr(), false) - if assert.Error(t, err) { - assert.Equal(t, ErrSwitchConnectToSelf, err) - } + assert.True(t, s1.addrBook.OurAddress(rp.Addr())) + + assert.False(t, s1.addrBook.HasAddress(rp.Addr())) rp.Stop() @@ -379,16 +373,27 @@ func BenchmarkSwitchBroadcast(b *testing.B) { } type addrBookMock struct { + addrs map[string]struct{} ourAddrs map[string]struct{} } var _ AddrBook = (*addrBookMock)(nil) -func (book *addrBookMock) AddAddress(addr *NetAddress, src *NetAddress) error { return nil } -func (book *addrBookMock) AddOurAddress(addr *NetAddress) { book.ourAddrs[addr.String()] = struct{}{} } +func (book *addrBookMock) AddAddress(addr *NetAddress, src *NetAddress) error { + book.addrs[addr.String()] = struct{}{} + return nil +} +func (book *addrBookMock) AddOurAddress(addr *NetAddress) { book.ourAddrs[addr.String()] = struct{}{} } func (book *addrBookMock) OurAddress(addr *NetAddress) bool { _, ok := book.ourAddrs[addr.String()] return ok } func (book *addrBookMock) MarkGood(*NetAddress) {} -func (book *addrBookMock) Save() {} +func (book *addrBookMock) HasAddress(addr *NetAddress) bool { + _, ok := book.addrs[addr.String()] + return ok +} +func (book *addrBookMock) RemoveAddress(addr *NetAddress) { + delete(book.addrs, addr.String()) +} +func (book *addrBookMock) Save() {} From 45ec5fd1702f22ef5517a0c16f52a503cc5ddb65 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Thu, 5 Apr 2018 07:05:45 -0700 Subject: [PATCH 061/143] WIP consensus --- consensus/byzantine_test.go | 57 +++------------------- consensus/common_test.go | 7 +-- consensus/reactor.go | 95 +++++++++++++++---------------------- consensus/replay_test.go | 14 +++--- consensus/state.go | 11 ++--- consensus/state_test.go | 1 + consensus/wal.go | 27 +++++------ consensus/wal_generator.go | 3 +- consensus/wal_test.go | 7 +-- consensus/wire.go | 14 ++++++ types/block.go | 4 +- types/priv_validator.go | 7 +++ 12 files changed, 104 insertions(+), 143 deletions(-) create mode 100644 consensus/wire.go diff --git a/consensus/byzantine_test.go b/consensus/byzantine_test.go index f8163c07d..5f04a3308 100644 --- a/consensus/byzantine_test.go +++ b/consensus/byzantine_test.go @@ -7,7 +7,6 @@ import ( "time" "github.com/stretchr/testify/require" - crypto "github.com/tendermint/go-crypto" "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/types" cmn "github.com/tendermint/tmlibs/common" @@ -48,7 +47,9 @@ func TestByzantine(t *testing.T) { for i := 0; i < N; i++ { // make first val byzantine if i == 0 { - css[i].privValidator = NewByzantinePrivValidator(css[i].privValidator) + // NOTE: Now, test validators are MockPV, which by default doesn't + // do any safety checks. + css[i].privValidator.(*types.MockPV).DisableChecks() css[i].decideProposal = func(j int) func(int64, int) { return func(height int64, round int) { byzantineDecideProposalFunc(t, height, round, css[j], switches[j]) @@ -203,7 +204,7 @@ func byzantineDecideProposalFunc(t *testing.T, height int64, round int, cs *Cons func sendProposalAndParts(height int64, round int, cs *ConsensusState, peer p2p.Peer, proposal *types.Proposal, blockHash []byte, parts *types.PartSet) { // proposal msg := &ProposalMessage{Proposal: proposal} - peer.Send(DataChannel, struct{ ConsensusMessage }{msg}) + peer.Send(DataChannel, cdc.MustMarshalBinaryBare(msg)) // parts for i := 0; i < parts.Total(); i++ { @@ -213,7 +214,7 @@ func sendProposalAndParts(height int64, round int, cs *ConsensusState, peer p2p. Round: round, // This tells peer that this part applies to us. Part: part, } - peer.Send(DataChannel, struct{ ConsensusMessage }{msg}) + peer.Send(DataChannel, cdc.MustMarshalBinaryBare(msg)) } // votes @@ -222,8 +223,8 @@ func sendProposalAndParts(height int64, round int, cs *ConsensusState, peer p2p. precommit, _ := cs.signVote(types.VoteTypePrecommit, blockHash, parts.Header()) cs.mtx.Unlock() - peer.Send(VoteChannel, struct{ ConsensusMessage }{&VoteMessage{prevote}}) - peer.Send(VoteChannel, struct{ ConsensusMessage }{&VoteMessage{precommit}}) + peer.Send(VoteChannel, cdc.MustMarshalBinaryBare(&VoteMessage{prevote})) + peer.Send(VoteChannel, cdc.MustMarshalBinaryBare(&VoteMessage{precommit})) } //---------------------------------------- @@ -264,47 +265,3 @@ func (br *ByzantineReactor) RemovePeer(peer p2p.Peer, reason interface{}) { func (br *ByzantineReactor) Receive(chID byte, peer p2p.Peer, msgBytes []byte) { br.reactor.Receive(chID, peer, msgBytes) } - -//---------------------------------------- -// byzantine privValidator - -type ByzantinePrivValidator struct { - types.Signer - - pv types.PrivValidator -} - -// Return a priv validator that will sign anything -func NewByzantinePrivValidator(pv types.PrivValidator) *ByzantinePrivValidator { - return &ByzantinePrivValidator{ - Signer: pv.(*types.PrivValidatorFS).Signer, - pv: pv, - } -} - -func (privVal *ByzantinePrivValidator) GetAddress() types.Address { - return privVal.pv.GetAddress() -} - -func (privVal *ByzantinePrivValidator) GetPubKey() crypto.PubKey { - return privVal.pv.GetPubKey() -} - -func (privVal *ByzantinePrivValidator) SignVote(chainID string, vote *types.Vote) (err error) { - vote.Signature, err = privVal.Sign(vote.SignBytes(chainID)) - return err -} - -func (privVal *ByzantinePrivValidator) SignProposal(chainID string, proposal *types.Proposal) (err error) { - proposal.Signature, _ = privVal.Sign(proposal.SignBytes(chainID)) - return nil -} - -func (privVal *ByzantinePrivValidator) SignHeartbeat(chainID string, heartbeat *types.Heartbeat) (err error) { - heartbeat.Signature, _ = privVal.Sign(heartbeat.SignBytes(chainID)) - return nil -} - -func (privVal *ByzantinePrivValidator) String() string { - return cmn.Fmt("PrivValidator{%X}", privVal.GetAddress()) -} diff --git a/consensus/common_test.go b/consensus/common_test.go index dbe39dda1..c8de40011 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -21,6 +21,7 @@ import ( "github.com/tendermint/tendermint/p2p" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" + privval "github.com/tendermint/tendermint/types/priv_validator" cmn "github.com/tendermint/tmlibs/common" dbm "github.com/tendermint/tmlibs/db" "github.com/tendermint/tmlibs/log" @@ -277,10 +278,10 @@ func newConsensusStateWithConfigAndBlockStore(thisConfig *cfg.Config, state sm.S return cs } -func loadPrivValidator(config *cfg.Config) *types.PrivValidatorFS { +func loadPrivValidator(config *cfg.Config) *privval.FilePV { privValidatorFile := config.PrivValidatorFile() ensureDir(path.Dir(privValidatorFile), 0700) - privValidator := types.LoadOrGenPrivValidatorFS(privValidatorFile) + privValidator := privval.LoadOrGenFilePV(privValidatorFile) privValidator.Reset() return privValidator } @@ -378,7 +379,7 @@ func randConsensusNetWithPeers(nValidators, nPeers int, testName string, tickerF privVal = privVals[i] } else { _, tempFilePath := cmn.Tempfile("priv_validator_") - privVal = types.GenPrivValidatorFS(tempFilePath) + privVal = privval.GenFilePV(tempFilePath) } app := appFunc() diff --git a/consensus/reactor.go b/consensus/reactor.go index 5c672a0c7..3ab8995d7 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -1,7 +1,6 @@ package consensus import ( - "bytes" "context" "fmt" "reflect" @@ -10,7 +9,7 @@ import ( "github.com/pkg/errors" - wire "github.com/tendermint/go-wire" + "github.com/tendermint/go-amino" cmn "github.com/tendermint/tmlibs/common" "github.com/tendermint/tmlibs/log" @@ -176,7 +175,7 @@ func (conR *ConsensusReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) return } - _, msg, err := DecodeMessage(msgBytes) + msg, err := DecodeMessage(msgBytes) if err != nil { conR.Logger.Error("Error decoding message", "src", src, "chId", chID, "msg", msg, "err", err, "bytes", msgBytes) conR.Switch.StopPeerForError(src, err) @@ -222,13 +221,13 @@ func (conR *ConsensusReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) conR.Logger.Error("Bad VoteSetBitsMessage field Type") return } - src.TrySend(VoteSetBitsChannel, struct{ ConsensusMessage }{&VoteSetBitsMessage{ + src.TrySend(VoteSetBitsChannel, cdc.MustMarshalBinaryBare(&VoteSetBitsMessage{ Height: msg.Height, Round: msg.Round, Type: msg.Type, BlockID: msg.BlockID, Votes: ourVotes, - }}) + })) case *ProposalHeartbeatMessage: hb := msg.Heartbeat conR.Logger.Debug("Received proposal heartbeat message", @@ -401,16 +400,16 @@ func (conR *ConsensusReactor) broadcastProposalHeartbeatMessage(heartbeat types. conR.Logger.Debug("Broadcasting proposal heartbeat message", "height", hb.Height, "round", hb.Round, "sequence", hb.Sequence) msg := &ProposalHeartbeatMessage{hb} - conR.Switch.Broadcast(StateChannel, struct{ ConsensusMessage }{msg}) + conR.Switch.Broadcast(StateChannel, cdc.MustMarshalBinaryBare(msg)) } func (conR *ConsensusReactor) broadcastNewRoundStep(rs *cstypes.RoundState) { nrsMsg, csMsg := makeRoundStepMessages(rs) if nrsMsg != nil { - conR.Switch.Broadcast(StateChannel, struct{ ConsensusMessage }{nrsMsg}) + conR.Switch.Broadcast(StateChannel, cdc.MustMarshalBinaryBare(nrsMsg)) } if csMsg != nil { - conR.Switch.Broadcast(StateChannel, struct{ ConsensusMessage }{csMsg}) + conR.Switch.Broadcast(StateChannel, cdc.MustMarshalBinaryBare(csMsg)) } } @@ -422,7 +421,7 @@ func (conR *ConsensusReactor) broadcastHasVoteMessage(vote *types.Vote) { Type: vote.Type, Index: vote.ValidatorIndex, } - conR.Switch.Broadcast(StateChannel, struct{ ConsensusMessage }{msg}) + conR.Switch.Broadcast(StateChannel, cdc.MustMarshalBinaryBare(msg)) /* // TODO: Make this broadcast more selective. for _, peer := range conR.Switch.Peers().List() { @@ -462,10 +461,10 @@ func (conR *ConsensusReactor) sendNewRoundStepMessages(peer p2p.Peer) { rs := conR.conS.GetRoundState() nrsMsg, csMsg := makeRoundStepMessages(rs) if nrsMsg != nil { - peer.Send(StateChannel, struct{ ConsensusMessage }{nrsMsg}) + peer.Send(StateChannel, cdc.MustMarshalBinaryBare(nrsMsg)) } if csMsg != nil { - peer.Send(StateChannel, struct{ ConsensusMessage }{csMsg}) + peer.Send(StateChannel, cdc.MustMarshalBinaryBare(csMsg)) } } @@ -492,7 +491,7 @@ OUTER_LOOP: Part: part, } logger.Debug("Sending block part", "height", prs.Height, "round", prs.Round) - if peer.Send(DataChannel, struct{ ConsensusMessage }{msg}) { + if peer.Send(DataChannel, cdc.MustMarshalBinaryBare(msg)) { ps.SetHasProposalBlockPart(prs.Height, prs.Round, index) } continue OUTER_LOOP @@ -536,7 +535,7 @@ OUTER_LOOP: { msg := &ProposalMessage{Proposal: rs.Proposal} logger.Debug("Sending proposal", "height", prs.Height, "round", prs.Round) - if peer.Send(DataChannel, struct{ ConsensusMessage }{msg}) { + if peer.Send(DataChannel, cdc.MustMarshalBinaryBare(msg)) { ps.SetHasProposal(rs.Proposal) } } @@ -551,7 +550,7 @@ OUTER_LOOP: ProposalPOL: rs.Votes.Prevotes(rs.Proposal.POLRound).BitArray(), } logger.Debug("Sending POL", "height", prs.Height, "round", prs.Round) - peer.Send(DataChannel, struct{ ConsensusMessage }{msg}) + peer.Send(DataChannel, cdc.MustMarshalBinaryBare(msg)) } continue OUTER_LOOP } @@ -594,7 +593,7 @@ func (conR *ConsensusReactor) gossipDataForCatchup(logger log.Logger, rs *cstype Part: part, } logger.Debug("Sending block part for catchup", "round", prs.Round, "index", index) - if peer.Send(DataChannel, struct{ ConsensusMessage }{msg}) { + if peer.Send(DataChannel, cdc.MustMarshalBinaryBare(msg)) { ps.SetHasProposalBlockPart(prs.Height, prs.Round, index) } else { logger.Debug("Sending block part for catchup failed") @@ -733,12 +732,12 @@ OUTER_LOOP: prs := ps.GetRoundState() if rs.Height == prs.Height { if maj23, ok := rs.Votes.Prevotes(prs.Round).TwoThirdsMajority(); ok { - peer.TrySend(StateChannel, struct{ ConsensusMessage }{&VoteSetMaj23Message{ + peer.TrySend(StateChannel, cdc.MustMarshalBinaryBare(&VoteSetMaj23Message{ Height: prs.Height, Round: prs.Round, Type: types.VoteTypePrevote, BlockID: maj23, - }}) + })) time.Sleep(conR.conS.config.PeerQueryMaj23Sleep()) } } @@ -750,12 +749,12 @@ OUTER_LOOP: prs := ps.GetRoundState() if rs.Height == prs.Height { if maj23, ok := rs.Votes.Precommits(prs.Round).TwoThirdsMajority(); ok { - peer.TrySend(StateChannel, struct{ ConsensusMessage }{&VoteSetMaj23Message{ + peer.TrySend(StateChannel, cdc.MustMarshalBinaryBare(&VoteSetMaj23Message{ Height: prs.Height, Round: prs.Round, Type: types.VoteTypePrecommit, BlockID: maj23, - }}) + })) time.Sleep(conR.conS.config.PeerQueryMaj23Sleep()) } } @@ -767,12 +766,12 @@ OUTER_LOOP: prs := ps.GetRoundState() if rs.Height == prs.Height && prs.ProposalPOLRound >= 0 { if maj23, ok := rs.Votes.Prevotes(prs.ProposalPOLRound).TwoThirdsMajority(); ok { - peer.TrySend(StateChannel, struct{ ConsensusMessage }{&VoteSetMaj23Message{ + peer.TrySend(StateChannel, cdc.MustMarshalBinaryBare(&VoteSetMaj23Message{ Height: prs.Height, Round: prs.ProposalPOLRound, Type: types.VoteTypePrevote, BlockID: maj23, - }}) + })) time.Sleep(conR.conS.config.PeerQueryMaj23Sleep()) } } @@ -786,12 +785,12 @@ OUTER_LOOP: prs := ps.GetRoundState() if prs.CatchupCommitRound != -1 && 0 < prs.Height && prs.Height <= conR.conS.blockStore.Height() { commit := conR.conS.LoadCommit(prs.Height) - peer.TrySend(StateChannel, struct{ ConsensusMessage }{&VoteSetMaj23Message{ + peer.TrySend(StateChannel, cdc.MustMarshalBinaryBare(&VoteSetMaj23Message{ Height: prs.Height, Round: commit.Round(), Type: types.VoteTypePrecommit, BlockID: commit.BlockID, - }}) + })) time.Sleep(conR.conS.config.PeerQueryMaj23Sleep()) } } @@ -938,7 +937,7 @@ func (ps *PeerState) PickSendVote(votes types.VoteSetReader) bool { if vote, ok := ps.PickVoteToSend(votes); ok { msg := &VoteMessage{vote} ps.logger.Debug("Sending vote message", "ps", ps, "vote", vote) - return ps.Peer.Send(VoteChannel, struct{ ConsensusMessage }{msg}) + return ps.Peer.Send(VoteChannel, cdc.MustMarshalBinaryBare(msg)) } return false } @@ -1261,45 +1260,27 @@ func (ps *PeerState) StringIndented(indent string) string { //----------------------------------------------------------------------------- // Messages -const ( - msgTypeNewRoundStep = byte(0x01) - msgTypeCommitStep = byte(0x02) - msgTypeProposal = byte(0x11) - msgTypeProposalPOL = byte(0x12) - msgTypeBlockPart = byte(0x13) // both block & POL - msgTypeVote = byte(0x14) - msgTypeHasVote = byte(0x15) - msgTypeVoteSetMaj23 = byte(0x16) - msgTypeVoteSetBits = byte(0x17) - - msgTypeProposalHeartbeat = byte(0x20) -) - // ConsensusMessage is a message that can be sent and received on the ConsensusReactor type ConsensusMessage interface{} -var _ = wire.RegisterInterface( - struct{ ConsensusMessage }{}, - wire.ConcreteType{&NewRoundStepMessage{}, msgTypeNewRoundStep}, - wire.ConcreteType{&CommitStepMessage{}, msgTypeCommitStep}, - wire.ConcreteType{&ProposalMessage{}, msgTypeProposal}, - wire.ConcreteType{&ProposalPOLMessage{}, msgTypeProposalPOL}, - wire.ConcreteType{&BlockPartMessage{}, msgTypeBlockPart}, - wire.ConcreteType{&VoteMessage{}, msgTypeVote}, - wire.ConcreteType{&HasVoteMessage{}, msgTypeHasVote}, - wire.ConcreteType{&VoteSetMaj23Message{}, msgTypeVoteSetMaj23}, - wire.ConcreteType{&VoteSetBitsMessage{}, msgTypeVoteSetBits}, - wire.ConcreteType{&ProposalHeartbeatMessage{}, msgTypeProposalHeartbeat}, -) +func RegisterConsensusMessages(cdc *amino.Codec) { + cdc.RegisterInterface((*ConsensusMessage)(nil), nil) + cdc.RegisterConcrete(&NewRoundStepMessage{}, "tendermint/NewRoundStepMessage", nil) + cdc.RegisterConcrete(&CommitStepMessage{}, "tendermint/CommitStep", nil) + cdc.RegisterConcrete(&ProposalMessage{}, "tendermint/Proposal", nil) + cdc.RegisterConcrete(&ProposalPOLMessage{}, "tendermint/ProposalPOL", nil) + cdc.RegisterConcrete(&BlockPartMessage{}, "tendermint/BlockPart", nil) + cdc.RegisterConcrete(&VoteMessage{}, "tendermint/Vote", nil) + cdc.RegisterConcrete(&HasVoteMessage{}, "tendermint/HasVote", nil) + cdc.RegisterConcrete(&VoteSetMaj23Message{}, "tendermint/VoteSetMaj23", nil) + cdc.RegisterConcrete(&VoteSetBitsMessage{}, "tendermint/VoteSetBits", nil) + cdc.RegisterConcrete(&ProposalHeartbeatMessage{}, "tendermint/ProposalHeartbeat", nil) +} // DecodeMessage decodes the given bytes into a ConsensusMessage. // TODO: check for unnecessary extra bytes at the end. -func DecodeMessage(bz []byte) (msgType byte, msg ConsensusMessage, err error) { - msgType = bz[0] - n := new(int) - r := bytes.NewReader(bz) - msgI := wire.ReadBinary(struct{ ConsensusMessage }{}, r, maxConsensusMessageSize, n, &err) - msg = msgI.(struct{ ConsensusMessage }).ConsensusMessage +func DecodeMessage(bz []byte) (msg ConsensusMessage, err error) { + err = cdc.UnmarshalBinaryBare(bz, &msg) return } diff --git a/consensus/replay_test.go b/consensus/replay_test.go index 9d7bbc8e4..b3c6ac4a4 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -17,8 +17,7 @@ import ( "github.com/tendermint/abci/example/kvstore" abci "github.com/tendermint/abci/types" - crypto "github.com/tendermint/go-crypto" - wire "github.com/tendermint/go-wire" + "github.com/tendermint/go-crypto" auto "github.com/tendermint/tmlibs/autofile" cmn "github.com/tendermint/tmlibs/common" dbm "github.com/tendermint/tmlibs/db" @@ -27,6 +26,7 @@ import ( "github.com/tendermint/tendermint/proxy" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" + privval "github.com/tendermint/tendermint/types/priv_validator" "github.com/tendermint/tmlibs/log" ) @@ -325,7 +325,7 @@ func testHandshakeReplay(t *testing.T, nBlocks int, mode uint) { walFile := tempWALWithData(walBody) config.Consensus.SetWalFile(walFile) - privVal := types.LoadPrivValidatorFS(config.PrivValidatorFile()) + privVal := privval.LoadFilePV(config.PrivValidatorFile()) wal, err := NewWAL(walFile, false) if err != nil { @@ -519,8 +519,8 @@ func makeBlockchainFromWAL(wal WAL) ([]*types.Block, []*types.Commit, error) { case EndHeightMessage: // if its not the first one, we have a full block if thisBlockParts != nil { - var n int - block := wire.ReadBinary(&types.Block{}, thisBlockParts.GetReader(), 0, &n, &err).(*types.Block) + var block = new(types.Block) + _, err = cdc.UnmarshalBinaryReader(thisBlockParts.GetReader(), block, 0) if err != nil { panic(err) } @@ -552,8 +552,8 @@ func makeBlockchainFromWAL(wal WAL) ([]*types.Block, []*types.Commit, error) { } } // grab the last block too - var n int - block := wire.ReadBinary(&types.Block{}, thisBlockParts.GetReader(), 0, &n, &err).(*types.Block) + var block = new(types.Block) + _, err = cdc.UnmarshalBinaryReader(thisBlockParts.GetReader(), block, 0) if err != nil { panic(err) } diff --git a/consensus/state.go b/consensus/state.go index 7722990b2..a64ef3e49 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -4,17 +4,14 @@ import ( "bytes" "errors" "fmt" + fail "github.com/ebuchman/fail-test" + cmn "github.com/tendermint/tmlibs/common" + "github.com/tendermint/tmlibs/log" "reflect" "runtime/debug" "sync" "time" - fail "github.com/ebuchman/fail-test" - - wire "github.com/tendermint/go-wire" - cmn "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tmlibs/log" - cfg "github.com/tendermint/tendermint/config" cstypes "github.com/tendermint/tendermint/consensus/types" "github.com/tendermint/tendermint/p2p" @@ -1301,7 +1298,7 @@ func (cs *ConsensusState) addProposalBlockPart(height int64, part *types.Part, v } if added && cs.ProposalBlockParts.IsComplete() { // Added and completed! - err = cdc.UnmarshalBinaryBare(cs.ProposalBlockParts.GetReader(), &cs.ProposalBlock, cs.state.ConsensusParams.BlockSize.MaxBytes) + _, err = cdc.UnmarshalBinaryReader(cs.ProposalBlockParts.GetReader(), &cs.ProposalBlock, int64(cs.state.ConsensusParams.BlockSize.MaxBytes)) if err != nil { return true, err } diff --git a/consensus/state_test.go b/consensus/state_test.go index e6b2a1354..fea6cf6ae 100644 --- a/consensus/state_test.go +++ b/consensus/state_test.go @@ -409,6 +409,7 @@ func TestStateLockNoPOL(t *testing.T) { validatePrevote(t, cs1, 1, vss[0], rs.LockedBlock.Hash()) // add a conflicting prevote from the other validator + fmt.Println(">>", rs.ProposalBlock) signAddVotes(cs1, types.VoteTypePrevote, hash, rs.ProposalBlock.MakePartSet(partSize).Header(), vs2) <-voteCh diff --git a/consensus/wal.go b/consensus/wal.go index 88218940d..a615720be 100644 --- a/consensus/wal.go +++ b/consensus/wal.go @@ -1,7 +1,6 @@ package consensus import ( - "bytes" "encoding/binary" "fmt" "hash/crc32" @@ -11,7 +10,7 @@ import ( "github.com/pkg/errors" - wire "github.com/tendermint/go-wire" + "github.com/tendermint/go-amino" "github.com/tendermint/tendermint/types" auto "github.com/tendermint/tmlibs/autofile" cmn "github.com/tendermint/tmlibs/common" @@ -38,13 +37,13 @@ type EndHeightMessage struct { type WALMessage interface{} -var _ = wire.RegisterInterface( - struct{ WALMessage }{}, - wire.ConcreteType{types.EventDataRoundState{}, 0x01}, - wire.ConcreteType{msgInfo{}, 0x02}, - wire.ConcreteType{timeoutInfo{}, 0x03}, - wire.ConcreteType{EndHeightMessage{}, 0x04}, -) +func RegisterWALMessages(cdc *amino.Codec) { + cdc.RegisterInterface((*WALMessage)(nil), nil) + cdc.RegisterConcrete(types.EventDataRoundState{}, "tendermint/wal/EventDataRoundState", nil) + cdc.RegisterConcrete(msgInfo{}, "tendermint/wal/MsgInfo", nil) + cdc.RegisterConcrete(timeoutInfo{}, "tendermint/wal/TimeoutInfo", nil) + cdc.RegisterConcrete(EndHeightMessage{}, "tendermint/wal/EndHeightMessagE", nil) +} //-------------------------------------------------------- // Simple write-ahead logger @@ -114,6 +113,7 @@ func (wal *baseWAL) OnStop() { // called in newStep and for each pass in receiveRoutine func (wal *baseWAL) Save(msg WALMessage) { + fmt.Println("!!", 1) if wal == nil { return } @@ -193,7 +193,7 @@ func (wal *baseWAL) SearchForEndHeight(height int64, options *WALSearchOptions) // A WALEncoder writes custom-encoded WAL messages to an output stream. // -// Format: 4 bytes CRC sum + 4 bytes length + arbitrary-length value (go-wire encoded) +// Format: 4 bytes CRC sum + 4 bytes length + arbitrary-length value (go-amino encoded) type WALEncoder struct { wr io.Writer } @@ -205,7 +205,7 @@ func NewWALEncoder(wr io.Writer) *WALEncoder { // Encode writes the custom encoding of v to the stream. func (enc *WALEncoder) Encode(v *TimedWALMessage) error { - data := wire.BinaryBytes(v) + data := cdc.MustMarshalBinaryBare(v) crc := crc32.Checksum(data, crc32c) length := uint32(len(data)) @@ -298,9 +298,8 @@ func (dec *WALDecoder) Decode() (*TimedWALMessage, error) { return nil, DataCorruptionError{fmt.Errorf("checksums do not match: (read: %v, actual: %v)", crc, actualCRC)} } - var nn int - var res *TimedWALMessage // nolint: gosimple - res = wire.ReadBinary(&TimedWALMessage{}, bytes.NewBuffer(data), int(length), &nn, &err).(*TimedWALMessage) + var res = new(TimedWALMessage) // nolint: gosimple + err = cdc.UnmarshalBinaryBare(data, res) if err != nil { return nil, DataCorruptionError{fmt.Errorf("failed to decode data: %v", err)} } diff --git a/consensus/wal_generator.go b/consensus/wal_generator.go index 14f82d8a8..bb0b5296c 100644 --- a/consensus/wal_generator.go +++ b/consensus/wal_generator.go @@ -17,6 +17,7 @@ import ( "github.com/tendermint/tendermint/proxy" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" + privval "github.com/tendermint/tendermint/types/priv_validator" auto "github.com/tendermint/tmlibs/autofile" cmn "github.com/tendermint/tmlibs/common" "github.com/tendermint/tmlibs/db" @@ -40,7 +41,7 @@ func WALWithNBlocks(numBlocks int) (data []byte, err error) { // COPY PASTE FROM node.go WITH A FEW MODIFICATIONS // NOTE: we can't import node package because of circular dependency privValidatorFile := config.PrivValidatorFile() - privValidator := types.LoadOrGenPrivValidatorFS(privValidatorFile) + privValidator := privval.LoadOrGenFilePV(privValidatorFile) genDoc, err := types.GenesisDocFromFile(config.GenesisFile()) if err != nil { return nil, errors.Wrap(err, "failed to read genesis file") diff --git a/consensus/wal_test.go b/consensus/wal_test.go index 3553591c9..8c95748eb 100644 --- a/consensus/wal_test.go +++ b/consensus/wal_test.go @@ -3,11 +3,10 @@ package consensus import ( "bytes" "crypto/rand" - "sync" + // "sync" "testing" "time" - wire "github.com/tendermint/go-wire" "github.com/tendermint/tendermint/consensus/types" tmtypes "github.com/tendermint/tendermint/types" cmn "github.com/tendermint/tmlibs/common" @@ -68,6 +67,7 @@ func TestWALSearchForEndHeight(t *testing.T) { assert.Equal(t, rs.Height, h+1, cmn.Fmt("wrong height")) } +/* var initOnce sync.Once func registerInterfacesOnce() { @@ -78,6 +78,7 @@ func registerInterfacesOnce() { ) }) } +*/ func nBytes(n int) []byte { buf := make([]byte, n) @@ -86,7 +87,7 @@ func nBytes(n int) []byte { } func benchmarkWalDecode(b *testing.B, n int) { - registerInterfacesOnce() + // registerInterfacesOnce() buf := new(bytes.Buffer) enc := NewWALEncoder(buf) diff --git a/consensus/wire.go b/consensus/wire.go new file mode 100644 index 000000000..81223c689 --- /dev/null +++ b/consensus/wire.go @@ -0,0 +1,14 @@ +package consensus + +import ( + "github.com/tendermint/go-amino" + "github.com/tendermint/go-crypto" +) + +var cdc = amino.NewCodec() + +func init() { + RegisterConsensusMessages(cdc) + RegisterWALMessages(cdc) + crypto.RegisterAmino(cdc) +} diff --git a/types/block.go b/types/block.go index 1a9b47803..df39d3305 100644 --- a/types/block.go +++ b/types/block.go @@ -94,7 +94,9 @@ func (b *Block) Hash() cmn.HexBytes { // MakePartSet returns a PartSet containing parts of a serialized block. // This is the form in which the block is gossipped to peers. func (b *Block) MakePartSet(partSize int) *PartSet { - bz, err := cdc.MarshalBinaryBare(b) + // We prefix the byte length, so that unmarshaling + // can easily happen via a reader. + bz, err := cdc.MarshalBinary(b) if err != nil { panic(err) } diff --git a/types/priv_validator.go b/types/priv_validator.go index 06a024215..ecba4456f 100644 --- a/types/priv_validator.go +++ b/types/priv_validator.go @@ -87,3 +87,10 @@ func (pv *MockPV) SignHeartbeat(chainID string, heartbeat *Heartbeat) error { func (pv *MockPV) String() string { return fmt.Sprintf("MockPV{%v}", pv.GetAddress()) } + +// XXX: Implement. +func (pv *MockPV) DisableChecks() { + // Currently this does nothing, + // as MockPV has no safety checks at all. + return +} From 799beebd362874144f5ca69ee496ddba91e351a2 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 5 Apr 2018 17:54:26 +0300 Subject: [PATCH 062/143] fix consensus tests --- config/toml.go | 2 +- consensus/state.go | 7 ++++--- consensus/state_test.go | 5 ++--- consensus/wal.go | 1 - consensus/wal_test.go | 2 +- 5 files changed, 8 insertions(+), 9 deletions(-) diff --git a/config/toml.go b/config/toml.go index 882d2289a..23a6fecde 100644 --- a/config/toml.go +++ b/config/toml.go @@ -287,7 +287,7 @@ var testGenesis = `{ }` var testPrivValidator = `{ - "address": "D028C9981F7A87F3093672BF0D5B0E2A1B3ED456", + "address": "515252BC8788931ADFF4796A389C9B1F7925A1B7", "pub_key": { "type": "AC26791624DE60", "value": "AT/+aaL1eB0477Mud9JMm8Sh8BIvOYlPGC9KkIUmFaE=" diff --git a/consensus/state.go b/consensus/state.go index a64ef3e49..42de5fec8 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -4,14 +4,15 @@ import ( "bytes" "errors" "fmt" - fail "github.com/ebuchman/fail-test" - cmn "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tmlibs/log" "reflect" "runtime/debug" "sync" "time" + fail "github.com/ebuchman/fail-test" + cmn "github.com/tendermint/tmlibs/common" + "github.com/tendermint/tmlibs/log" + cfg "github.com/tendermint/tendermint/config" cstypes "github.com/tendermint/tendermint/consensus/types" "github.com/tendermint/tendermint/p2p" diff --git a/consensus/state_test.go b/consensus/state_test.go index fea6cf6ae..6efc21205 100644 --- a/consensus/state_test.go +++ b/consensus/state_test.go @@ -409,8 +409,7 @@ func TestStateLockNoPOL(t *testing.T) { validatePrevote(t, cs1, 1, vss[0], rs.LockedBlock.Hash()) // add a conflicting prevote from the other validator - fmt.Println(">>", rs.ProposalBlock) - signAddVotes(cs1, types.VoteTypePrevote, hash, rs.ProposalBlock.MakePartSet(partSize).Header(), vs2) + signAddVotes(cs1, types.VoteTypePrevote, hash, rs.LockedBlock.MakePartSet(partSize).Header(), vs2) <-voteCh // now we're going to enter prevote again, but with invalid args @@ -425,7 +424,7 @@ func TestStateLockNoPOL(t *testing.T) { // add conflicting precommit from vs2 // NOTE: in practice we should never get to a point where there are precommits for different blocks at the same round - signAddVotes(cs1, types.VoteTypePrecommit, hash, rs.ProposalBlock.MakePartSet(partSize).Header(), vs2) + signAddVotes(cs1, types.VoteTypePrecommit, hash, rs.LockedBlock.MakePartSet(partSize).Header(), vs2) <-voteCh // (note we're entering precommit for a second time this round, but with invalid args diff --git a/consensus/wal.go b/consensus/wal.go index a615720be..bc9afff64 100644 --- a/consensus/wal.go +++ b/consensus/wal.go @@ -113,7 +113,6 @@ func (wal *baseWAL) OnStop() { // called in newStep and for each pass in receiveRoutine func (wal *baseWAL) Save(msg WALMessage) { - fmt.Println("!!", 1) if wal == nil { return } diff --git a/consensus/wal_test.go b/consensus/wal_test.go index 8c95748eb..9925b691b 100644 --- a/consensus/wal_test.go +++ b/consensus/wal_test.go @@ -35,7 +35,7 @@ func TestWALEncoderDecoder(t *testing.T) { decoded, err := dec.Decode() require.NoError(t, err) - assert.Equal(t, msg.Time.Truncate(time.Millisecond), decoded.Time) + assert.True(t, msg.Time.Equal(decoded.Time)) assert.Equal(t, msg.Msg, decoded.Msg) } } From e4492afbad1af28c4ac33298681d368d66d37d66 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Thu, 5 Apr 2018 08:17:10 -0700 Subject: [PATCH 063/143] Merge --- Gopkg.lock | 7 +-- benchmarks/codec_test.go | 15 ++++-- blockchain/store.go | 2 +- cmd/tendermint/commands/show_validator.go | 8 ++-- cmd/tendermint/commands/wire.go | 12 +++++ consensus/common_test.go | 2 +- consensus/mempool_test.go | 2 +- consensus/reactor.go | 6 +-- consensus/reactor_test.go | 6 +-- consensus/state.go | 2 +- consensus/state_test.go | 28 ++++++------ consensus/wal_generator.go | 1 + consensus/wal_test.go | 2 +- lite/proxy/wrapper.go | 2 +- node/node.go | 14 +++--- rpc/client/event_test.go | 8 ++-- rpc/client/helpers.go | 4 +- rpc/core/events.go | 2 +- rpc/core/mempool.go | 2 +- rpc/test/helpers.go | 4 +- state/txindex/indexer_service.go | 2 +- types/event_bus.go | 28 ++++++------ types/event_bus_test.go | 2 +- types/events.go | 56 ++++------------------- 24 files changed, 97 insertions(+), 120 deletions(-) create mode 100644 cmd/tendermint/commands/wire.go diff --git a/Gopkg.lock b/Gopkg.lock index 19a15cc5d..c0704d722 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -265,10 +265,7 @@ [[projects]] name = "github.com/tendermint/go-wire" - packages = [ - ".", - "data" - ] + packages = ["."] revision = "fa721242b042ecd4c6ed1a934ee740db4f74e45c" version = "v0.7.3" @@ -386,6 +383,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "8cd32c0a5faec4d8cbca3d9ea5d97b0e4e90bfc2a9f0e6b71a4846cb6030be94" + inputs-digest = "0dacd2eb1550ca01e0c64f77b721eda1a381dde1d246a56bfe5a2746b78b7bad" solver-name = "gps-cdcl" solver-version = 1 diff --git a/benchmarks/codec_test.go b/benchmarks/codec_test.go index 8ac62a247..b34417992 100644 --- a/benchmarks/codec_test.go +++ b/benchmarks/codec_test.go @@ -5,7 +5,6 @@ import ( "time" "github.com/tendermint/go-crypto" - "github.com/tendermint/go-wire" proto "github.com/tendermint/tendermint/benchmarks/proto" "github.com/tendermint/tendermint/p2p" @@ -33,7 +32,10 @@ func BenchmarkEncodeStatusWire(b *testing.B) { counter := 0 for i := 0; i < b.N; i++ { - jsonBytes := wire.JSONBytes(status) + jsonBytes, err := cdc.MarshalJSON(status) + if err != nil { + panic(err) + } counter += len(jsonBytes) } @@ -54,7 +56,10 @@ func BenchmarkEncodeNodeInfoWire(b *testing.B) { counter := 0 for i := 0; i < b.N; i++ { - jsonBytes := wire.JSONBytes(nodeInfo) + jsonBytes, err := cdc.MarshalJSON(nodeInfo) + if err != nil { + panic(err) + } counter += len(jsonBytes) } } @@ -74,7 +79,7 @@ func BenchmarkEncodeNodeInfoBinary(b *testing.B) { counter := 0 for i := 0; i < b.N; i++ { - jsonBytes := wire.BinaryBytes(nodeInfo) + jsonBytes := cdc.MustMarshalBinaryBare(nodeInfo) counter += len(jsonBytes) } @@ -82,7 +87,7 @@ func BenchmarkEncodeNodeInfoBinary(b *testing.B) { func BenchmarkEncodeNodeInfoProto(b *testing.B) { b.StopTimer() - pubKey := crypto.GenPrivKeyEd25519().PubKey().Unwrap().(crypto.PubKeyEd25519) + pubKey := crypto.GenPrivKeyEd25519().PubKey().(crypto.PubKeyEd25519) pubKey2 := &proto.PubKey{Ed25519: &proto.PubKeyEd25519{Bytes: pubKey[:]}} nodeInfo := proto.NodeInfo{ PubKey: pubKey2, diff --git a/blockchain/store.go b/blockchain/store.go index 8210f1439..e322e4145 100644 --- a/blockchain/store.go +++ b/blockchain/store.go @@ -69,7 +69,7 @@ func (bs *BlockStore) LoadBlock(height int64) *types.Block { part := bs.LoadBlockPart(height, i) buf = append(buf, part.Bytes...) } - err = cdc.UnmarshalBinaryBare(buf, block) + err = cdc.UnmarshalBinary(buf, block) if err != nil { // NOTE: The existence of meta should imply the existence of the // block. So, make sure meta is only saved after blocks are saved. diff --git a/cmd/tendermint/commands/show_validator.go b/cmd/tendermint/commands/show_validator.go index 458f11c2d..6ead4bada 100644 --- a/cmd/tendermint/commands/show_validator.go +++ b/cmd/tendermint/commands/show_validator.go @@ -2,11 +2,9 @@ package commands import ( "fmt" - "github.com/spf13/cobra" - "github.com/tendermint/go-wire/data" - "github.com/tendermint/tendermint/types" + privval "github.com/tendermint/tendermint/types/priv_validator" ) // ShowValidatorCmd adds capabilities for showing the validator info. @@ -17,7 +15,7 @@ var ShowValidatorCmd = &cobra.Command{ } func showValidator(cmd *cobra.Command, args []string) { - privValidator := types.LoadOrGenPrivValidatorFS(config.PrivValidatorFile()) - pubKeyJSONBytes, _ := data.ToJSON(privValidator.PubKey) + privValidator := privval.LoadOrGenFilePV(config.PrivValidatorFile()) + pubKeyJSONBytes, _ := cdc.MarshalJSON(privValidator.GetPubKey()) fmt.Println(string(pubKeyJSONBytes)) } diff --git a/cmd/tendermint/commands/wire.go b/cmd/tendermint/commands/wire.go new file mode 100644 index 000000000..4c133a8c5 --- /dev/null +++ b/cmd/tendermint/commands/wire.go @@ -0,0 +1,12 @@ +package commands + +import ( + "github.com/tendermint/go-amino" + "github.com/tendermint/go-crypto" +) + +var cdc = amino.NewCodec() + +func init() { + crypto.RegisterAmino(cdc) +} diff --git a/consensus/common_test.go b/consensus/common_test.go index c8de40011..fe95210a9 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -223,7 +223,7 @@ func subscribeToVoter(cs *ConsensusState, addr []byte) chan interface{} { voteCh := make(chan interface{}) go func() { for v := range voteCh0 { - vote := v.(types.TMEventData).Unwrap().(types.EventDataVote) + vote := v.(types.EventDataVote) // we only fire for our own votes if bytes.Equal(addr, vote.Vote.ValidatorAddress) { voteCh <- v diff --git a/consensus/mempool_test.go b/consensus/mempool_test.go index d1714a741..823f98656 100644 --- a/consensus/mempool_test.go +++ b/consensus/mempool_test.go @@ -108,7 +108,7 @@ func TestMempoolTxConcurrentWithCommit(t *testing.T) { ticker := time.NewTicker(time.Second * 30) select { case b := <-newBlockCh: - evt := b.(types.TMEventData).Unwrap().(types.EventDataNewBlock) + evt := b.(types.EventDataNewBlock) nTxs += int(evt.Block.Header.NumTxs) case <-ticker.C: panic("Timed out waiting to commit blocks with transactions") diff --git a/consensus/reactor.go b/consensus/reactor.go index 3ab8995d7..85943c214 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -372,17 +372,17 @@ func (conR *ConsensusReactor) startBroadcastRoutine() error { select { case data, ok := <-stepsCh: if ok { // a receive from a closed channel returns the zero value immediately - edrs := data.(types.TMEventData).Unwrap().(types.EventDataRoundState) + edrs := data.(types.EventDataRoundState) conR.broadcastNewRoundStep(edrs.RoundState.(*cstypes.RoundState)) } case data, ok := <-votesCh: if ok { - edv := data.(types.TMEventData).Unwrap().(types.EventDataVote) + edv := data.(types.EventDataVote) conR.broadcastHasVoteMessage(edv.Vote) } case data, ok := <-heartbeatsCh: if ok { - edph := data.(types.TMEventData).Unwrap().(types.EventDataProposalHeartbeat) + edph := data.(types.EventDataProposalHeartbeat) conR.broadcastProposalHeartbeatMessage(edph) } case <-conR.Quit(): diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index 8e96de2bb..f6f941375 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -301,7 +301,7 @@ func waitForAndValidateBlock(t *testing.T, n int, activeVals map[string]struct{} if !ok { return } - newBlock := newBlockI.(types.TMEventData).Unwrap().(types.EventDataNewBlock).Block + newBlock := newBlockI.(types.EventDataNewBlock).Block css[j].Logger.Debug("waitForAndValidateBlock: Got block", "height", newBlock.Height) err := validateBlock(newBlock, activeVals) assert.Nil(t, err) @@ -322,7 +322,7 @@ func waitForAndValidateBlockWithTx(t *testing.T, n int, activeVals map[string]st if !ok { return } - newBlock := newBlockI.(types.TMEventData).Unwrap().(types.EventDataNewBlock).Block + newBlock := newBlockI.(types.EventDataNewBlock).Block css[j].Logger.Debug("waitForAndValidateBlockWithTx: Got block", "height", newBlock.Height) err := validateBlock(newBlock, activeVals) assert.Nil(t, err) @@ -354,7 +354,7 @@ func waitForBlockWithUpdatedValsAndValidateIt(t *testing.T, n int, updatedVals m if !ok { return } - newBlock = newBlockI.(types.TMEventData).Unwrap().(types.EventDataNewBlock).Block + newBlock = newBlockI.(types.EventDataNewBlock).Block if newBlock.LastCommit.Size() == len(updatedVals) { css[j].Logger.Debug("waitForBlockWithUpdatedValsAndValidateIt: Got block", "height", newBlock.Height) break LOOP diff --git a/consensus/state.go b/consensus/state.go index 42de5fec8..0f086172c 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -778,7 +778,7 @@ func (cs *ConsensusState) enterPropose(height int64, round int) { // if not a validator, we're done if !cs.Validators.HasAddress(cs.privValidator.GetAddress()) { - cs.Logger.Debug("This node is not a validator") + cs.Logger.Debug("This node is not a validator 2", cs.privValidator.GetAddress(), cs.Validators) return } cs.Logger.Debug("This node is a validator") diff --git a/consensus/state_test.go b/consensus/state_test.go index 6efc21205..0d7cad484 100644 --- a/consensus/state_test.go +++ b/consensus/state_test.go @@ -261,7 +261,7 @@ func TestStateFullRound1(t *testing.T) { // grab proposal re := <-propCh - propBlockHash := re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*cstypes.RoundState).ProposalBlock.Hash() + propBlockHash := re.(types.EventDataRoundState).RoundState.(*cstypes.RoundState).ProposalBlock.Hash() <-voteCh // wait for prevote validatePrevote(t, cs, round, vss[0], propBlockHash) @@ -356,7 +356,7 @@ func TestStateLockNoPOL(t *testing.T) { cs1.startRoutines(0) re := <-proposalCh - rs := re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*cstypes.RoundState) + rs := re.(types.EventDataRoundState).RoundState.(*cstypes.RoundState) theBlockHash := rs.ProposalBlock.Hash() <-voteCh // prevote @@ -396,7 +396,7 @@ func TestStateLockNoPOL(t *testing.T) { // now we're on a new round and not the proposer, so wait for timeout re = <-timeoutProposeCh - rs = re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*cstypes.RoundState) + rs = re.(types.EventDataRoundState).RoundState.(*cstypes.RoundState) if rs.ProposalBlock != nil { panic("Expected proposal block to be nil") @@ -440,7 +440,7 @@ func TestStateLockNoPOL(t *testing.T) { incrementRound(vs2) re = <-proposalCh - rs = re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*cstypes.RoundState) + rs = re.(types.EventDataRoundState).RoundState.(*cstypes.RoundState) // now we're on a new round and are the proposer if !bytes.Equal(rs.ProposalBlock.Hash(), rs.LockedBlock.Hash()) { @@ -529,7 +529,7 @@ func TestStateLockPOLRelock(t *testing.T) { <-newRoundCh re := <-proposalCh - rs := re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*cstypes.RoundState) + rs := re.(types.EventDataRoundState).RoundState.(*cstypes.RoundState) theBlockHash := rs.ProposalBlock.Hash() <-voteCh // prevote @@ -605,9 +605,9 @@ func TestStateLockPOLRelock(t *testing.T) { discardFromChan(voteCh, 2) be := <-newBlockCh - b := be.(types.TMEventData).Unwrap().(types.EventDataNewBlockHeader) + b := be.(types.EventDataNewBlockHeader) re = <-newRoundCh - rs = re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*cstypes.RoundState) + rs = re.(types.EventDataRoundState).RoundState.(*cstypes.RoundState) if rs.Height != 2 { panic("Expected height to increment") } @@ -643,7 +643,7 @@ func TestStateLockPOLUnlock(t *testing.T) { startTestRound(cs1, cs1.Height, 0) <-newRoundCh re := <-proposalCh - rs := re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*cstypes.RoundState) + rs := re.(types.EventDataRoundState).RoundState.(*cstypes.RoundState) theBlockHash := rs.ProposalBlock.Hash() <-voteCh // prevote @@ -669,7 +669,7 @@ func TestStateLockPOLUnlock(t *testing.T) { // timeout to new round re = <-timeoutWaitCh - rs = re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*cstypes.RoundState) + rs = re.(types.EventDataRoundState).RoundState.(*cstypes.RoundState) lockedBlockHash := rs.LockedBlock.Hash() //XXX: this isnt guaranteed to get there before the timeoutPropose ... @@ -731,7 +731,7 @@ func TestStateLockPOLSafety1(t *testing.T) { startTestRound(cs1, cs1.Height, 0) <-newRoundCh re := <-proposalCh - rs := re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*cstypes.RoundState) + rs := re.(types.EventDataRoundState).RoundState.(*cstypes.RoundState) propBlock := rs.ProposalBlock <-voteCh // prevote @@ -781,7 +781,7 @@ func TestStateLockPOLSafety1(t *testing.T) { re = <-proposalCh } - rs = re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*cstypes.RoundState) + rs = re.(types.EventDataRoundState).RoundState.(*cstypes.RoundState) if rs.LockedBlock != nil { panic("we should not be locked!") @@ -1033,7 +1033,7 @@ func TestStateHalt1(t *testing.T) { startTestRound(cs1, cs1.Height, 0) <-newRoundCh re := <-proposalCh - rs := re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*cstypes.RoundState) + rs := re.(types.EventDataRoundState).RoundState.(*cstypes.RoundState) propBlock := rs.ProposalBlock propBlockParts := propBlock.MakePartSet(partSize) @@ -1056,7 +1056,7 @@ func TestStateHalt1(t *testing.T) { // timeout to new round <-timeoutWaitCh re = <-newRoundCh - rs = re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*cstypes.RoundState) + rs = re.(types.EventDataRoundState).RoundState.(*cstypes.RoundState) t.Log("### ONTO ROUND 1") /*Round2 @@ -1074,7 +1074,7 @@ func TestStateHalt1(t *testing.T) { // receiving that precommit should take us straight to commit <-newBlockCh re = <-newRoundCh - rs = re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*cstypes.RoundState) + rs = re.(types.EventDataRoundState).RoundState.(*cstypes.RoundState) if rs.Height != 2 { panic("expected height to increment") diff --git a/consensus/wal_generator.go b/consensus/wal_generator.go index bb0b5296c..65de399d4 100644 --- a/consensus/wal_generator.go +++ b/consensus/wal_generator.go @@ -72,6 +72,7 @@ func WALWithNBlocks(numBlocks int) (data []byte, err error) { consensusState := NewConsensusState(config.Consensus, state.Copy(), blockExec, blockStore, mempool, evpool) consensusState.SetLogger(logger) consensusState.SetEventBus(eventBus) + fmt.Println(">>privval", privValidator) if privValidator != nil { consensusState.SetPrivValidator(privValidator) } diff --git a/consensus/wal_test.go b/consensus/wal_test.go index 9925b691b..45af36e12 100644 --- a/consensus/wal_test.go +++ b/consensus/wal_test.go @@ -35,7 +35,7 @@ func TestWALEncoderDecoder(t *testing.T) { decoded, err := dec.Decode() require.NoError(t, err) - assert.True(t, msg.Time.Equal(decoded.Time)) + assert.Equal(t, msg.Time.UTC(), decoded.Time) assert.Equal(t, msg.Msg, decoded.Msg) } } diff --git a/lite/proxy/wrapper.go b/lite/proxy/wrapper.go index e8aa011e1..5fb12a40a 100644 --- a/lite/proxy/wrapper.go +++ b/lite/proxy/wrapper.go @@ -146,7 +146,7 @@ func (w Wrapper) Commit(height *int64) (*ctypes.ResultCommit, error) { // } // // check to validate it if possible, and drop if not valid -// switch t := tm.Unwrap().(type) { +// switch t := tm.(type) { // case types.EventDataNewBlockHeader: // err := verifyHeader(s.client, t.Header) // if err != nil { diff --git a/node/node.go b/node/node.go index dffdb83e8..be744f845 100644 --- a/node/node.go +++ b/node/node.go @@ -10,8 +10,8 @@ import ( "strings" abci "github.com/tendermint/abci/types" + amino "github.com/tendermint/go-amino" crypto "github.com/tendermint/go-crypto" - wire "github.com/tendermint/go-wire" cmn "github.com/tendermint/tmlibs/common" dbm "github.com/tendermint/tmlibs/db" "github.com/tendermint/tmlibs/log" @@ -34,7 +34,7 @@ import ( "github.com/tendermint/tendermint/state/txindex/kv" "github.com/tendermint/tendermint/state/txindex/null" "github.com/tendermint/tendermint/types" - priv_val "github.com/tendermint/tendermint/types/priv_validator" + privval "github.com/tendermint/tendermint/types/priv_validator" "github.com/tendermint/tendermint/version" _ "net/http/pprof" @@ -79,7 +79,7 @@ type NodeProvider func(*cfg.Config, log.Logger) (*Node, error) // It implements NodeProvider. func DefaultNewNode(config *cfg.Config, logger log.Logger) (*Node, error) { return NewNode(config, - types.LoadOrGenPrivValidatorFS(config.PrivValidatorFile()), + privval.LoadOrGenFilePV(config.PrivValidatorFile()), proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir()), DefaultGenesisDocProviderFunc(config), DefaultDBProvider, @@ -180,8 +180,8 @@ func NewNode(config *cfg.Config, // TODO: persist this key so external signer // can actually authenticate us privKey = crypto.GenPrivKeyEd25519() - pvsc = priv_val.NewSocketClient( - logger.With("module", "priv_val"), + pvsc = privval.NewSocketPV( + logger.With("module", "privval"), config.PrivValidatorListenAddr, privKey, ) @@ -445,7 +445,7 @@ func (n *Node) OnStop() { n.eventBus.Stop() n.indexerService.Stop() - if pvsc, ok := n.privValidator.(*priv_val.SocketClient); ok { + if pvsc, ok := n.privValidator.(*privval.SocketPV); ok { if err := pvsc.Stop(); err != nil { n.Logger.Error("Error stopping priv validator socket client", "err", err) } @@ -591,7 +591,7 @@ func (n *Node) makeNodeInfo(pubKey crypto.PubKey) p2p.NodeInfo { }, Moniker: n.config.Moniker, Other: []string{ - cmn.Fmt("wire_version=%v", wire.Version), + cmn.Fmt("amino_version=%v", amino.Version), cmn.Fmt("p2p_version=%v", p2p.Version), cmn.Fmt("consensus_version=%v", cs.Version), cmn.Fmt("rpc_version=%v/%v", rpc.Version, rpccore.Version), diff --git a/rpc/client/event_test.go b/rpc/client/event_test.go index 40a42c186..e90ace43f 100644 --- a/rpc/client/event_test.go +++ b/rpc/client/event_test.go @@ -36,7 +36,7 @@ func TestHeaderEvents(t *testing.T) { evtTyp := types.EventNewBlockHeader evt, err := client.WaitForOneEvent(c, evtTyp, waitForEventTimeout) require.Nil(err, "%d: %+v", i, err) - _, ok := evt.Unwrap().(types.EventDataNewBlockHeader) + _, ok := evt.(types.EventDataNewBlockHeader) require.True(ok, "%d: %#v", i, evt) // TODO: more checks... } @@ -59,7 +59,7 @@ func TestBlockEvents(t *testing.T) { evtTyp := types.EventNewBlock evt, err := client.WaitForOneEvent(c, evtTyp, waitForEventTimeout) require.Nil(err, "%d: %+v", j, err) - blockEvent, ok := evt.Unwrap().(types.EventDataNewBlock) + blockEvent, ok := evt.(types.EventDataNewBlock) require.True(ok, "%d: %#v", j, evt) block := blockEvent.Block @@ -97,7 +97,7 @@ func TestTxEventsSentWithBroadcastTxAsync(t *testing.T) { evt, err := client.WaitForOneEvent(c, evtTyp, waitForEventTimeout) require.Nil(err, "%d: %+v", i, err) // and make sure it has the proper info - txe, ok := evt.Unwrap().(types.EventDataTx) + txe, ok := evt.(types.EventDataTx) require.True(ok, "%d: %#v", i, evt) // make sure this is the proper tx require.EqualValues(tx, txe.Tx) @@ -129,7 +129,7 @@ func TestTxEventsSentWithBroadcastTxSync(t *testing.T) { evt, err := client.WaitForOneEvent(c, evtTyp, waitForEventTimeout) require.Nil(err, "%d: %+v", i, err) // and make sure it has the proper info - txe, ok := evt.Unwrap().(types.EventDataTx) + txe, ok := evt.(types.EventDataTx) require.True(ok, "%d: %#v", i, evt) // make sure this is the proper tx require.EqualValues(tx, txe.Tx) diff --git a/rpc/client/helpers.go b/rpc/client/helpers.go index e7a84b6b4..86c65919d 100644 --- a/rpc/client/helpers.go +++ b/rpc/client/helpers.go @@ -65,7 +65,7 @@ func WaitForOneEvent(c EventsClient, evtTyp string, timeout time.Duration) (type query := types.QueryForEvent(evtTyp) err := c.Subscribe(ctx, subscriber, query, evts) if err != nil { - return types.TMEventData{}, errors.Wrap(err, "failed to subscribe") + return nil, errors.Wrap(err, "failed to subscribe") } // make sure to unregister after the test is over @@ -75,6 +75,6 @@ func WaitForOneEvent(c EventsClient, evtTyp string, timeout time.Duration) (type case evt := <-evts: return evt.(types.TMEventData), nil case <-ctx.Done(): - return types.TMEventData{}, errors.New("timed out waiting for event") + return nil, errors.New("timed out waiting for event") } } diff --git a/rpc/core/events.go b/rpc/core/events.go index 9353ace6a..beeaf4c90 100644 --- a/rpc/core/events.go +++ b/rpc/core/events.go @@ -61,7 +61,7 @@ import ( // // go func() { // for e := range txs { -// fmt.Println("got ", e.(types.TMEventData).Unwrap().(types.EventDataTx)) +// fmt.Println("got ", e.(types.EventDataTx)) // } // }() // ``` diff --git a/rpc/core/mempool.go b/rpc/core/mempool.go index 1dbdd8012..77c8c8446 100644 --- a/rpc/core/mempool.go +++ b/rpc/core/mempool.go @@ -189,7 +189,7 @@ func BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { timer := time.NewTimer(60 * 2 * time.Second) select { case deliverTxResMsg := <-deliverTxResCh: - deliverTxRes := deliverTxResMsg.(types.TMEventData).Unwrap().(types.EventDataTx) + deliverTxRes := deliverTxResMsg.(types.EventDataTx) // The tx was included in a block. deliverTxR := deliverTxRes.Result logger.Info("DeliverTx passed ", "tx", cmn.HexBytes(tx), "response", deliverTxR) diff --git a/rpc/test/helpers.go b/rpc/test/helpers.go index b67d76349..c9dc6d219 100644 --- a/rpc/test/helpers.go +++ b/rpc/test/helpers.go @@ -18,7 +18,7 @@ import ( ctypes "github.com/tendermint/tendermint/rpc/core/types" core_grpc "github.com/tendermint/tendermint/rpc/grpc" rpcclient "github.com/tendermint/tendermint/rpc/lib/client" - "github.com/tendermint/tendermint/types" + privval "github.com/tendermint/tendermint/types/priv_validator" ) var globalConfig *cfg.Config @@ -113,7 +113,7 @@ func NewTendermint(app abci.Application) *nm.Node { logger := log.NewTMLogger(log.NewSyncWriter(os.Stdout)) logger = log.NewFilter(logger, log.AllowError()) privValidatorFile := config.PrivValidatorFile() - privValidator := types.LoadOrGenPrivValidatorFS(privValidatorFile) + privValidator := privval.LoadOrGenFilePV(privValidatorFile) papp := proxy.NewLocalClientCreator(app) node, err := nm.NewNode(config, privValidator, papp, nm.DefaultGenesisDocProviderFunc(config), diff --git a/state/txindex/indexer_service.go b/state/txindex/indexer_service.go index 3e5fab127..f5420f631 100644 --- a/state/txindex/indexer_service.go +++ b/state/txindex/indexer_service.go @@ -34,7 +34,7 @@ func (is *IndexerService) OnStart() error { go func() { for event := range ch { // TODO: may be not perfomant to write one event at a time - txResult := event.(types.TMEventData).Unwrap().(types.EventDataTx).TxResult + txResult := event.(types.EventDataTx).TxResult is.idr.Index(&txResult) } }() diff --git a/types/event_bus.go b/types/event_bus.go index 4edaea588..3e16500b4 100644 --- a/types/event_bus.go +++ b/types/event_bus.go @@ -74,15 +74,15 @@ func (b *EventBus) Publish(eventType string, eventData TMEventData) error { //--- block, tx, and vote events func (b *EventBus) PublishEventNewBlock(event EventDataNewBlock) error { - return b.Publish(EventNewBlock, TMEventData{event}) + return b.Publish(EventNewBlock, event) } func (b *EventBus) PublishEventNewBlockHeader(event EventDataNewBlockHeader) error { - return b.Publish(EventNewBlockHeader, TMEventData{event}) + return b.Publish(EventNewBlockHeader, event) } func (b *EventBus) PublishEventVote(event EventDataVote) error { - return b.Publish(EventVote, TMEventData{event}) + return b.Publish(EventVote, event) } // PublishEventTx publishes tx event with tags from Result. Note it will add @@ -114,50 +114,50 @@ func (b *EventBus) PublishEventTx(event EventDataTx) error { logIfTagExists(TxHeightKey, tags, b.Logger) tags[TxHeightKey] = event.Height - b.pubsub.PublishWithTags(ctx, TMEventData{event}, tags) + b.pubsub.PublishWithTags(ctx, event, tags) return nil } func (b *EventBus) PublishEventProposalHeartbeat(event EventDataProposalHeartbeat) error { - return b.Publish(EventProposalHeartbeat, TMEventData{event}) + return b.Publish(EventProposalHeartbeat, event) } //--- EventDataRoundState events func (b *EventBus) PublishEventNewRoundStep(event EventDataRoundState) error { - return b.Publish(EventNewRoundStep, TMEventData{event}) + return b.Publish(EventNewRoundStep, event) } func (b *EventBus) PublishEventTimeoutPropose(event EventDataRoundState) error { - return b.Publish(EventTimeoutPropose, TMEventData{event}) + return b.Publish(EventTimeoutPropose, event) } func (b *EventBus) PublishEventTimeoutWait(event EventDataRoundState) error { - return b.Publish(EventTimeoutWait, TMEventData{event}) + return b.Publish(EventTimeoutWait, event) } func (b *EventBus) PublishEventNewRound(event EventDataRoundState) error { - return b.Publish(EventNewRound, TMEventData{event}) + return b.Publish(EventNewRound, event) } func (b *EventBus) PublishEventCompleteProposal(event EventDataRoundState) error { - return b.Publish(EventCompleteProposal, TMEventData{event}) + return b.Publish(EventCompleteProposal, event) } func (b *EventBus) PublishEventPolka(event EventDataRoundState) error { - return b.Publish(EventPolka, TMEventData{event}) + return b.Publish(EventPolka, event) } func (b *EventBus) PublishEventUnlock(event EventDataRoundState) error { - return b.Publish(EventUnlock, TMEventData{event}) + return b.Publish(EventUnlock, event) } func (b *EventBus) PublishEventRelock(event EventDataRoundState) error { - return b.Publish(EventRelock, TMEventData{event}) + return b.Publish(EventRelock, event) } func (b *EventBus) PublishEventLock(event EventDataRoundState) error { - return b.Publish(EventLock, TMEventData{event}) + return b.Publish(EventLock, event) } func logIfTagExists(tag string, tags map[string]interface{}, logger log.Logger) { diff --git a/types/event_bus_test.go b/types/event_bus_test.go index aa97092f6..9002b531c 100644 --- a/types/event_bus_test.go +++ b/types/event_bus_test.go @@ -73,7 +73,7 @@ func benchmarkEventBus(numClients int, randQueries bool, randEvents bool, b *tes eventType = randEvent() } - eventBus.Publish(eventType, TMEventData{"Gamora"}) + eventBus.Publish(eventType, "Gamora") } } diff --git a/types/events.go b/types/events.go index d6f7b012c..1323b65bb 100644 --- a/types/events.go +++ b/types/events.go @@ -3,7 +3,7 @@ package types import ( "fmt" - "github.com/tendermint/go-wire/data" + "github.com/tendermint/go-amino" tmpubsub "github.com/tendermint/tmlibs/pubsub" tmquery "github.com/tendermint/tmlibs/pubsub/query" ) @@ -45,56 +45,20 @@ var ( ) // implements events.EventData -type TMEventDataInner interface { +type TMEventData interface { // empty interface } -type TMEventData struct { - TMEventDataInner `json:"unwrap"` +func RegisterEventDatas(cdc *amino.Codec) { + cdc.RegisterInterface((*TMEventData)(nil), nil) + cdc.RegisterConcrete(EventDataNewBlock{}, "tendermint/EventDataNameNewBlock", nil) + cdc.RegisterConcrete(EventDataNewBlockHeader{}, "tendermint/EventDataNameNewBlockHeader", nil) + cdc.RegisterConcrete(EventDataTx{}, "tendermint/EventDataNameTx", nil) + cdc.RegisterConcrete(EventDataRoundState{}, "tendermint/EventDataNameRoundState", nil) + cdc.RegisterConcrete(EventDataVote{}, "tendermint/EventDataNameVote", nil) + cdc.RegisterConcrete(EventDataProposalHeartbeat{}, "tendermint/EventDataNameProposalHeartbeat", nil) } -func (tmr TMEventData) MarshalJSON() ([]byte, error) { - return tmEventDataMapper.ToJSON(tmr.TMEventDataInner) -} - -func (tmr *TMEventData) UnmarshalJSON(data []byte) (err error) { - parsed, err := tmEventDataMapper.FromJSON(data) - if err == nil && parsed != nil { - tmr.TMEventDataInner = parsed.(TMEventDataInner) - } - return -} - -func (tmr TMEventData) Unwrap() TMEventDataInner { - tmrI := tmr.TMEventDataInner - for wrap, ok := tmrI.(TMEventData); ok; wrap, ok = tmrI.(TMEventData) { - tmrI = wrap.TMEventDataInner - } - return tmrI -} - -func (tmr TMEventData) Empty() bool { - return tmr.TMEventDataInner == nil -} - -const ( - EventDataTypeNewBlock = byte(0x01) - EventDataTypeFork = byte(0x02) - EventDataTypeTx = byte(0x03) - EventDataTypeNewBlockHeader = byte(0x04) - EventDataTypeRoundState = byte(0x11) - EventDataTypeVote = byte(0x12) - EventDataTypeProposalHeartbeat = byte(0x20) -) - -var tmEventDataMapper = data.NewMapper(TMEventData{}). - RegisterImplementation(EventDataNewBlock{}, EventDataNameNewBlock, EventDataTypeNewBlock). - RegisterImplementation(EventDataNewBlockHeader{}, EventDataNameNewBlockHeader, EventDataTypeNewBlockHeader). - RegisterImplementation(EventDataTx{}, EventDataNameTx, EventDataTypeTx). - RegisterImplementation(EventDataRoundState{}, EventDataNameRoundState, EventDataTypeRoundState). - RegisterImplementation(EventDataVote{}, EventDataNameVote, EventDataTypeVote). - RegisterImplementation(EventDataProposalHeartbeat{}, EventDataNameProposalHeartbeat, EventDataTypeProposalHeartbeat) - // Most event messages are basic types (a block, a transaction) // but some (an input to a call tx or a receive) are more exotic From 3037b5b7ca7f0a947c11e6fd7b2a26ad68be1e9a Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Thu, 5 Apr 2018 15:45:11 -0700 Subject: [PATCH 064/143] Fix rpc/lib/... --- rpc/lib/client/args_test.go | 6 ++- rpc/lib/client/http_client.go | 41 +++++++++------ rpc/lib/client/ws_client.go | 11 +++- rpc/lib/rpc_test.go | 12 +++-- rpc/lib/server/handlers.go | 89 ++++++++++++++++++--------------- rpc/lib/server/handlers_test.go | 4 +- rpc/lib/server/parse_test.go | 4 +- rpc/lib/test/main.go | 4 +- rpc/lib/types/types.go | 29 ++++++++--- rpc/lib/types/types_test.go | 4 +- 10 files changed, 132 insertions(+), 72 deletions(-) diff --git a/rpc/lib/client/args_test.go b/rpc/lib/client/args_test.go index ccabd0d2c..4442ac2bc 100644 --- a/rpc/lib/client/args_test.go +++ b/rpc/lib/client/args_test.go @@ -5,6 +5,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/tendermint/go-amino" ) type Tx []byte @@ -27,9 +29,11 @@ func TestArgToJSON(t *testing.T) { {Foo{7, "hello"}, `{"Bar":7,"Baz":"hello"}`}, } + cdc := amino.NewCodec() + for i, tc := range cases { args := map[string]interface{}{"data": tc.input} - err := argsToJson(args) + err := argsToJSON(cdc, args) require.Nil(err, "%d: %+v", i, err) require.Equal(1, len(args), "%d", i) data, ok := args["data"].(string) diff --git a/rpc/lib/client/http_client.go b/rpc/lib/client/http_client.go index 902b7eebc..a41f3125c 100644 --- a/rpc/lib/client/http_client.go +++ b/rpc/lib/client/http_client.go @@ -12,6 +12,7 @@ import ( "strings" "github.com/pkg/errors" + "github.com/tendermint/go-amino" types "github.com/tendermint/tendermint/rpc/lib/types" ) @@ -19,6 +20,7 @@ import ( // HTTPClient is a common interface for JSONRPCClient and URIClient. type HTTPClient interface { Call(method string, params map[string]interface{}, result interface{}) (interface{}, error) + Codec() *amino.Codec } // TODO: Deprecate support for IP:PORT or /path/to/socket @@ -66,6 +68,7 @@ func makeHTTPClient(remoteAddr string) (string, *http.Client) { type JSONRPCClient struct { address string client *http.Client + cdc *amino.Codec } // NewJSONRPCClient returns a JSONRPCClient pointed at the given address. @@ -74,11 +77,12 @@ func NewJSONRPCClient(remote string) *JSONRPCClient { return &JSONRPCClient{ address: address, client: client, + cdc: amino.NewCodec(), } } func (c *JSONRPCClient) Call(method string, params map[string]interface{}, result interface{}) (interface{}, error) { - request, err := types.MapToRequest("jsonrpc-client", method, params) + request, err := types.MapToRequest(c.cdc, "jsonrpc-client", method, params) if err != nil { return nil, err } @@ -100,7 +104,11 @@ func (c *JSONRPCClient) Call(method string, params map[string]interface{}, resul return nil, err } // log.Info(Fmt("RPC response: %v", string(responseBytes))) - return unmarshalResponseBytes(responseBytes, result) + return unmarshalResponseBytes(c.cdc, responseBytes, result) +} + +func (c *JSONRPCClient) Codec() *amino.Codec { + return c.cdc } //------------------------------------------------------------- @@ -109,6 +117,7 @@ func (c *JSONRPCClient) Call(method string, params map[string]interface{}, resul type URIClient struct { address string client *http.Client + cdc *amino.Codec } func NewURIClient(remote string) *URIClient { @@ -116,11 +125,12 @@ func NewURIClient(remote string) *URIClient { return &URIClient{ address: address, client: client, + cdc: amino.NewCodec(), } } func (c *URIClient) Call(method string, params map[string]interface{}, result interface{}) (interface{}, error) { - values, err := argsToURLValues(params) + values, err := argsToURLValues(c.cdc, params) if err != nil { return nil, err } @@ -135,15 +145,18 @@ func (c *URIClient) Call(method string, params map[string]interface{}, result in if err != nil { return nil, err } - return unmarshalResponseBytes(responseBytes, result) + return unmarshalResponseBytes(c.cdc, responseBytes, result) +} + +func (c *URIClient) Codec() *amino.Codec { + return c.cdc } //------------------------------------------------ -func unmarshalResponseBytes(responseBytes []byte, result interface{}) (interface{}, error) { - // read response - // if rpc/core/types is imported, the result will unmarshal - // into the correct type +func unmarshalResponseBytes(cdc *amino.Codec, responseBytes []byte, result interface{}) (interface{}, error) { + // Read response. If rpc/core/types is imported, the result will unmarshal + // into the correct type. // log.Notice("response", "response", string(responseBytes)) var err error response := &types.RPCResponse{} @@ -154,20 +167,20 @@ func unmarshalResponseBytes(responseBytes []byte, result interface{}) (interface if response.Error != nil { return nil, errors.Errorf("Response error: %v", response.Error) } - // unmarshal the RawMessage into the result - err = json.Unmarshal(response.Result, result) + // Unmarshal the RawMessage into the result. + err = cdc.UnmarshalJSON(response.Result, result) if err != nil { return nil, errors.Errorf("Error unmarshalling rpc response result: %v", err) } return result, nil } -func argsToURLValues(args map[string]interface{}) (url.Values, error) { +func argsToURLValues(cdc *amino.Codec, args map[string]interface{}) (url.Values, error) { values := make(url.Values) if len(args) == 0 { return values, nil } - err := argsToJson(args) + err := argsToJSON(cdc, args) if err != nil { return nil, err } @@ -177,7 +190,7 @@ func argsToURLValues(args map[string]interface{}) (url.Values, error) { return values, nil } -func argsToJson(args map[string]interface{}) error { +func argsToJSON(cdc *amino.Codec, args map[string]interface{}) error { for k, v := range args { rt := reflect.TypeOf(v) isByteSlice := rt.Kind() == reflect.Slice && rt.Elem().Kind() == reflect.Uint8 @@ -187,7 +200,7 @@ func argsToJson(args map[string]interface{}) error { continue } - data, err := json.Marshal(v) + data, err := cdc.MarshalJSON(v) if err != nil { return err } diff --git a/rpc/lib/client/ws_client.go b/rpc/lib/client/ws_client.go index fe15cda21..43aa02f40 100644 --- a/rpc/lib/client/ws_client.go +++ b/rpc/lib/client/ws_client.go @@ -14,6 +14,7 @@ import ( "github.com/pkg/errors" metrics "github.com/rcrowley/go-metrics" + "github.com/tendermint/go-amino" types "github.com/tendermint/tendermint/rpc/lib/types" cmn "github.com/tendermint/tmlibs/common" ) @@ -31,6 +32,7 @@ type WSClient struct { cmn.BaseService conn *websocket.Conn + cdc *amino.Codec Address string // IP:PORT or /path/to/socket Endpoint string // /websocket/url/endpoint @@ -77,6 +79,7 @@ type WSClient struct { func NewWSClient(remoteAddr, endpoint string, options ...func(*WSClient)) *WSClient { addr, dialer := makeHTTPDialer(remoteAddr) c := &WSClient{ + cdc: amino.NewCodec(), Address: addr, Dialer: dialer, Endpoint: endpoint, @@ -206,7 +209,7 @@ func (c *WSClient) Send(ctx context.Context, request types.RPCRequest) error { // Call the given method. See Send description. func (c *WSClient) Call(ctx context.Context, method string, params map[string]interface{}) error { - request, err := types.MapToRequest("ws-client", method, params) + request, err := types.MapToRequest(c.cdc, "ws-client", method, params) if err != nil { return err } @@ -216,13 +219,17 @@ func (c *WSClient) Call(ctx context.Context, method string, params map[string]in // CallWithArrayParams the given method with params in a form of array. See // Send description. func (c *WSClient) CallWithArrayParams(ctx context.Context, method string, params []interface{}) error { - request, err := types.ArrayToRequest("ws-client", method, params) + request, err := types.ArrayToRequest(c.cdc, "ws-client", method, params) if err != nil { return err } return c.Send(ctx, request) } +func (c *WSClient) Codec() *amino.Codec { + return c.cdc +} + /////////////////////////////////////////////////////////////////////////////// // Private methods diff --git a/rpc/lib/rpc_test.go b/rpc/lib/rpc_test.go index cadc68f8e..f34b09f68 100644 --- a/rpc/lib/rpc_test.go +++ b/rpc/lib/rpc_test.go @@ -17,6 +17,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/tendermint/go-amino" cmn "github.com/tendermint/tmlibs/common" "github.com/tendermint/tmlibs/log" @@ -60,6 +61,9 @@ var Routes = map[string]*server.RPCFunc{ "echo_int": server.NewRPCFunc(EchoIntResult, "arg"), } +// Amino codec required to encode/decode everything above. +var RoutesCdc = amino.NewCodec() + func EchoResult(v string) (*ResultEcho, error) { return &ResultEcho{v}, nil } @@ -114,8 +118,8 @@ func setup() { tcpLogger := logger.With("socket", "tcp") mux := http.NewServeMux() - server.RegisterRPCFuncs(mux, Routes, tcpLogger) - wm := server.NewWebsocketManager(Routes, server.ReadWait(5*time.Second), server.PingPeriod(1*time.Second)) + server.RegisterRPCFuncs(mux, Routes, RoutesCdc, tcpLogger) + wm := server.NewWebsocketManager(Routes, RoutesCdc, server.ReadWait(5*time.Second), server.PingPeriod(1*time.Second)) wm.SetLogger(tcpLogger) mux.HandleFunc(websocketEndpoint, wm.WebsocketHandler) go func() { @@ -127,8 +131,8 @@ func setup() { unixLogger := logger.With("socket", "unix") mux2 := http.NewServeMux() - server.RegisterRPCFuncs(mux2, Routes, unixLogger) - wm = server.NewWebsocketManager(Routes) + server.RegisterRPCFuncs(mux2, Routes, RoutesCdc, unixLogger) + wm = server.NewWebsocketManager(Routes, RoutesCdc) wm.SetLogger(unixLogger) mux2.HandleFunc(websocketEndpoint, wm.WebsocketHandler) go func() { diff --git a/rpc/lib/server/handlers.go b/rpc/lib/server/handlers.go index 19fc0f6e4..31aaaedaf 100644 --- a/rpc/lib/server/handlers.go +++ b/rpc/lib/server/handlers.go @@ -17,6 +17,7 @@ import ( "github.com/gorilla/websocket" "github.com/pkg/errors" + "github.com/tendermint/go-amino" types "github.com/tendermint/tendermint/rpc/lib/types" cmn "github.com/tendermint/tmlibs/common" "github.com/tendermint/tmlibs/log" @@ -24,14 +25,14 @@ import ( // RegisterRPCFuncs adds a route for each function in the funcMap, as well as general jsonrpc and websocket handlers for all functions. // "result" is the interface on which the result objects are registered, and is popualted with every RPCResponse -func RegisterRPCFuncs(mux *http.ServeMux, funcMap map[string]*RPCFunc, logger log.Logger) { +func RegisterRPCFuncs(mux *http.ServeMux, funcMap map[string]*RPCFunc, cdc *amino.Codec, logger log.Logger) { // HTTP endpoints for funcName, rpcFunc := range funcMap { - mux.HandleFunc("/"+funcName, makeHTTPHandler(rpcFunc, logger)) + mux.HandleFunc("/"+funcName, makeHTTPHandler(rpcFunc, cdc, logger)) } // JSONRPC endpoints - mux.HandleFunc("/", makeJSONRPCHandler(funcMap, logger)) + mux.HandleFunc("/", makeJSONRPCHandler(funcMap, cdc, logger)) } //------------------------------------- @@ -98,7 +99,7 @@ func funcReturnTypes(f interface{}) []reflect.Type { // rpc.json // jsonrpc calls grab the given method's function info and runs reflect.Call -func makeJSONRPCHandler(funcMap map[string]*RPCFunc, logger log.Logger) http.HandlerFunc { +func makeJSONRPCHandler(funcMap map[string]*RPCFunc, cdc *amino.Codec, logger log.Logger) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { b, err := ioutil.ReadAll(r.Body) if err != nil { @@ -135,7 +136,7 @@ func makeJSONRPCHandler(funcMap map[string]*RPCFunc, logger log.Logger) http.Han } var args []reflect.Value if len(request.Params) > 0 { - args, err = jsonParamsToArgsRPC(rpcFunc, request.Params) + args, err = jsonParamsToArgsRPC(rpcFunc, cdc, request.Params) if err != nil { WriteRPCResponseHTTP(w, types.RPCInvalidParamsError(request.ID, errors.Wrap(err, "Error converting json params to arguments"))) return @@ -148,18 +149,18 @@ func makeJSONRPCHandler(funcMap map[string]*RPCFunc, logger log.Logger) http.Han WriteRPCResponseHTTP(w, types.RPCInternalError(request.ID, err)) return } - WriteRPCResponseHTTP(w, types.NewRPCSuccessResponse(request.ID, result)) + WriteRPCResponseHTTP(w, types.NewRPCSuccessResponse(cdc, request.ID, result)) } } -func mapParamsToArgs(rpcFunc *RPCFunc, params map[string]*json.RawMessage, argsOffset int) ([]reflect.Value, error) { +func mapParamsToArgs(rpcFunc *RPCFunc, cdc *amino.Codec, params map[string]json.RawMessage, argsOffset int) ([]reflect.Value, error) { values := make([]reflect.Value, len(rpcFunc.argNames)) for i, argName := range rpcFunc.argNames { argType := rpcFunc.args[i+argsOffset] - if p, ok := params[argName]; ok && p != nil && len(*p) > 0 { + if p, ok := params[argName]; ok && p != nil && len(p) > 0 { val := reflect.New(argType) - err := json.Unmarshal(*p, val.Interface()) + err := cdc.UnmarshalJSON(p, val.Interface()) if err != nil { return nil, err } @@ -172,7 +173,7 @@ func mapParamsToArgs(rpcFunc *RPCFunc, params map[string]*json.RawMessage, argsO return values, nil } -func arrayParamsToArgs(rpcFunc *RPCFunc, params []*json.RawMessage, argsOffset int) ([]reflect.Value, error) { +func arrayParamsToArgs(rpcFunc *RPCFunc, cdc *amino.Codec, params []json.RawMessage, argsOffset int) ([]reflect.Value, error) { if len(rpcFunc.argNames) != len(params) { return nil, errors.Errorf("Expected %v parameters (%v), got %v (%v)", len(rpcFunc.argNames), rpcFunc.argNames, len(params), params) @@ -182,7 +183,7 @@ func arrayParamsToArgs(rpcFunc *RPCFunc, params []*json.RawMessage, argsOffset i for i, p := range params { argType := rpcFunc.args[i+argsOffset] val := reflect.New(argType) - err := json.Unmarshal(*p, val.Interface()) + err := cdc.UnmarshalJSON(p, val.Interface()) if err != nil { return nil, err } @@ -191,39 +192,41 @@ func arrayParamsToArgs(rpcFunc *RPCFunc, params []*json.RawMessage, argsOffset i return values, nil } -// raw is unparsed json (from json.RawMessage) encoding either a map or an array. +// `raw` is unparsed json (from json.RawMessage) encoding either a map or an array. +// `argsOffset` should be 0 for RPC calls, and 1 for WS requests, where len(rpcFunc.args) != len(rpcFunc.argNames). // -// argsOffset should be 0 for RPC calls, and 1 for WS requests, where len(rpcFunc.args) != len(rpcFunc.argNames). // Example: // rpcFunc.args = [rpctypes.WSRPCContext string] // rpcFunc.argNames = ["arg"] -func jsonParamsToArgs(rpcFunc *RPCFunc, raw []byte, argsOffset int) ([]reflect.Value, error) { - // first, try to get the map.. - var m map[string]*json.RawMessage +func jsonParamsToArgs(rpcFunc *RPCFunc, cdc *amino.Codec, raw []byte, argsOffset int) ([]reflect.Value, error) { + + // TODO: Make more efficient, perhaps by checking the first character for '{' or '['? + // First, try to get the map. + var m map[string]json.RawMessage err := json.Unmarshal(raw, &m) if err == nil { - return mapParamsToArgs(rpcFunc, m, argsOffset) + return mapParamsToArgs(rpcFunc, cdc, m, argsOffset) } - // otherwise, try an array - var a []*json.RawMessage + // Ttherwise, try an array. + var a []json.RawMessage err = json.Unmarshal(raw, &a) if err == nil { - return arrayParamsToArgs(rpcFunc, a, argsOffset) + return arrayParamsToArgs(rpcFunc, cdc, a, argsOffset) } - // otherwise, bad format, we cannot parse + // Otherwise, bad format, we cannot parse return nil, errors.Errorf("Unknown type for JSON params: %v. Expected map or array", err) } // Convert a []interface{} OR a map[string]interface{} to properly typed values -func jsonParamsToArgsRPC(rpcFunc *RPCFunc, params json.RawMessage) ([]reflect.Value, error) { - return jsonParamsToArgs(rpcFunc, params, 0) +func jsonParamsToArgsRPC(rpcFunc *RPCFunc, cdc *amino.Codec, params json.RawMessage) ([]reflect.Value, error) { + return jsonParamsToArgs(rpcFunc, cdc, params, 0) } // Same as above, but with the first param the websocket connection -func jsonParamsToArgsWS(rpcFunc *RPCFunc, params json.RawMessage, wsCtx types.WSRPCContext) ([]reflect.Value, error) { - values, err := jsonParamsToArgs(rpcFunc, params, 1) +func jsonParamsToArgsWS(rpcFunc *RPCFunc, cdc *amino.Codec, params json.RawMessage, wsCtx types.WSRPCContext) ([]reflect.Value, error) { + values, err := jsonParamsToArgs(rpcFunc, cdc, params, 1) if err != nil { return nil, err } @@ -235,7 +238,7 @@ func jsonParamsToArgsWS(rpcFunc *RPCFunc, params json.RawMessage, wsCtx types.WS // rpc.http // convert from a function name to the http handler -func makeHTTPHandler(rpcFunc *RPCFunc, logger log.Logger) func(http.ResponseWriter, *http.Request) { +func makeHTTPHandler(rpcFunc *RPCFunc, cdc *amino.Codec, logger log.Logger) func(http.ResponseWriter, *http.Request) { // Exception for websocket endpoints if rpcFunc.ws { return func(w http.ResponseWriter, r *http.Request) { @@ -245,7 +248,7 @@ func makeHTTPHandler(rpcFunc *RPCFunc, logger log.Logger) func(http.ResponseWrit // All other endpoints return func(w http.ResponseWriter, r *http.Request) { logger.Debug("HTTP HANDLER", "req", r) - args, err := httpParamsToArgs(rpcFunc, r) + args, err := httpParamsToArgs(rpcFunc, cdc, r) if err != nil { WriteRPCResponseHTTP(w, types.RPCInvalidParamsError("", errors.Wrap(err, "Error converting http params to arguments"))) return @@ -257,13 +260,13 @@ func makeHTTPHandler(rpcFunc *RPCFunc, logger log.Logger) func(http.ResponseWrit WriteRPCResponseHTTP(w, types.RPCInternalError("", err)) return } - WriteRPCResponseHTTP(w, types.NewRPCSuccessResponse("", result)) + WriteRPCResponseHTTP(w, types.NewRPCSuccessResponse(cdc, "", result)) } } // Covert an http query to a list of properly typed values. // To be properly decoded the arg must be a concrete type from tendermint (if its an interface). -func httpParamsToArgs(rpcFunc *RPCFunc, r *http.Request) ([]reflect.Value, error) { +func httpParamsToArgs(rpcFunc *RPCFunc, cdc *amino.Codec, r *http.Request) ([]reflect.Value, error) { values := make([]reflect.Value, len(rpcFunc.args)) for i, name := range rpcFunc.argNames { @@ -278,7 +281,7 @@ func httpParamsToArgs(rpcFunc *RPCFunc, r *http.Request) ([]reflect.Value, error continue } - v, err, ok := nonJsonToArg(argType, arg) + v, err, ok := nonJSONToArg(cdc, argType, arg) if err != nil { return nil, err } @@ -287,7 +290,7 @@ func httpParamsToArgs(rpcFunc *RPCFunc, r *http.Request) ([]reflect.Value, error continue } - values[i], err = _jsonStringToArg(argType, arg) + values[i], err = _jsonStringToArg(cdc, argType, arg) if err != nil { return nil, err } @@ -296,9 +299,9 @@ func httpParamsToArgs(rpcFunc *RPCFunc, r *http.Request) ([]reflect.Value, error return values, nil } -func _jsonStringToArg(ty reflect.Type, arg string) (reflect.Value, error) { +func _jsonStringToArg(cdc *amino.Codec, ty reflect.Type, arg string) (reflect.Value, error) { v := reflect.New(ty) - err := json.Unmarshal([]byte(arg), v.Interface()) + err := cdc.UnmarshalJSON([]byte(arg), v.Interface()) if err != nil { return v, err } @@ -306,7 +309,7 @@ func _jsonStringToArg(ty reflect.Type, arg string) (reflect.Value, error) { return v, nil } -func nonJsonToArg(ty reflect.Type, arg string) (reflect.Value, error, bool) { +func nonJSONToArg(cdc *amino.Codec, ty reflect.Type, arg string) (reflect.Value, error, bool) { isQuotedString := strings.HasPrefix(arg, `"`) && strings.HasSuffix(arg, `"`) isHexString := strings.HasPrefix(strings.ToLower(arg), "0x") expectingString := ty.Kind() == reflect.String @@ -332,7 +335,7 @@ func nonJsonToArg(ty reflect.Type, arg string) (reflect.Value, error, bool) { if isQuotedString && expectingByteSlice { v := reflect.New(reflect.TypeOf("")) - err := json.Unmarshal([]byte(arg), v.Interface()) + err := cdc.UnmarshalJSON([]byte(arg), v.Interface()) if err != nil { return reflect.ValueOf(nil), err, false } @@ -366,6 +369,7 @@ type wsConnection struct { writeChan chan types.RPCResponse funcMap map[string]*RPCFunc + cdc *amino.Codec // write channel capacity writeChanCapacity int @@ -389,11 +393,12 @@ type wsConnection struct { // description of how to configure ping period and pong wait time. NOTE: if the // write buffer is full, pongs may be dropped, which may cause clients to // disconnect. see https://github.com/gorilla/websocket/issues/97 -func NewWSConnection(baseConn *websocket.Conn, funcMap map[string]*RPCFunc, options ...func(*wsConnection)) *wsConnection { +func NewWSConnection(baseConn *websocket.Conn, funcMap map[string]*RPCFunc, cdc *amino.Codec, options ...func(*wsConnection)) *wsConnection { wsc := &wsConnection{ remoteAddr: baseConn.RemoteAddr().String(), baseConn: baseConn, funcMap: funcMap, + cdc: cdc, writeWait: defaultWSWriteWait, writeChanCapacity: defaultWSWriteChanCapacity, readWait: defaultWSReadWait, @@ -569,11 +574,11 @@ func (wsc *wsConnection) readRoutine() { if rpcFunc.ws { wsCtx := types.WSRPCContext{Request: request, WSRPCConnection: wsc} if len(request.Params) > 0 { - args, err = jsonParamsToArgsWS(rpcFunc, request.Params, wsCtx) + args, err = jsonParamsToArgsWS(rpcFunc, wsc.cdc, request.Params, wsCtx) } } else { if len(request.Params) > 0 { - args, err = jsonParamsToArgsRPC(rpcFunc, request.Params) + args, err = jsonParamsToArgsRPC(rpcFunc, wsc.cdc, request.Params) } } if err != nil { @@ -590,7 +595,7 @@ func (wsc *wsConnection) readRoutine() { wsc.WriteRPCResponse(types.RPCInternalError(request.ID, err)) continue } else { - wsc.WriteRPCResponse(types.NewRPCSuccessResponse(request.ID, result)) + wsc.WriteRPCResponse(types.NewRPCSuccessResponse(wsc.cdc, request.ID, result)) continue } @@ -666,6 +671,7 @@ func (wsc *wsConnection) writeMessageWithDeadline(msgType int, msg []byte) error type WebsocketManager struct { websocket.Upgrader funcMap map[string]*RPCFunc + cdc *amino.Codec logger log.Logger wsConnOptions []func(*wsConnection) } @@ -673,9 +679,10 @@ type WebsocketManager struct { // NewWebsocketManager returns a new WebsocketManager that routes according to // the given funcMap and connects to the server with the given connection // options. -func NewWebsocketManager(funcMap map[string]*RPCFunc, wsConnOptions ...func(*wsConnection)) *WebsocketManager { +func NewWebsocketManager(funcMap map[string]*RPCFunc, cdc *amino.Codec, wsConnOptions ...func(*wsConnection)) *WebsocketManager { return &WebsocketManager{ funcMap: funcMap, + cdc: cdc, Upgrader: websocket.Upgrader{ CheckOrigin: func(r *http.Request) bool { // TODO ??? @@ -702,7 +709,7 @@ func (wm *WebsocketManager) WebsocketHandler(w http.ResponseWriter, r *http.Requ } // register connection - con := NewWSConnection(wsConn, wm.funcMap, wm.wsConnOptions...) + con := NewWSConnection(wsConn, wm.funcMap, wm.cdc, wm.wsConnOptions...) con.SetLogger(wm.logger.With("remote", wsConn.RemoteAddr())) wm.logger.Info("New websocket connection", "remote", con.remoteAddr) err = con.Start() // Blocking diff --git a/rpc/lib/server/handlers_test.go b/rpc/lib/server/handlers_test.go index 664bbd917..92a2d9909 100644 --- a/rpc/lib/server/handlers_test.go +++ b/rpc/lib/server/handlers_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/tendermint/go-amino" rs "github.com/tendermint/tendermint/rpc/lib/server" types "github.com/tendermint/tendermint/rpc/lib/types" "github.com/tendermint/tmlibs/log" @@ -21,10 +22,11 @@ func testMux() *http.ServeMux { funcMap := map[string]*rs.RPCFunc{ "c": rs.NewRPCFunc(func(s string, i int) (string, error) { return "foo", nil }, "s,i"), } + cdc := amino.NewCodec() mux := http.NewServeMux() buf := new(bytes.Buffer) logger := log.NewTMLogger(buf) - rs.RegisterRPCFuncs(mux, funcMap, logger) + rs.RegisterRPCFuncs(mux, funcMap, cdc, logger) return mux } diff --git a/rpc/lib/server/parse_test.go b/rpc/lib/server/parse_test.go index 0703d50ed..f4323ef5d 100644 --- a/rpc/lib/server/parse_test.go +++ b/rpc/lib/server/parse_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/tendermint/go-amino" cmn "github.com/tendermint/tmlibs/common" ) @@ -138,6 +139,7 @@ func TestParseRPC(t *testing.T) { demo := func(height int, name string) {} call := NewRPCFunc(demo, "height,name") + cdc := amino.NewCodec() cases := []struct { raw string @@ -158,7 +160,7 @@ func TestParseRPC(t *testing.T) { for idx, tc := range cases { i := strconv.Itoa(idx) data := []byte(tc.raw) - vals, err := jsonParamsToArgs(call, data, 0) + vals, err := jsonParamsToArgs(call, cdc, data, 0) if tc.fail { assert.NotNil(err, i) } else { diff --git a/rpc/lib/test/main.go b/rpc/lib/test/main.go index 702ed9f73..604cbd3d8 100644 --- a/rpc/lib/test/main.go +++ b/rpc/lib/test/main.go @@ -5,6 +5,7 @@ import ( "net/http" "os" + "github.com/tendermint/go-amino" rpcserver "github.com/tendermint/tendermint/rpc/lib/server" cmn "github.com/tendermint/tmlibs/common" "github.com/tendermint/tmlibs/log" @@ -24,8 +25,9 @@ type Result struct { func main() { mux := http.NewServeMux() + cdc := amino.NewCodec() logger := log.NewTMLogger(log.NewSyncWriter(os.Stdout)) - rpcserver.RegisterRPCFuncs(mux, routes, logger) + rpcserver.RegisterRPCFuncs(mux, routes, cdc, logger) _, err := rpcserver.StartHTTPServer("0.0.0.0:8008", mux, logger) if err != nil { cmn.Exit(err.Error()) diff --git a/rpc/lib/types/types.go b/rpc/lib/types/types.go index 37d451457..e8eea9fff 100644 --- a/rpc/lib/types/types.go +++ b/rpc/lib/types/types.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/pkg/errors" + "github.com/tendermint/go-amino" tmpubsub "github.com/tendermint/tmlibs/pubsub" ) @@ -33,8 +34,16 @@ func (req RPCRequest) String() string { return fmt.Sprintf("[%s %s]", req.ID, req.Method) } -func MapToRequest(id string, method string, params map[string]interface{}) (RPCRequest, error) { - payload, err := json.Marshal(params) +func MapToRequest(cdc *amino.Codec, id string, method string, params map[string]interface{}) (RPCRequest, error) { + var params_ = make(map[string]json.RawMessage, len(params)) + for name, value := range params { + valueJSON, err := cdc.MarshalJSON(value) + if err != nil { + return RPCRequest{}, err + } + params_[name] = valueJSON + } + payload, err := json.Marshal(params_) // NOTE: Amino doesn't handle maps yet. if err != nil { return RPCRequest{}, err } @@ -42,8 +51,16 @@ func MapToRequest(id string, method string, params map[string]interface{}) (RPCR return request, nil } -func ArrayToRequest(id string, method string, params []interface{}) (RPCRequest, error) { - payload, err := json.Marshal(params) +func ArrayToRequest(cdc *amino.Codec, id string, method string, params []interface{}) (RPCRequest, error) { + var params_ = make([]json.RawMessage, len(params)) + for i, value := range params { + valueJSON, err := cdc.MarshalJSON(value) + if err != nil { + return RPCRequest{}, err + } + params_[i] = valueJSON + } + payload, err := json.Marshal(params_) // NOTE: Amino doesn't handle maps yet. if err != nil { return RPCRequest{}, err } @@ -75,12 +92,12 @@ type RPCResponse struct { Error *RPCError `json:"error,omitempty"` } -func NewRPCSuccessResponse(id string, res interface{}) RPCResponse { +func NewRPCSuccessResponse(cdc *amino.Codec, id string, res interface{}) RPCResponse { var rawMsg json.RawMessage if res != nil { var js []byte - js, err := json.Marshal(res) + js, err := cdc.MarshalJSON(res) if err != nil { return RPCInternalError(id, errors.Wrap(err, "Error marshalling response")) } diff --git a/rpc/lib/types/types_test.go b/rpc/lib/types/types_test.go index 60f7b2213..9dd1b7a18 100644 --- a/rpc/lib/types/types_test.go +++ b/rpc/lib/types/types_test.go @@ -8,6 +8,7 @@ import ( "github.com/pkg/errors" "github.com/stretchr/testify/assert" + "github.com/tendermint/go-amino" ) type SampleResult struct { @@ -16,8 +17,9 @@ type SampleResult struct { func TestResponses(t *testing.T) { assert := assert.New(t) + cdc := amino.NewCodec() - a := NewRPCSuccessResponse("1", &SampleResult{"hello"}) + a := NewRPCSuccessResponse(cdc, "1", &SampleResult{"hello"}) b, _ := json.Marshal(a) s := `{"jsonrpc":"2.0","id":"1","result":{"Value":"hello"}}` assert.Equal(string(s), string(b)) From c541d58d2f6d47d551ec6f6bb57080417c9e64d2 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Thu, 5 Apr 2018 16:07:29 -0700 Subject: [PATCH 065/143] WIP: fix rpc/core --- node/node.go | 4 ++-- rpc/core/events.go | 2 +- rpc/core/routes.go | 14 ++++++++++++++ rpc/lib/server/handlers.go | 8 +++++++- rpc/lib/types/types.go | 1 + 5 files changed, 25 insertions(+), 4 deletions(-) diff --git a/node/node.go b/node/node.go index be744f845..0b5f73725 100644 --- a/node/node.go +++ b/node/node.go @@ -499,10 +499,10 @@ func (n *Node) startRPC() ([]net.Listener, error) { for i, listenAddr := range listenAddrs { mux := http.NewServeMux() rpcLogger := n.Logger.With("module", "rpc-server") - wm := rpcserver.NewWebsocketManager(rpccore.Routes, rpcserver.EventSubscriber(n.eventBus)) + wm := rpcserver.NewWebsocketManager(rpccore.Routes, rpccore.RoutesCodec, rpcserver.EventSubscriber(n.eventBus)) wm.SetLogger(rpcLogger.With("protocol", "websocket")) mux.HandleFunc("/websocket", wm.WebsocketHandler) - rpcserver.RegisterRPCFuncs(mux, rpccore.Routes, rpcLogger) + rpcserver.RegisterRPCFuncs(mux, rpccore.Routes, rpccore.RoutesCodec, rpcLogger) listener, err := rpcserver.StartHTTPServer(listenAddr, mux, rpcLogger) if err != nil { return nil, err diff --git a/rpc/core/events.go b/rpc/core/events.go index beeaf4c90..a46e0947c 100644 --- a/rpc/core/events.go +++ b/rpc/core/events.go @@ -104,7 +104,7 @@ func Subscribe(wsCtx rpctypes.WSRPCContext, query string) (*ctypes.ResultSubscri go func() { for event := range ch { tmResult := &ctypes.ResultEvent{query, event.(tmtypes.TMEventData)} - wsCtx.TryWriteRPCResponse(rpctypes.NewRPCSuccessResponse(wsCtx.Request.ID+"#event", tmResult)) + wsCtx.TryWriteRPCResponse(rpctypes.NewRPCSuccessResponse(wsCtx.Codec(), wsCtx.Request.ID+"#event", tmResult)) } }() diff --git a/rpc/core/routes.go b/rpc/core/routes.go index 3ea7aa08c..9368dc3df 100644 --- a/rpc/core/routes.go +++ b/rpc/core/routes.go @@ -1,7 +1,10 @@ package core import ( + "github.com/tendermint/go-amino" + "github.com/tendermint/go-crypto" rpc "github.com/tendermint/tendermint/rpc/lib/server" + "github.com/tendermint/tendermint/types" ) // TODO: better system than "unsafe" prefix @@ -47,3 +50,14 @@ func AddUnsafeRoutes() { Routes["unsafe_stop_cpu_profiler"] = rpc.NewRPCFunc(UnsafeStopCPUProfiler, "") Routes["unsafe_write_heap_profile"] = rpc.NewRPCFunc(UnsafeWriteHeapProfile, "filename") } + +var RoutesCodec *amino.Codec + +func init() { + cdc := amino.NewCodec() + RoutesCodec = cdc + + types.RegisterEventDatas(cdc) + types.RegisterEvidences(cdc) + crypto.RegisterAmino(cdc) +} diff --git a/rpc/lib/server/handlers.go b/rpc/lib/server/handlers.go index 31aaaedaf..d59372206 100644 --- a/rpc/lib/server/handlers.go +++ b/rpc/lib/server/handlers.go @@ -357,7 +357,7 @@ const ( defaultWSPingPeriod = (defaultWSReadWait * 9) / 10 ) -// a single websocket connection contains listener id, underlying ws +// A single websocket connection contains listener id, underlying ws // connection, and the event switch for subscribing to events. // // In case of an error, the connection is stopped. @@ -508,6 +508,12 @@ func (wsc *wsConnection) TryWriteRPCResponse(resp types.RPCResponse) bool { } } +// Codec returns an amino codec used to decode parameters and encode results. +// It implements WSRPCConnection. +func (wsc *wsConnection) Codec() *amino.Codec { + return wsc.cdc +} + // Read from the socket and subscribe to or unsubscribe from events func (wsc *wsConnection) readRoutine() { defer func() { diff --git a/rpc/lib/types/types.go b/rpc/lib/types/types.go index e8eea9fff..9d27f6c68 100644 --- a/rpc/lib/types/types.go +++ b/rpc/lib/types/types.go @@ -155,6 +155,7 @@ type WSRPCConnection interface { WriteRPCResponse(resp RPCResponse) TryWriteRPCResponse(resp RPCResponse) bool GetEventSubscriber() EventSubscriber + Codec() *amino.Codec } // EventSubscriber mirros tendermint/tendermint/types.EventBusSubscriber From 3ca5292dc95d7cae164a61bce65b92532bd7715e Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Thu, 5 Apr 2018 21:19:14 -0700 Subject: [PATCH 066/143] Fix rpc tests --- Gopkg.lock | 6 +- Gopkg.toml | 2 +- node/node.go | 7 +- rpc/client/event_test.go | 185 ++++++++++++++++++---------------- rpc/client/httpclient.go | 18 +++- rpc/core/routes.go | 15 +-- rpc/core/types/wire.go | 13 +++ rpc/lib/client/http_client.go | 9 ++ rpc/lib/client/ws_client.go | 4 + rpc/test/helpers.go | 5 + types/events.go | 17 ++-- 11 files changed, 160 insertions(+), 121 deletions(-) create mode 100644 rpc/core/types/wire.go diff --git a/Gopkg.lock b/Gopkg.lock index c0704d722..2d6fe8b93 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -254,8 +254,8 @@ [[projects]] name = "github.com/tendermint/go-amino" packages = ["."] - revision = "26718ab6738f938d4b33d593543cee7681f2a6a6" - version = "0.9.5" + revision = "42246108ff925a457fb709475070a03dfd3e2b5c" + version = "0.9.6" [[projects]] name = "github.com/tendermint/go-crypto" @@ -383,6 +383,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "0dacd2eb1550ca01e0c64f77b721eda1a381dde1d246a56bfe5a2746b78b7bad" + inputs-digest = "d14dbd59436d0ea3b322c42ce33c213b26cd2451ba023a466764b8002e0e649d" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 73f73bebc..d689da986 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -79,7 +79,7 @@ [[constraint]] name = "github.com/tendermint/go-amino" - version = "0.9.5" + version = "0.9.6" [[constraint]] name = "github.com/tendermint/tmlibs" diff --git a/node/node.go b/node/node.go index 0b5f73725..221eb1eb0 100644 --- a/node/node.go +++ b/node/node.go @@ -26,6 +26,7 @@ import ( "github.com/tendermint/tendermint/p2p/trust" "github.com/tendermint/tendermint/proxy" rpccore "github.com/tendermint/tendermint/rpc/core" + ctypes "github.com/tendermint/tendermint/rpc/core/types" grpccore "github.com/tendermint/tendermint/rpc/grpc" rpc "github.com/tendermint/tendermint/rpc/lib" rpcserver "github.com/tendermint/tendermint/rpc/lib/server" @@ -489,6 +490,8 @@ func (n *Node) ConfigureRPC() { func (n *Node) startRPC() ([]net.Listener, error) { n.ConfigureRPC() listenAddrs := strings.Split(n.config.RPC.ListenAddress, ",") + coreCodec := amino.NewCodec() + ctypes.RegisterAmino(coreCodec) if n.config.RPC.Unsafe { rpccore.AddUnsafeRoutes() @@ -499,10 +502,10 @@ func (n *Node) startRPC() ([]net.Listener, error) { for i, listenAddr := range listenAddrs { mux := http.NewServeMux() rpcLogger := n.Logger.With("module", "rpc-server") - wm := rpcserver.NewWebsocketManager(rpccore.Routes, rpccore.RoutesCodec, rpcserver.EventSubscriber(n.eventBus)) + wm := rpcserver.NewWebsocketManager(rpccore.Routes, coreCodec, rpcserver.EventSubscriber(n.eventBus)) wm.SetLogger(rpcLogger.With("protocol", "websocket")) mux.HandleFunc("/websocket", wm.WebsocketHandler) - rpcserver.RegisterRPCFuncs(mux, rpccore.Routes, rpccore.RoutesCodec, rpcLogger) + rpcserver.RegisterRPCFuncs(mux, rpccore.Routes, coreCodec, rpcLogger) listener, err := rpcserver.StartHTTPServer(listenAddr, mux, rpcLogger) if err != nil { return nil, err diff --git a/rpc/client/event_test.go b/rpc/client/event_test.go index e90ace43f..2254c1d1e 100644 --- a/rpc/client/event_test.go +++ b/rpc/client/event_test.go @@ -1,16 +1,16 @@ package client_test import ( + "reflect" "testing" "time" "github.com/stretchr/testify/require" abci "github.com/tendermint/abci/types" - cmn "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tendermint/rpc/client" "github.com/tendermint/tendermint/types" + cmn "github.com/tendermint/tmlibs/common" ) var waitForEventTimeout = 5 * time.Second @@ -23,116 +23,127 @@ func MakeTxKV() ([]byte, []byte, []byte) { } func TestHeaderEvents(t *testing.T) { - require := require.New(t) for i, c := range GetClients() { - // start for this test it if it wasn't already running - if !c.IsRunning() { - // if so, then we start it, listen, and stop it. - err := c.Start() - require.Nil(err, "%d: %+v", i, err) - defer c.Stop() - } + i, c := i, c // capture params + t.Run(reflect.TypeOf(c).String(), func(t *testing.T) { + // start for this test it if it wasn't already running + if !c.IsRunning() { + // if so, then we start it, listen, and stop it. + err := c.Start() + require.Nil(t, err, "%d: %+v", i, err) + defer c.Stop() + } - evtTyp := types.EventNewBlockHeader - evt, err := client.WaitForOneEvent(c, evtTyp, waitForEventTimeout) - require.Nil(err, "%d: %+v", i, err) - _, ok := evt.(types.EventDataNewBlockHeader) - require.True(ok, "%d: %#v", i, evt) - // TODO: more checks... + evtTyp := types.EventNewBlockHeader + evt, err := client.WaitForOneEvent(c, evtTyp, waitForEventTimeout) + require.Nil(t, err, "%d: %+v", i, err) + _, ok := evt.(types.EventDataNewBlockHeader) + require.True(t, ok, "%d: %#v", i, evt) + // TODO: more checks... + }) } } func TestBlockEvents(t *testing.T) { - require := require.New(t) for i, c := range GetClients() { - // start for this test it if it wasn't already running - if !c.IsRunning() { - // if so, then we start it, listen, and stop it. - err := c.Start() - require.Nil(err, "%d: %+v", i, err) - defer c.Stop() - } + i, c := i, c // capture params + t.Run(reflect.TypeOf(c).String(), func(t *testing.T) { - // listen for a new block; ensure height increases by 1 - var firstBlockHeight int64 - for j := 0; j < 3; j++ { - evtTyp := types.EventNewBlock - evt, err := client.WaitForOneEvent(c, evtTyp, waitForEventTimeout) - require.Nil(err, "%d: %+v", j, err) - blockEvent, ok := evt.(types.EventDataNewBlock) - require.True(ok, "%d: %#v", j, evt) - - block := blockEvent.Block - if j == 0 { - firstBlockHeight = block.Header.Height - continue + // start for this test it if it wasn't already running + if !c.IsRunning() { + // if so, then we start it, listen, and stop it. + err := c.Start() + require.Nil(t, err, "%d: %+v", i, err) + defer c.Stop() } - require.Equal(block.Header.Height, firstBlockHeight+int64(j)) - } + // listen for a new block; ensure height increases by 1 + var firstBlockHeight int64 + for j := 0; j < 3; j++ { + evtTyp := types.EventNewBlock + evt, err := client.WaitForOneEvent(c, evtTyp, waitForEventTimeout) + require.Nil(t, err, "%d: %+v", j, err) + blockEvent, ok := evt.(types.EventDataNewBlock) + require.True(t, ok, "%d: %#v", j, evt) + + block := blockEvent.Block + if j == 0 { + firstBlockHeight = block.Header.Height + continue + } + + require.Equal(t, block.Header.Height, firstBlockHeight+int64(j)) + } + }) } } func TestTxEventsSentWithBroadcastTxAsync(t *testing.T) { - require := require.New(t) for i, c := range GetClients() { - // start for this test it if it wasn't already running - if !c.IsRunning() { - // if so, then we start it, listen, and stop it. - err := c.Start() - require.Nil(err, "%d: %+v", i, err) - defer c.Stop() - } + i, c := i, c // capture params + t.Run(reflect.TypeOf(c).String(), func(t *testing.T) { - // make the tx - _, _, tx := MakeTxKV() - evtTyp := types.EventTx + // start for this test it if it wasn't already running + if !c.IsRunning() { + // if so, then we start it, listen, and stop it. + err := c.Start() + require.Nil(t, err, "%d: %+v", i, err) + defer c.Stop() + } - // send async - txres, err := c.BroadcastTxAsync(tx) - require.Nil(err, "%+v", err) - require.Equal(txres.Code, abci.CodeTypeOK) // FIXME + // make the tx + _, _, tx := MakeTxKV() + evtTyp := types.EventTx - // and wait for confirmation - evt, err := client.WaitForOneEvent(c, evtTyp, waitForEventTimeout) - require.Nil(err, "%d: %+v", i, err) - // and make sure it has the proper info - txe, ok := evt.(types.EventDataTx) - require.True(ok, "%d: %#v", i, evt) - // make sure this is the proper tx - require.EqualValues(tx, txe.Tx) - require.True(txe.Result.IsOK()) + // send async + txres, err := c.BroadcastTxAsync(tx) + require.Nil(t, err, "%+v", err) + require.Equal(t, txres.Code, abci.CodeTypeOK) // FIXME + + // and wait for confirmation + evt, err := client.WaitForOneEvent(c, evtTyp, waitForEventTimeout) + require.Nil(t, err, "%d: %+v", i, err) + // and make sure it has the proper info + txe, ok := evt.(types.EventDataTx) + require.True(t, ok, "%d: %#v", i, evt) + // make sure this is the proper tx + require.EqualValues(t, tx, txe.Tx) + require.True(t, txe.Result.IsOK()) + }) } } func TestTxEventsSentWithBroadcastTxSync(t *testing.T) { - require := require.New(t) for i, c := range GetClients() { - // start for this test it if it wasn't already running - if !c.IsRunning() { - // if so, then we start it, listen, and stop it. - err := c.Start() - require.Nil(err, "%d: %+v", i, err) - defer c.Stop() - } + i, c := i, c // capture params + t.Run(reflect.TypeOf(c).String(), func(t *testing.T) { - // make the tx - _, _, tx := MakeTxKV() - evtTyp := types.EventTx + // start for this test it if it wasn't already running + if !c.IsRunning() { + // if so, then we start it, listen, and stop it. + err := c.Start() + require.Nil(t, err, "%d: %+v", i, err) + defer c.Stop() + } - // send sync - txres, err := c.BroadcastTxSync(tx) - require.Nil(err, "%+v", err) - require.Equal(txres.Code, abci.CodeTypeOK) // FIXME + // make the tx + _, _, tx := MakeTxKV() + evtTyp := types.EventTx - // and wait for confirmation - evt, err := client.WaitForOneEvent(c, evtTyp, waitForEventTimeout) - require.Nil(err, "%d: %+v", i, err) - // and make sure it has the proper info - txe, ok := evt.(types.EventDataTx) - require.True(ok, "%d: %#v", i, evt) - // make sure this is the proper tx - require.EqualValues(tx, txe.Tx) - require.True(txe.Result.IsOK()) + // send sync + txres, err := c.BroadcastTxSync(tx) + require.Nil(t, err, "%+v", err) + require.Equal(t, txres.Code, abci.CodeTypeOK) // FIXME + + // and wait for confirmation + evt, err := client.WaitForOneEvent(c, evtTyp, waitForEventTimeout) + require.Nil(t, err, "%d: %+v", i, err) + // and make sure it has the proper info + txe, ok := evt.(types.EventDataTx) + require.True(t, ok, "%d: %#v", i, evt) + // make sure this is the proper tx + require.EqualValues(t, tx, txe.Tx) + require.True(t, txe.Result.IsOK()) + }) } } diff --git a/rpc/client/httpclient.go b/rpc/client/httpclient.go index bc6cf759e..496ce9c66 100644 --- a/rpc/client/httpclient.go +++ b/rpc/client/httpclient.go @@ -2,11 +2,11 @@ package client import ( "context" - "encoding/json" "sync" "github.com/pkg/errors" + amino "github.com/tendermint/go-amino" ctypes "github.com/tendermint/tendermint/rpc/core/types" rpcclient "github.com/tendermint/tendermint/rpc/lib/client" "github.com/tendermint/tendermint/types" @@ -32,10 +32,14 @@ type HTTP struct { // New takes a remote endpoint in the form tcp://: // and the websocket path (which always seems to be "/websocket") func NewHTTP(remote, wsEndpoint string) *HTTP { + rc := rpcclient.NewJSONRPCClient(remote) + cdc := rc.Codec() + ctypes.RegisterAmino(cdc) + return &HTTP{ - rpc: rpcclient.NewJSONRPCClient(remote), + rpc: rc, remote: remote, - WSEvents: newWSEvents(remote, wsEndpoint), + WSEvents: newWSEvents(cdc, remote, wsEndpoint), } } @@ -208,6 +212,7 @@ func (c *HTTP) Validators(height *int64) (*ctypes.ResultValidators, error) { type WSEvents struct { cmn.BaseService + cdc *amino.Codec remote string endpoint string ws *rpcclient.WSClient @@ -216,8 +221,9 @@ type WSEvents struct { subscriptions map[string]chan<- interface{} } -func newWSEvents(remote, endpoint string) *WSEvents { +func newWSEvents(cdc *amino.Codec, remote, endpoint string) *WSEvents { wsEvents := &WSEvents{ + cdc: cdc, endpoint: endpoint, remote: remote, subscriptions: make(map[string]chan<- interface{}), @@ -231,6 +237,8 @@ func (w *WSEvents) OnStart() error { w.ws = rpcclient.NewWSClient(w.remote, w.endpoint, rpcclient.OnReconnect(func() { w.redoSubscriptions() })) + w.ws.SetCodec(w.cdc) + err := w.ws.Start() if err != nil { return err @@ -326,7 +334,7 @@ func (w *WSEvents) eventListener() { continue } result := new(ctypes.ResultEvent) - err := json.Unmarshal(resp.Result, result) + err := w.cdc.UnmarshalJSON(resp.Result, result) if err != nil { w.Logger.Error("failed to unmarshal response", "err", err) continue diff --git a/rpc/core/routes.go b/rpc/core/routes.go index 9368dc3df..337dd64aa 100644 --- a/rpc/core/routes.go +++ b/rpc/core/routes.go @@ -1,13 +1,11 @@ package core import ( - "github.com/tendermint/go-amino" - "github.com/tendermint/go-crypto" rpc "github.com/tendermint/tendermint/rpc/lib/server" - "github.com/tendermint/tendermint/types" ) // TODO: better system than "unsafe" prefix +// NOTE: Amino is registered in rpc/core/types/wire.go. var Routes = map[string]*rpc.RPCFunc{ // subscribe/unsubscribe are reserved for websocket events. "subscribe": rpc.NewWSRPCFunc(Subscribe, "query"), @@ -50,14 +48,3 @@ func AddUnsafeRoutes() { Routes["unsafe_stop_cpu_profiler"] = rpc.NewRPCFunc(UnsafeStopCPUProfiler, "") Routes["unsafe_write_heap_profile"] = rpc.NewRPCFunc(UnsafeWriteHeapProfile, "filename") } - -var RoutesCodec *amino.Codec - -func init() { - cdc := amino.NewCodec() - RoutesCodec = cdc - - types.RegisterEventDatas(cdc) - types.RegisterEvidences(cdc) - crypto.RegisterAmino(cdc) -} diff --git a/rpc/core/types/wire.go b/rpc/core/types/wire.go new file mode 100644 index 000000000..6648364b1 --- /dev/null +++ b/rpc/core/types/wire.go @@ -0,0 +1,13 @@ +package core_types + +import ( + "github.com/tendermint/go-amino" + "github.com/tendermint/go-crypto" + "github.com/tendermint/tendermint/types" +) + +func RegisterAmino(cdc *amino.Codec) { + types.RegisterEventDatas(cdc) + types.RegisterEvidences(cdc) + crypto.RegisterAmino(cdc) +} diff --git a/rpc/lib/client/http_client.go b/rpc/lib/client/http_client.go index a41f3125c..e26d8f274 100644 --- a/rpc/lib/client/http_client.go +++ b/rpc/lib/client/http_client.go @@ -21,6 +21,7 @@ import ( type HTTPClient interface { Call(method string, params map[string]interface{}, result interface{}) (interface{}, error) Codec() *amino.Codec + SetCodec(*amino.Codec) } // TODO: Deprecate support for IP:PORT or /path/to/socket @@ -111,6 +112,10 @@ func (c *JSONRPCClient) Codec() *amino.Codec { return c.cdc } +func (c *JSONRPCClient) SetCodec(cdc *amino.Codec) { + c.cdc = cdc +} + //------------------------------------------------------------- // URI takes params as a map @@ -152,6 +157,10 @@ func (c *URIClient) Codec() *amino.Codec { return c.cdc } +func (c *URIClient) SetCodec(cdc *amino.Codec) { + c.cdc = cdc +} + //------------------------------------------------ func unmarshalResponseBytes(cdc *amino.Codec, responseBytes []byte, result interface{}) (interface{}, error) { diff --git a/rpc/lib/client/ws_client.go b/rpc/lib/client/ws_client.go index 43aa02f40..de5df6a2c 100644 --- a/rpc/lib/client/ws_client.go +++ b/rpc/lib/client/ws_client.go @@ -230,6 +230,10 @@ func (c *WSClient) Codec() *amino.Codec { return c.cdc } +func (c *WSClient) SetCodec(cdc *amino.Codec) { + c.cdc = cdc +} + /////////////////////////////////////////////////////////////////////////////// // Private methods diff --git a/rpc/test/helpers.go b/rpc/test/helpers.go index c9dc6d219..38c2dfaf0 100644 --- a/rpc/test/helpers.go +++ b/rpc/test/helpers.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "strings" + "time" "github.com/tendermint/tmlibs/log" @@ -26,11 +27,15 @@ var globalConfig *cfg.Config func waitForRPC() { laddr := GetConfig().RPC.ListenAddress client := rpcclient.NewJSONRPCClient(laddr) + ctypes.RegisterAmino(client.Codec()) result := new(ctypes.ResultStatus) for { _, err := client.Call("status", map[string]interface{}{}, result) if err == nil { return + } else { + fmt.Println("error", err) + time.Sleep(time.Millisecond) } } } diff --git a/types/events.go b/types/events.go index 1323b65bb..cdffc0ee5 100644 --- a/types/events.go +++ b/types/events.go @@ -35,20 +35,19 @@ const ( // ENCODING / DECODING /////////////////////////////////////////////////////////////////////////////// -var ( - EventDataNameNewBlock = "new_block" - EventDataNameNewBlockHeader = "new_block_header" - EventDataNameTx = "tx" - EventDataNameRoundState = "round_state" - EventDataNameVote = "vote" - EventDataNameProposalHeartbeat = "proposal_heartbeat" -) - // implements events.EventData type TMEventData interface { + AssertIsTMEventData() // empty interface } +func (_ EventDataNewBlock) AssertIsTMEventData() {} +func (_ EventDataNewBlockHeader) AssertIsTMEventData() {} +func (_ EventDataTx) AssertIsTMEventData() {} +func (_ EventDataRoundState) AssertIsTMEventData() {} +func (_ EventDataVote) AssertIsTMEventData() {} +func (_ EventDataProposalHeartbeat) AssertIsTMEventData() {} + func RegisterEventDatas(cdc *amino.Codec) { cdc.RegisterInterface((*TMEventData)(nil), nil) cdc.RegisterConcrete(EventDataNewBlock{}, "tendermint/EventDataNameNewBlock", nil) From 32e1d195a08d8195b120d01358c5e075549e2a0b Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Thu, 5 Apr 2018 22:05:30 -0700 Subject: [PATCH 067/143] Fix cmd and lite --- benchmarks/codec_test.go | 7 +++ cmd/priv_val_server/main.go | 2 +- cmd/tendermint/commands/gen_validator.go | 9 ++-- cmd/tendermint/commands/init.go | 11 +++-- .../commands/reset_priv_validator.go | 16 +++---- cmd/tendermint/commands/testnet.go | 17 +++---- lite/files/commit.go | 46 ++++++++++++------- lite/files/wire.go | 12 +++++ lite/helpers.go | 6 +-- lite/proxy/proxy.go | 8 +++- types/event_bus_test.go | 2 +- types/events.go | 16 ++++--- 12 files changed, 97 insertions(+), 55 deletions(-) create mode 100644 lite/files/wire.go diff --git a/benchmarks/codec_test.go b/benchmarks/codec_test.go index b34417992..83e25e212 100644 --- a/benchmarks/codec_test.go +++ b/benchmarks/codec_test.go @@ -4,6 +4,7 @@ import ( "testing" "time" + "github.com/tendermint/go-amino" "github.com/tendermint/go-crypto" proto "github.com/tendermint/tendermint/benchmarks/proto" @@ -13,6 +14,8 @@ import ( func BenchmarkEncodeStatusWire(b *testing.B) { b.StopTimer() + cdc := amino.NewCodec() + ctypes.RegisterAmino(cdc) pubKey := crypto.GenPrivKeyEd25519().PubKey() status := &ctypes.ResultStatus{ NodeInfo: p2p.NodeInfo{ @@ -43,6 +46,8 @@ func BenchmarkEncodeStatusWire(b *testing.B) { func BenchmarkEncodeNodeInfoWire(b *testing.B) { b.StopTimer() + cdc := amino.NewCodec() + ctypes.RegisterAmino(cdc) pubKey := crypto.GenPrivKeyEd25519().PubKey() nodeInfo := p2p.NodeInfo{ PubKey: pubKey, @@ -66,6 +71,8 @@ func BenchmarkEncodeNodeInfoWire(b *testing.B) { func BenchmarkEncodeNodeInfoBinary(b *testing.B) { b.StopTimer() + cdc := amino.NewCodec() + ctypes.RegisterAmino(cdc) pubKey := crypto.GenPrivKeyEd25519().PubKey() nodeInfo := p2p.NodeInfo{ PubKey: pubKey, diff --git a/cmd/priv_val_server/main.go b/cmd/priv_val_server/main.go index 9f3ec73ca..6b10b8178 100644 --- a/cmd/priv_val_server/main.go +++ b/cmd/priv_val_server/main.go @@ -30,7 +30,7 @@ func main() { "privPath", *privValPath, ) - privVal := priv_val.LoadPrivValidatorJSON(*privValPath) + privVal := priv_val.LoadFilePV(*privValPath) rs := priv_val.NewRemoteSigner( logger, diff --git a/cmd/tendermint/commands/gen_validator.go b/cmd/tendermint/commands/gen_validator.go index 59fe30128..ff0d97d76 100644 --- a/cmd/tendermint/commands/gen_validator.go +++ b/cmd/tendermint/commands/gen_validator.go @@ -1,12 +1,11 @@ package commands import ( - "encoding/json" "fmt" "github.com/spf13/cobra" - "github.com/tendermint/tendermint/types" + pvm "github.com/tendermint/tendermint/types/priv_validator" ) // GenValidatorCmd allows the generation of a keypair for a @@ -18,11 +17,11 @@ var GenValidatorCmd = &cobra.Command{ } func genValidator(cmd *cobra.Command, args []string) { - privValidator := types.GenPrivValidatorFS("") - privValidatorJSONBytes, err := json.MarshalIndent(privValidator, "", "\t") + pv := pvm.GenFilePV("") + jsbz, err := cdc.MarshalJSON(pv) if err != nil { panic(err) } fmt.Printf(`%v -`, string(privValidatorJSONBytes)) +`, string(jsbz)) } diff --git a/cmd/tendermint/commands/init.go b/cmd/tendermint/commands/init.go index 70648f8ff..224452646 100644 --- a/cmd/tendermint/commands/init.go +++ b/cmd/tendermint/commands/init.go @@ -4,6 +4,7 @@ import ( "github.com/spf13/cobra" "github.com/tendermint/tendermint/types" + pvm "github.com/tendermint/tendermint/types/priv_validator" cmn "github.com/tendermint/tmlibs/common" ) @@ -17,13 +18,13 @@ var InitFilesCmd = &cobra.Command{ func initFiles(cmd *cobra.Command, args []string) { // private validator privValFile := config.PrivValidatorFile() - var privValidator *types.PrivValidatorFS + var pv *pvm.FilePV if cmn.FileExists(privValFile) { - privValidator = types.LoadPrivValidatorFS(privValFile) + pv = pvm.LoadFilePV(privValFile) logger.Info("Found private validator", "path", privValFile) } else { - privValidator = types.GenPrivValidatorFS(privValFile) - privValidator.Save() + pv = pvm.GenFilePV(privValFile) + pv.Save() logger.Info("Generated private validator", "path", privValFile) } @@ -36,7 +37,7 @@ func initFiles(cmd *cobra.Command, args []string) { ChainID: cmn.Fmt("test-chain-%v", cmn.RandStr(6)), } genDoc.Validators = []types.GenesisValidator{{ - PubKey: privValidator.GetPubKey(), + PubKey: pv.GetPubKey(), Power: 10, }} diff --git a/cmd/tendermint/commands/reset_priv_validator.go b/cmd/tendermint/commands/reset_priv_validator.go index 513365237..78db87ded 100644 --- a/cmd/tendermint/commands/reset_priv_validator.go +++ b/cmd/tendermint/commands/reset_priv_validator.go @@ -5,7 +5,7 @@ import ( "github.com/spf13/cobra" - "github.com/tendermint/tendermint/types" + pvm "github.com/tendermint/tendermint/types/priv_validator" "github.com/tendermint/tmlibs/log" ) @@ -27,7 +27,7 @@ var ResetPrivValidatorCmd = &cobra.Command{ // ResetAll removes the privValidator files. // Exported so other CLI tools can use it. func ResetAll(dbDir, privValFile string, logger log.Logger) { - resetPrivValidatorFS(privValFile, logger) + resetFilePV(privValFile, logger) if err := os.RemoveAll(dbDir); err != nil { logger.Error("Error removing directory", "err", err) return @@ -44,18 +44,18 @@ func resetAll(cmd *cobra.Command, args []string) { // XXX: this is totally unsafe. // it's only suitable for testnets. func resetPrivValidator(cmd *cobra.Command, args []string) { - resetPrivValidatorFS(config.PrivValidatorFile(), logger) + resetFilePV(config.PrivValidatorFile(), logger) } -func resetPrivValidatorFS(privValFile string, logger log.Logger) { +func resetFilePV(privValFile string, logger log.Logger) { // Get PrivValidator if _, err := os.Stat(privValFile); err == nil { - privValidator := types.LoadPrivValidatorFS(privValFile) - privValidator.Reset() + pv := pvm.LoadFilePV(privValFile) + pv.Reset() logger.Info("Reset PrivValidator", "file", privValFile) } else { - privValidator := types.GenPrivValidatorFS(privValFile) - privValidator.Save() + pv := pvm.GenFilePV(privValFile) + pv.Save() logger.Info("Generated PrivValidator", "file", privValFile) } } diff --git a/cmd/tendermint/commands/testnet.go b/cmd/tendermint/commands/testnet.go index 3864e7ad5..eb86e4f8d 100644 --- a/cmd/tendermint/commands/testnet.go +++ b/cmd/tendermint/commands/testnet.go @@ -9,6 +9,7 @@ import ( cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/types" + pvm "github.com/tendermint/tendermint/types/priv_validator" cmn "github.com/tendermint/tmlibs/common" ) @@ -46,10 +47,10 @@ func testnetFiles(cmd *cobra.Command, args []string) { cmn.Exit(err.Error()) } // Read priv_validator.json to populate vals - privValFile := filepath.Join(dataDir, mach, defaultConfig.PrivValidator) - privVal := types.LoadPrivValidatorFS(privValFile) + pvFile := filepath.Join(dataDir, mach, defaultConfig.PrivValidator) + pv := pvm.LoadFilePV(pvFile) genVals[i] = types.GenesisValidator{ - PubKey: privVal.GetPubKey(), + PubKey: pv.GetPubKey(), Power: 1, Name: mach, } @@ -78,13 +79,13 @@ func initMachCoreDirectory(base, mach string) error { // Create priv_validator.json file if not present defaultConfig := cfg.DefaultBaseConfig() dir := filepath.Join(base, mach) - privValPath := filepath.Join(dir, defaultConfig.PrivValidator) - dir = filepath.Dir(privValPath) + pvPath := filepath.Join(dir, defaultConfig.PrivValidator) + dir = filepath.Dir(pvPath) err := cmn.EnsureDir(dir, 0700) if err != nil { return err } - ensurePrivValidator(privValPath) + ensurePrivValidator(pvPath) return nil } @@ -93,6 +94,6 @@ func ensurePrivValidator(file string) { if cmn.FileExists(file) { return } - privValidator := types.GenPrivValidatorFS(file) - privValidator.Save() + pv := pvm.GenFilePV(file) + pv.Save() } diff --git a/lite/files/commit.go b/lite/files/commit.go index 33f5bb677..8a7e4721e 100644 --- a/lite/files/commit.go +++ b/lite/files/commit.go @@ -1,13 +1,11 @@ package files import ( - "encoding/json" + "io/ioutil" "os" "github.com/pkg/errors" - wire "github.com/tendermint/go-wire" - "github.com/tendermint/tendermint/lite" liteErr "github.com/tendermint/tendermint/lite/errors" ) @@ -19,7 +17,7 @@ const ( MaxFullCommitSize = 1024 * 1024 ) -// SaveFullCommit exports the seed in binary / go-wire style +// SaveFullCommit exports the seed in binary / go-amino style func SaveFullCommit(fc lite.FullCommit, path string) error { f, err := os.Create(path) if err != nil { @@ -27,9 +25,11 @@ func SaveFullCommit(fc lite.FullCommit, path string) error { } defer f.Close() - var n int - wire.WriteBinary(fc, f, &n, &err) - return errors.WithStack(err) + _, err = cdc.MarshalBinaryWriter(f, fc) + if err != nil { + return errors.WithStack(err) + } + return nil } // SaveFullCommitJSON exports the seed in a json format @@ -39,9 +39,15 @@ func SaveFullCommitJSON(fc lite.FullCommit, path string) error { return errors.WithStack(err) } defer f.Close() - stream := json.NewEncoder(f) - err = stream.Encode(fc) - return errors.WithStack(err) + bz, err := cdc.MarshalJSON(fc) + if err != nil { + return errors.WithStack(err) + } + _, err = f.Write(bz) + if err != nil { + return errors.WithStack(err) + } + return nil } // LoadFullCommit loads the full commit from the file system. @@ -56,9 +62,11 @@ func LoadFullCommit(path string) (lite.FullCommit, error) { } defer f.Close() - var n int - wire.ReadBinaryPtr(&fc, f, MaxFullCommitSize, &n, &err) - return fc, errors.WithStack(err) + _, err = cdc.UnmarshalBinaryReader(f, &fc, 0) + if err != nil { + return fc, errors.WithStack(err) + } + return fc, nil } // LoadFullCommitJSON loads the commit from the file system in JSON format. @@ -73,7 +81,13 @@ func LoadFullCommitJSON(path string) (lite.FullCommit, error) { } defer f.Close() - stream := json.NewDecoder(f) - err = stream.Decode(&fc) - return fc, errors.WithStack(err) + bz, err := ioutil.ReadAll(f) + if err != nil { + return fc, errors.WithStack(err) + } + err = cdc.UnmarshalJSON(bz, &fc) + if err != nil { + return fc, errors.WithStack(err) + } + return fc, nil } diff --git a/lite/files/wire.go b/lite/files/wire.go new file mode 100644 index 000000000..99f98931e --- /dev/null +++ b/lite/files/wire.go @@ -0,0 +1,12 @@ +package files + +import ( + "github.com/tendermint/go-amino" + "github.com/tendermint/go-crypto" +) + +var cdc = amino.NewCodec() + +func init() { + crypto.RegisterAmino(cdc) +} diff --git a/lite/helpers.go b/lite/helpers.go index 7df77027c..714675afb 100644 --- a/lite/helpers.go +++ b/lite/helpers.go @@ -23,7 +23,7 @@ type ValKeys []crypto.PrivKey func GenValKeys(n int) ValKeys { res := make(ValKeys, n) for i := range res { - res[i] = crypto.GenPrivKeyEd25519().Wrap() + res[i] = crypto.GenPrivKeyEd25519() } return res } @@ -32,7 +32,7 @@ func GenValKeys(n int) ValKeys { func (v ValKeys) Change(i int) ValKeys { res := make(ValKeys, len(v)) copy(res, v) - res[i] = crypto.GenPrivKeyEd25519().Wrap() + res[i] = crypto.GenPrivKeyEd25519() return res } @@ -46,7 +46,7 @@ func (v ValKeys) Extend(n int) ValKeys { func GenSecpValKeys(n int) ValKeys { res := make(ValKeys, n) for i := range res { - res[i] = crypto.GenPrivKeySecp256k1().Wrap() + res[i] = crypto.GenPrivKeySecp256k1() } return res } diff --git a/lite/proxy/proxy.go b/lite/proxy/proxy.go index 34aa99fa0..fe10399dc 100644 --- a/lite/proxy/proxy.go +++ b/lite/proxy/proxy.go @@ -3,10 +3,12 @@ package proxy import ( "net/http" + "github.com/tendermint/go-amino" "github.com/tendermint/tmlibs/log" rpcclient "github.com/tendermint/tendermint/rpc/client" "github.com/tendermint/tendermint/rpc/core" + ctypes "github.com/tendermint/tendermint/rpc/core/types" rpc "github.com/tendermint/tendermint/rpc/lib/server" ) @@ -23,13 +25,15 @@ func StartProxy(c rpcclient.Client, listenAddr string, logger log.Logger) error return err } + cdc := amino.NewCodec() + ctypes.RegisterAmino(cdc) r := RPCRoutes(c) // build the handler... mux := http.NewServeMux() - rpc.RegisterRPCFuncs(mux, r, logger) + rpc.RegisterRPCFuncs(mux, r, cdc, logger) - wm := rpc.NewWebsocketManager(r, rpc.EventSubscriber(c)) + wm := rpc.NewWebsocketManager(r, cdc, rpc.EventSubscriber(c)) wm.SetLogger(logger) core.SetLogger(logger) mux.HandleFunc(wsEndpoint, wm.WebsocketHandler) diff --git a/types/event_bus_test.go b/types/event_bus_test.go index 9002b531c..40aa0746a 100644 --- a/types/event_bus_test.go +++ b/types/event_bus_test.go @@ -73,7 +73,7 @@ func benchmarkEventBus(numClients int, randQueries bool, randEvents bool, b *tes eventType = randEvent() } - eventBus.Publish(eventType, "Gamora") + eventBus.Publish(eventType, EventDataString("Gamora")) } } diff --git a/types/events.go b/types/events.go index cdffc0ee5..342d4bc20 100644 --- a/types/events.go +++ b/types/events.go @@ -47,15 +47,17 @@ func (_ EventDataTx) AssertIsTMEventData() {} func (_ EventDataRoundState) AssertIsTMEventData() {} func (_ EventDataVote) AssertIsTMEventData() {} func (_ EventDataProposalHeartbeat) AssertIsTMEventData() {} +func (_ EventDataString) AssertIsTMEventData() {} func RegisterEventDatas(cdc *amino.Codec) { cdc.RegisterInterface((*TMEventData)(nil), nil) - cdc.RegisterConcrete(EventDataNewBlock{}, "tendermint/EventDataNameNewBlock", nil) - cdc.RegisterConcrete(EventDataNewBlockHeader{}, "tendermint/EventDataNameNewBlockHeader", nil) - cdc.RegisterConcrete(EventDataTx{}, "tendermint/EventDataNameTx", nil) - cdc.RegisterConcrete(EventDataRoundState{}, "tendermint/EventDataNameRoundState", nil) - cdc.RegisterConcrete(EventDataVote{}, "tendermint/EventDataNameVote", nil) - cdc.RegisterConcrete(EventDataProposalHeartbeat{}, "tendermint/EventDataNameProposalHeartbeat", nil) + cdc.RegisterConcrete(EventDataNewBlock{}, "tendermint/event/NewBlock", nil) + cdc.RegisterConcrete(EventDataNewBlockHeader{}, "tendermint/event/NewBlockHeader", nil) + cdc.RegisterConcrete(EventDataTx{}, "tendermint/event/Tx", nil) + cdc.RegisterConcrete(EventDataRoundState{}, "tendermint/event/RoundState", nil) + cdc.RegisterConcrete(EventDataVote{}, "tendermint/event/Vote", nil) + cdc.RegisterConcrete(EventDataProposalHeartbeat{}, "tendermint/event/ProposalHeartbeat", nil) + cdc.RegisterConcrete(EventDataString(""), "tendermint/event/ProposalString", nil) } // Most event messages are basic types (a block, a transaction) @@ -93,6 +95,8 @@ type EventDataVote struct { Vote *Vote } +type EventDataString string + /////////////////////////////////////////////////////////////////////////////// // PUBSUB /////////////////////////////////////////////////////////////////////////////// From 3233c318ea567d732c7905532ad77946690eb26e Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Fri, 6 Apr 2018 12:35:48 +0200 Subject: [PATCH 068/143] only log errors, dial correct addresses "this means if there are lookup errors or typos in the persistent_peers, tendermint will fail to start ? didn't some one ask for us not to do this previously ?" --- p2p/switch.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/p2p/switch.go b/p2p/switch.go index 9f65a85e0..e412d3cc9 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -350,12 +350,10 @@ func (sw *Switch) IsDialing(id ID) bool { // DialPeersAsync dials a list of peers asynchronously in random order (optionally, making them persistent). func (sw *Switch) DialPeersAsync(addrBook AddrBook, peers []string, persistent bool) error { netAddrs, errs := NewNetAddressStrings(peers) + // only log errors, dial correct addresses for _, err := range errs { sw.Logger.Error("Error in peer's address", "err", err) } - if len(errs) > 0 { - return errors.New("Errors in peer addresses (see errors above)") - } ourAddr := sw.nodeInfo.NetAddress() From 3d32474da8fd687e9408a5a692a89c9d99afd173 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Fri, 6 Apr 2018 13:26:05 +0200 Subject: [PATCH 069/143] make linter happy --- p2p/switch_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/p2p/switch_test.go b/p2p/switch_test.go index 8fec3e7ab..c02eb26d3 100644 --- a/p2p/switch_test.go +++ b/p2p/switch_test.go @@ -91,8 +91,8 @@ func MakeSwitchPair(t testing.TB, initSwitch func(int, *Switch) *Switch) (*Switc func initSwitchFunc(i int, sw *Switch) *Switch { sw.SetAddrBook(&addrBookMock{ - addrs: make(map[string]struct{}, 0), - ourAddrs: make(map[string]struct{}, 0)}) + addrs: make(map[string]struct{}), + ourAddrs: make(map[string]struct{})}) // Make two reactors of two channels each sw.AddReactor("foo", NewTestReactor([]*conn.ChannelDescriptor{ From 9cc2cf362fdbd15821bcc832beeb67425b9c06f0 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 6 Apr 2018 23:03:27 +0300 Subject: [PATCH 070/143] changelog and version --- CHANGELOG.md | 21 ++++++++++++++++----- version/version.go | 6 +++--- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 267100510..c5dc06753 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,6 @@ BREAKING CHANGES: - Upgrade consensus for more real-time use of evidence FEATURES: -- Peer reputation management - Use the chain as its own CA for nodes and validators - Tooling to run multiple blockchains/apps, possibly in a single process - State syncing (without transaction replay) @@ -25,20 +24,32 @@ BUG FIXES: - Graceful handling/recovery for apps that have non-determinism or fail to halt - Graceful handling/recovery for violations of safety, or liveness -## 0.18.0 (TBD) +## 0.18.0 (April 6th, 2018) BREAKING: + +- [types] Merkle tree uses different encoding for varints (see tmlibs v0.8.0) +- [types] ValidtorSet.GetByAddress returns -1 if no validator found - [p2p] require all addresses come with an ID no matter what +- [rpc] Listening address must contain tcp:// or unix:// prefix + +FEATURES: + +- [rpc] StartHTTPAndTLSServer (not used yet) +- [rpc] Include validator's voting power in `/status` +- [rpc] `/tx` and `/tx_search` responses now include the transaction hash +- [rpc] Include peer NodeIDs in `/net_info` IMPROVEMENTS: -- [p2p] seeds respond with a bias towards good peers - [config] trim whitespace from elements of lists (like `persistent_peers`) -- [rpc] `/tx` and `/tx_search` responses now include the transaction hash -- [rpc] include validator power in `/status` +- [rpc] `/tx_search` results are sorted by height - [p2p] do not try to connect to ourselves (ok, maybe only once) +- [p2p] seeds respond with a bias towards good peers BUG FIXES: - [rpc] fix subscribing using an abci.ResponseDeliverTx tag +- [rpc] fix tx_indexers matchRange +- [rpc] fix unsubscribing (see tmlibs v0.8.0) ## 0.17.1 (March 27th, 2018) diff --git a/version/version.go b/version/version.go index 74662b808..a8fbfa84f 100644 --- a/version/version.go +++ b/version/version.go @@ -1,13 +1,13 @@ package version const Maj = "0" -const Min = "17" -const Fix = "1" +const Min = "18" +const Fix = "0" var ( // Version is the current version of Tendermint // Must be a string because scripts like dist.sh read this file. - Version = "0.17.1" + Version = "0.18.0" // GitCommit is the current HEAD set using ldflags. GitCommit string From fb64314d1c567826e502c417db8b1de56dc196a8 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Fri, 6 Apr 2018 13:46:40 -0700 Subject: [PATCH 071/143] Review from Anton --- blockchain/reactor.go | 20 ++++++++++------- blockchain/store.go | 13 ++++------- blockchain/store_test.go | 2 +- consensus/common_test.go | 8 +++---- consensus/reactor.go | 35 ++++++++++++++++-------------- consensus/replay_test.go | 4 ++-- consensus/state.go | 2 +- consensus/wal.go | 2 +- consensus/wal_generator.go | 5 ++--- evidence/reactor.go | 2 +- node/node.go | 10 ++++----- p2p/conn/connection.go | 4 ++-- p2p/conn/connection_test.go | 3 ++- p2p/conn/secret_connection_test.go | 5 +++-- rpc/lib/server/handlers.go | 2 +- rpc/test/helpers.go | 8 +++---- 16 files changed, 64 insertions(+), 61 deletions(-) diff --git a/blockchain/reactor.go b/blockchain/reactor.go index 314e0b0e0..83fea54e4 100644 --- a/blockchain/reactor.go +++ b/blockchain/reactor.go @@ -26,6 +26,13 @@ const ( statusUpdateIntervalSeconds = 10 // check if we should switch to consensus reactor switchToConsensusIntervalSeconds = 1 + + // NOTE: keep up to date with bcBlockResponseMessage + bcBlockResponseMessagePrefixSize = 4 + bcBlockResponseMessageFieldKeySize = 1 + maxMessageSize = types.MaxBlockSizeBytes + + bcBlockResponseMessagePrefixSize + + bcBlockResponseMessageFieldKeySize ) type consensusReactor interface { @@ -122,9 +129,11 @@ func (bcR *BlockchainReactor) OnStop() { func (bcR *BlockchainReactor) GetChannels() []*p2p.ChannelDescriptor { return []*p2p.ChannelDescriptor{ { - ID: BlockchainChannel, - Priority: 10, - SendQueueCapacity: 1000, + ID: BlockchainChannel, + Priority: 10, + SendQueueCapacity: 1000, + RecvBufferCapacity: 50 * 4096, + RecvMessageCapacity: maxMessageSize, }, } } @@ -336,11 +345,6 @@ func RegisterBlockchainMessages(cdc *amino.Codec) { // DecodeMessage decodes BlockchainMessage. // TODO: ensure that bz is completely read. func DecodeMessage(bz []byte) (msg BlockchainMessage, err error) { - const ( - prefixSize = 4 - fieldKeySize = 1 - maxMessageSize = types.MaxBlockSizeBytes + prefixSize + fieldKeySize - ) err = cdc.UnmarshalBinaryBare(bz, &msg) if err != nil { err = cmn.ErrorWrap(err, "DecodeMessage() had bytes left over") diff --git a/blockchain/store.go b/blockchain/store.go index e322e4145..a974009e2 100644 --- a/blockchain/store.go +++ b/blockchain/store.go @@ -53,15 +53,10 @@ func (bs *BlockStore) Height() int64 { // LoadBlock returns the block with the given height. // If no block is found for that height, it returns nil. func (bs *BlockStore) LoadBlock(height int64) *types.Block { - var blockMeta = new(types.BlockMeta) - bz := bs.db.Get(calcBlockMetaKey(height)) - if len(bz) == 0 { + var blockMeta = bs.LoadBlockMeta(height) + if blockMeta == nil { return nil } - err := cdc.UnmarshalBinaryBare(bz, blockMeta) - if err != nil { - panic(cmn.ErrorWrap(err, "Error reading block meta")) - } var block = new(types.Block) buf := []byte{} @@ -69,7 +64,7 @@ func (bs *BlockStore) LoadBlock(height int64) *types.Block { part := bs.LoadBlockPart(height, i) buf = append(buf, part.Bytes...) } - err = cdc.UnmarshalBinary(buf, block) + err := cdc.UnmarshalBinary(buf, block) if err != nil { // NOTE: The existence of meta should imply the existence of the // block. So, make sure meta is only saved after blocks are saved. @@ -137,7 +132,7 @@ func (bs *BlockStore) LoadSeenCommit(height int64) *types.Commit { } err := cdc.UnmarshalBinaryBare(bz, commit) if err != nil { - panic(cmn.ErrorWrap(err, "Error reading block commit")) + panic(cmn.ErrorWrap(err, "Error reading block seen commit")) } return commit } diff --git a/blockchain/store_test.go b/blockchain/store_test.go index b2188043e..8e690e6d2 100644 --- a/blockchain/store_test.go +++ b/blockchain/store_test.go @@ -179,7 +179,7 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) { seenCommit: seenCommit1, corruptSeenCommitInDB: true, - wantPanic: "Error reading block commit", + wantPanic: "Error reading block seen commit", }, { diff --git a/consensus/common_test.go b/consensus/common_test.go index fe95210a9..6f6b1967d 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -21,7 +21,7 @@ import ( "github.com/tendermint/tendermint/p2p" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" - privval "github.com/tendermint/tendermint/types/priv_validator" + pvm "github.com/tendermint/tendermint/types/priv_validator" cmn "github.com/tendermint/tmlibs/common" dbm "github.com/tendermint/tmlibs/db" "github.com/tendermint/tmlibs/log" @@ -278,10 +278,10 @@ func newConsensusStateWithConfigAndBlockStore(thisConfig *cfg.Config, state sm.S return cs } -func loadPrivValidator(config *cfg.Config) *privval.FilePV { +func loadPrivValidator(config *cfg.Config) *pvm.FilePV { privValidatorFile := config.PrivValidatorFile() ensureDir(path.Dir(privValidatorFile), 0700) - privValidator := privval.LoadOrGenFilePV(privValidatorFile) + privValidator := pvm.LoadOrGenFilePV(privValidatorFile) privValidator.Reset() return privValidator } @@ -379,7 +379,7 @@ func randConsensusNetWithPeers(nValidators, nPeers int, testName string, tickerF privVal = privVals[i] } else { _, tempFilePath := cmn.Tempfile("priv_validator_") - privVal = privval.GenFilePV(tempFilePath) + privVal = pvm.GenFilePV(tempFilePath) } app := appFunc() diff --git a/consensus/reactor.go b/consensus/reactor.go index 85943c214..052b58e93 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -107,27 +107,31 @@ func (conR *ConsensusReactor) GetChannels() []*p2p.ChannelDescriptor { // TODO optimize return []*p2p.ChannelDescriptor{ { - ID: StateChannel, - Priority: 5, - SendQueueCapacity: 100, + ID: StateChannel, + Priority: 5, + SendQueueCapacity: 100, + RecvMessageCapacity: maxConsensusMessageSize, }, { - ID: DataChannel, // maybe split between gossiping current block and catchup stuff - Priority: 10, // once we gossip the whole block there's nothing left to send until next height or round - SendQueueCapacity: 100, - RecvBufferCapacity: 50 * 4096, + ID: DataChannel, // maybe split between gossiping current block and catchup stuff + Priority: 10, // once we gossip the whole block there's nothing left to send until next height or round + SendQueueCapacity: 100, + RecvBufferCapacity: 50 * 4096, + RecvMessageCapacity: maxConsensusMessageSize, }, { - ID: VoteChannel, - Priority: 5, - SendQueueCapacity: 100, - RecvBufferCapacity: 100 * 100, + ID: VoteChannel, + Priority: 5, + SendQueueCapacity: 100, + RecvBufferCapacity: 100 * 100, + RecvMessageCapacity: maxConsensusMessageSize, }, { - ID: VoteSetBitsChannel, - Priority: 1, - SendQueueCapacity: 2, - RecvBufferCapacity: 1024, + ID: VoteSetBitsChannel, + Priority: 1, + SendQueueCapacity: 2, + RecvBufferCapacity: 1024, + RecvMessageCapacity: maxConsensusMessageSize, }, } } @@ -1278,7 +1282,6 @@ func RegisterConsensusMessages(cdc *amino.Codec) { } // DecodeMessage decodes the given bytes into a ConsensusMessage. -// TODO: check for unnecessary extra bytes at the end. func DecodeMessage(bz []byte) (msg ConsensusMessage, err error) { err = cdc.UnmarshalBinaryBare(bz, &msg) return diff --git a/consensus/replay_test.go b/consensus/replay_test.go index b3c6ac4a4..7a69de379 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -26,7 +26,7 @@ import ( "github.com/tendermint/tendermint/proxy" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" - privval "github.com/tendermint/tendermint/types/priv_validator" + pvm "github.com/tendermint/tendermint/types/priv_validator" "github.com/tendermint/tmlibs/log" ) @@ -325,7 +325,7 @@ func testHandshakeReplay(t *testing.T, nBlocks int, mode uint) { walFile := tempWALWithData(walBody) config.Consensus.SetWalFile(walFile) - privVal := privval.LoadFilePV(config.PrivValidatorFile()) + privVal := pvm.LoadFilePV(config.PrivValidatorFile()) wal, err := NewWAL(walFile, false) if err != nil { diff --git a/consensus/state.go b/consensus/state.go index 0f086172c..dde97612f 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -778,7 +778,7 @@ func (cs *ConsensusState) enterPropose(height int64, round int) { // if not a validator, we're done if !cs.Validators.HasAddress(cs.privValidator.GetAddress()) { - cs.Logger.Debug("This node is not a validator 2", cs.privValidator.GetAddress(), cs.Validators) + cs.Logger.Debug("This node is not a validator", "addr", cs.privValidator.GetAddress(), "vals", cs.Validators) return } cs.Logger.Debug("This node is a validator") diff --git a/consensus/wal.go b/consensus/wal.go index bc9afff64..694e2516e 100644 --- a/consensus/wal.go +++ b/consensus/wal.go @@ -42,7 +42,7 @@ func RegisterWALMessages(cdc *amino.Codec) { cdc.RegisterConcrete(types.EventDataRoundState{}, "tendermint/wal/EventDataRoundState", nil) cdc.RegisterConcrete(msgInfo{}, "tendermint/wal/MsgInfo", nil) cdc.RegisterConcrete(timeoutInfo{}, "tendermint/wal/TimeoutInfo", nil) - cdc.RegisterConcrete(EndHeightMessage{}, "tendermint/wal/EndHeightMessagE", nil) + cdc.RegisterConcrete(EndHeightMessage{}, "tendermint/wal/EndHeightMessage", nil) } //-------------------------------------------------------- diff --git a/consensus/wal_generator.go b/consensus/wal_generator.go index 65de399d4..d8226ab5d 100644 --- a/consensus/wal_generator.go +++ b/consensus/wal_generator.go @@ -17,7 +17,7 @@ import ( "github.com/tendermint/tendermint/proxy" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" - privval "github.com/tendermint/tendermint/types/priv_validator" + pvm "github.com/tendermint/tendermint/types/priv_validator" auto "github.com/tendermint/tmlibs/autofile" cmn "github.com/tendermint/tmlibs/common" "github.com/tendermint/tmlibs/db" @@ -41,7 +41,7 @@ func WALWithNBlocks(numBlocks int) (data []byte, err error) { // COPY PASTE FROM node.go WITH A FEW MODIFICATIONS // NOTE: we can't import node package because of circular dependency privValidatorFile := config.PrivValidatorFile() - privValidator := privval.LoadOrGenFilePV(privValidatorFile) + privValidator := pvm.LoadOrGenFilePV(privValidatorFile) genDoc, err := types.GenesisDocFromFile(config.GenesisFile()) if err != nil { return nil, errors.Wrap(err, "failed to read genesis file") @@ -72,7 +72,6 @@ func WALWithNBlocks(numBlocks int) (data []byte, err error) { consensusState := NewConsensusState(config.Consensus, state.Copy(), blockExec, blockStore, mempool, evpool) consensusState.SetLogger(logger) consensusState.SetEventBus(eventBus) - fmt.Println(">>privval", privValidator) if privValidator != nil { consensusState.SetPrivValidator(privValidator) } diff --git a/evidence/reactor.go b/evidence/reactor.go index aeab4d831..27cbdbf31 100644 --- a/evidence/reactor.go +++ b/evidence/reactor.go @@ -141,7 +141,7 @@ type EvidenceMessage interface{} func RegisterEvidenceMessages(cdc *amino.Codec) { cdc.RegisterInterface((*EvidenceMessage)(nil), nil) cdc.RegisterConcrete(&EvidenceListMessage{}, - "tendermint/evidence/EvidenceListMessagE", nil) + "tendermint/evidence/EvidenceListMessage", nil) } // DecodeMessage decodes a byte-array into a EvidenceMessage. diff --git a/node/node.go b/node/node.go index 221eb1eb0..3ed790390 100644 --- a/node/node.go +++ b/node/node.go @@ -35,7 +35,7 @@ import ( "github.com/tendermint/tendermint/state/txindex/kv" "github.com/tendermint/tendermint/state/txindex/null" "github.com/tendermint/tendermint/types" - privval "github.com/tendermint/tendermint/types/priv_validator" + pvm "github.com/tendermint/tendermint/types/priv_validator" "github.com/tendermint/tendermint/version" _ "net/http/pprof" @@ -80,7 +80,7 @@ type NodeProvider func(*cfg.Config, log.Logger) (*Node, error) // It implements NodeProvider. func DefaultNewNode(config *cfg.Config, logger log.Logger) (*Node, error) { return NewNode(config, - privval.LoadOrGenFilePV(config.PrivValidatorFile()), + pvm.LoadOrGenFilePV(config.PrivValidatorFile()), proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir()), DefaultGenesisDocProviderFunc(config), DefaultDBProvider, @@ -181,8 +181,8 @@ func NewNode(config *cfg.Config, // TODO: persist this key so external signer // can actually authenticate us privKey = crypto.GenPrivKeyEd25519() - pvsc = privval.NewSocketPV( - logger.With("module", "privval"), + pvsc = pvm.NewSocketPV( + logger.With("module", "pvm"), config.PrivValidatorListenAddr, privKey, ) @@ -446,7 +446,7 @@ func (n *Node) OnStop() { n.eventBus.Stop() n.indexerService.Stop() - if pvsc, ok := n.privValidator.(*privval.SocketPV); ok { + if pvsc, ok := n.privValidator.(*pvm.SocketPV); ok { if err := pvsc.Stop(); err != nil { n.Logger.Error("Error stopping priv validator socket client", "err", err) } diff --git a/p2p/conn/connection.go b/p2p/conn/connection.go index 2970d960f..d0736ad3c 100644 --- a/p2p/conn/connection.go +++ b/p2p/conn/connection.go @@ -18,8 +18,8 @@ import ( ) const ( - maxPacketMsgPayloadSizeDefault = 1024 - maxPacketMsgOverheadSize = 14 + maxPacketMsgPayloadSizeDefault = 1024 // NOTE: Must be below 16,384 bytes for 14 below. + maxPacketMsgOverheadSize = 14 // NOTE: See connection_test for derivation. numBatchPacketMsgs = 10 minReadBufferSize = 1024 diff --git a/p2p/conn/connection_test.go b/p2p/conn/connection_test.go index 9f03ec62b..a927d6959 100644 --- a/p2p/conn/connection_test.go +++ b/p2p/conn/connection_test.go @@ -136,7 +136,8 @@ func TestMConnectionPongTimeoutResultsInError(t *testing.T) { go func() { // read ping var pkt PacketPing - _, err = cdc.UnmarshalBinaryReader(server, &pkt, 1024) + const maxPacketPingSize = 1024 + _, err = cdc.UnmarshalBinaryReader(server, &pkt, maxPacketPingSize) assert.Nil(t, err) serverGotPing <- struct{}{} }() diff --git a/p2p/conn/secret_connection_test.go b/p2p/conn/secret_connection_test.go index 7d3f17b2f..436423020 100644 --- a/p2p/conn/secret_connection_test.go +++ b/p2p/conn/secret_connection_test.go @@ -240,9 +240,10 @@ func BenchmarkSecretConnection(b *testing.B) { } func fingerprint(bz []byte) []byte { - if len(bz) < 40 { + const fbsize = 40 + if len(bz) < fbsize { return bz } else { - return bz[:40] + return bz[:fbsize] } } diff --git a/rpc/lib/server/handlers.go b/rpc/lib/server/handlers.go index d59372206..07ccfb6f3 100644 --- a/rpc/lib/server/handlers.go +++ b/rpc/lib/server/handlers.go @@ -208,7 +208,7 @@ func jsonParamsToArgs(rpcFunc *RPCFunc, cdc *amino.Codec, raw []byte, argsOffset return mapParamsToArgs(rpcFunc, cdc, m, argsOffset) } - // Ttherwise, try an array. + // Otherwise, try an array. var a []json.RawMessage err = json.Unmarshal(raw, &a) if err == nil { diff --git a/rpc/test/helpers.go b/rpc/test/helpers.go index 38c2dfaf0..11cafa0cd 100644 --- a/rpc/test/helpers.go +++ b/rpc/test/helpers.go @@ -19,7 +19,7 @@ import ( ctypes "github.com/tendermint/tendermint/rpc/core/types" core_grpc "github.com/tendermint/tendermint/rpc/grpc" rpcclient "github.com/tendermint/tendermint/rpc/lib/client" - privval "github.com/tendermint/tendermint/types/priv_validator" + pvm "github.com/tendermint/tendermint/types/priv_validator" ) var globalConfig *cfg.Config @@ -117,10 +117,10 @@ func NewTendermint(app abci.Application) *nm.Node { config := GetConfig() logger := log.NewTMLogger(log.NewSyncWriter(os.Stdout)) logger = log.NewFilter(logger, log.AllowError()) - privValidatorFile := config.PrivValidatorFile() - privValidator := privval.LoadOrGenFilePV(privValidatorFile) + pvFile := config.PrivValidatorFile() + pv := pvm.LoadOrGenFilePV(pvFile) papp := proxy.NewLocalClientCreator(app) - node, err := nm.NewNode(config, privValidator, papp, + node, err := nm.NewNode(config, pv, papp, nm.DefaultGenesisDocProviderFunc(config), nm.DefaultDBProvider, logger) if err != nil { From d24e4cb82194e1a3b91b98b74ebbf4dbb7317bb0 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Fri, 6 Apr 2018 17:02:29 -0700 Subject: [PATCH 072/143] Fix race testing --- consensus/replay_test.go | 2 +- types/block.go | 16 ++++++++++++---- types/priv_validator/socket_test.go | 2 +- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/consensus/replay_test.go b/consensus/replay_test.go index 7a69de379..1291fb462 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -60,7 +60,7 @@ func startNewConsensusStateAndWaitForBlock(t *testing.T, lastBlockHeight int64, bytes, _ := ioutil.ReadFile(cs.config.WalFile()) // fmt.Printf("====== WAL: \n\r%s\n", bytes) - t.Logf("====== WAL: \n\r%s\n", bytes) + t.Logf("====== WAL: \n\r%X\n", bytes) err := cs.Start() require.NoError(t, err) diff --git a/types/block.go b/types/block.go index df39d3305..3a12462e3 100644 --- a/types/block.go +++ b/types/block.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "strings" + "sync" "time" cmn "github.com/tendermint/tmlibs/common" @@ -15,6 +16,7 @@ import ( // Block defines the atomic unit of a Tendermint blockchain. // TODO: add Version byte type Block struct { + mtx sync.Mutex *Header `json:"header"` *Data `json:"data"` Evidence EvidenceData `json:"evidence"` @@ -35,7 +37,7 @@ func MakeBlock(height int64, txs []Tx, commit *Commit) *Block { Txs: txs, }, } - block.FillHeader() + block.fillHeader() return block } @@ -68,8 +70,8 @@ func (b *Block) ValidateBasic() error { return nil } -// FillHeader fills in any remaining header fields that are a function of the block data -func (b *Block) FillHeader() { +// fillHeader fills in any remaining header fields that are a function of the block data +func (b *Block) fillHeader() { if b.LastCommitHash == nil { b.LastCommitHash = b.LastCommit.Hash() } @@ -84,10 +86,16 @@ func (b *Block) FillHeader() { // Hash computes and returns the block hash. // If the block is incomplete, block hash is nil for safety. func (b *Block) Hash() cmn.HexBytes { + if b == nil { + return nil + } + b.mtx.Lock() + defer b.mtx.Unlock() + if b == nil || b.Header == nil || b.Data == nil || b.LastCommit == nil { return nil } - b.FillHeader() + b.fillHeader() return b.Header.Hash() } diff --git a/types/priv_validator/socket_test.go b/types/priv_validator/socket_test.go index eb0f4c12d..197c2508e 100644 --- a/types/priv_validator/socket_test.go +++ b/types/priv_validator/socket_test.go @@ -133,7 +133,7 @@ func TestSocketPVDeadline(t *testing.T) { ) ) - SocketPVConnDeadline(10 * time.Millisecond)(sc) + SocketPVConnDeadline(100 * time.Millisecond)(sc) SocketPVConnWait(500 * time.Millisecond)(sc) go func(sc *SocketPV) { From 02531ca5a3a3d521e33bffa528583ff5b78d0594 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Fri, 6 Apr 2018 17:03:31 -0700 Subject: [PATCH 073/143] Fix race testing (cont;) Bump version to 0.19.0 --- types/block.go | 6 ++++++ version/version.go | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/types/block.go b/types/block.go index 3a12462e3..bd2a1f00f 100644 --- a/types/block.go +++ b/types/block.go @@ -49,6 +49,12 @@ func (b *Block) AddEvidence(evidence []Evidence) { // ValidateBasic performs basic validation that doesn't involve state data. // It checks the internal consistency of the block. func (b *Block) ValidateBasic() error { + if b == nil { + return errors.New("Nil blocks are invalid") + } + b.mtx.Lock() + defer b.mtx.Unlock() + newTxs := int64(len(b.Data.Txs)) if b.NumTxs != newTxs { return fmt.Errorf("Wrong Block.Header.NumTxs. Expected %v, got %v", newTxs, b.NumTxs) diff --git a/version/version.go b/version/version.go index a45095428..b0123eb1f 100644 --- a/version/version.go +++ b/version/version.go @@ -1,13 +1,13 @@ package version const Maj = "0" -const Min = "16" +const Min = "19" const Fix = "0" var ( // Version is the current version of Tendermint // Must be a string because scripts like dist.sh read this file. - Version = "0.16.0" + Version = "0.19.0" // GitCommit is the current HEAD set using ldflags. GitCommit string From 7afe74a963b5222bbc063545f4a9261d27b188de Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Sat, 7 Apr 2018 02:01:45 -0700 Subject: [PATCH 074/143] Update go-crypto to 0.6.1 and change config/toml.go privval address --- Gopkg.lock | 6 +++--- Gopkg.toml | 2 +- config/toml.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 2d6fe8b93..6e6baee27 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -260,8 +260,8 @@ [[projects]] name = "github.com/tendermint/go-crypto" packages = ["."] - revision = "a04f2ae5c6ba037014531241ed973116290b029e" - version = "v0.6.0" + revision = "5d5f580f49ca66c13400938c64334186068c8b7c" + version = "v0.6.1" [[projects]] name = "github.com/tendermint/go-wire" @@ -383,6 +383,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "d14dbd59436d0ea3b322c42ce33c213b26cd2451ba023a466764b8002e0e649d" + inputs-digest = "ed1149ed5293b7b28b7505627648c6a1152aaff0ed06a3849995b29751ae00f3" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index d689da986..a617b6e56 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -75,7 +75,7 @@ [[constraint]] name = "github.com/tendermint/go-crypto" - version = "0.6.0" + version = "0.6.1" [[constraint]] name = "github.com/tendermint/go-amino" diff --git a/config/toml.go b/config/toml.go index 23a6fecde..67b801f77 100644 --- a/config/toml.go +++ b/config/toml.go @@ -287,7 +287,7 @@ var testGenesis = `{ }` var testPrivValidator = `{ - "address": "515252BC8788931ADFF4796A389C9B1F7925A1B7", + "address": "849CB2C877F87A20925F35D00AE6688342D25B47", "pub_key": { "type": "AC26791624DE60", "value": "AT/+aaL1eB0477Mud9JMm8Sh8BIvOYlPGC9KkIUmFaE=" From 02c0835e9b690ee03d048cc8b84ec772372d6555 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sat, 7 Apr 2018 16:25:10 +0300 Subject: [PATCH 075/143] fixes post merge --- Gopkg.lock | 426 ------------------------------ Gopkg.toml | 18 +- consensus/reactor_test.go | 13 +- p2p/dummy/peer.go | 4 +- types/priv_validator/sign_info.go | 238 ----------------- 5 files changed, 9 insertions(+), 690 deletions(-) delete mode 100644 Gopkg.lock delete mode 100644 types/priv_validator/sign_info.go diff --git a/Gopkg.lock b/Gopkg.lock deleted file mode 100644 index 3b5c21615..000000000 --- a/Gopkg.lock +++ /dev/null @@ -1,426 +0,0 @@ -# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. - - -[[projects]] - branch = "master" - name = "github.com/btcsuite/btcd" - packages = ["btcec"] - revision = "2be2f12b358dc57d70b8f501b00be450192efbc3" - -[[projects]] - name = "github.com/davecgh/go-spew" - packages = ["spew"] - revision = "346938d642f2ec3594ed81d874461961cd0faa76" - version = "v1.1.0" - -[[projects]] - branch = "master" - name = "github.com/ebuchman/fail-test" - packages = ["."] - revision = "95f809107225be108efcf10a3509e4ea6ceef3c4" - -[[projects]] - name = "github.com/fortytw2/leaktest" - packages = ["."] - revision = "a5ef70473c97b71626b9abeda80ee92ba2a7de9e" - version = "v1.2.0" - -[[projects]] - name = "github.com/fsnotify/fsnotify" - packages = ["."] - revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9" - version = "v1.4.7" - -[[projects]] - name = "github.com/go-kit/kit" - packages = [ - "log", - "log/level", - "log/term" - ] - revision = "4dc7be5d2d12881735283bcab7352178e190fc71" - version = "v0.6.0" - -[[projects]] - name = "github.com/go-logfmt/logfmt" - packages = ["."] - revision = "390ab7935ee28ec6b286364bba9b4dd6410cb3d5" - version = "v0.3.0" - -[[projects]] - name = "github.com/go-stack/stack" - packages = ["."] - revision = "259ab82a6cad3992b4e21ff5cac294ccb06474bc" - version = "v1.7.0" - -[[projects]] - name = "github.com/gogo/protobuf" - packages = [ - "gogoproto", - "jsonpb", - "proto", - "protoc-gen-gogo/descriptor", - "sortkeys", - "types" - ] - revision = "1adfc126b41513cc696b209667c8656ea7aac67c" - version = "v1.0.0" - -[[projects]] - name = "github.com/golang/protobuf" - packages = [ - "proto", - "ptypes", - "ptypes/any", - "ptypes/duration", - "ptypes/timestamp" - ] - revision = "925541529c1fa6821df4e44ce2723319eb2be768" - version = "v1.0.0" - -[[projects]] - branch = "master" - name = "github.com/golang/snappy" - packages = ["."] - revision = "553a641470496b2327abcac10b36396bd98e45c9" - -[[projects]] - name = "github.com/gorilla/websocket" - packages = ["."] - revision = "ea4d1f681babbce9545c9c5f3d5194a789c89f5b" - version = "v1.2.0" - -[[projects]] - branch = "master" - name = "github.com/hashicorp/hcl" - packages = [ - ".", - "hcl/ast", - "hcl/parser", - "hcl/printer", - "hcl/scanner", - "hcl/strconv", - "hcl/token", - "json/parser", - "json/scanner", - "json/token" - ] -<<<<<<< HEAD - revision = "f40e974e75af4e271d97ce0fc917af5898ae7bda" -======= - revision = "ef8a98b0bbce4a65b5aa4c368430a80ddc533168" ->>>>>>> develop - -[[projects]] - name = "github.com/inconshreveable/mousetrap" - packages = ["."] - revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75" - version = "v1.0" - -[[projects]] - branch = "master" - name = "github.com/jmhodges/levigo" - packages = ["."] - revision = "c42d9e0ca023e2198120196f842701bb4c55d7b9" - -[[projects]] - branch = "master" - name = "github.com/kr/logfmt" - packages = ["."] - revision = "b84e30acd515aadc4b783ad4ff83aff3299bdfe0" - -[[projects]] - name = "github.com/magiconair/properties" - packages = ["."] - revision = "c3beff4c2358b44d0493c7dda585e7db7ff28ae6" - version = "v1.7.6" - -[[projects]] - branch = "master" - name = "github.com/mitchellh/mapstructure" - packages = ["."] - revision = "00c29f56e2386353d58c599509e8dc3801b0d716" - -[[projects]] - name = "github.com/pelletier/go-toml" - packages = ["."] - revision = "acdc4509485b587f5e675510c4f2c63e90ff68a8" - version = "v1.1.0" - -[[projects]] - name = "github.com/pkg/errors" - packages = ["."] - revision = "645ef00459ed84a119197bfb8d8205042c6df63d" - version = "v0.8.0" - -[[projects]] - name = "github.com/pmezard/go-difflib" - packages = ["difflib"] - revision = "792786c7400a136282c1664665ae0a8db921c6c2" - version = "v1.0.0" - -[[projects]] - branch = "master" - name = "github.com/rcrowley/go-metrics" - packages = ["."] - revision = "8732c616f52954686704c8645fe1a9d59e9df7c1" - -[[projects]] - name = "github.com/spf13/afero" - packages = [ - ".", - "mem" - ] - revision = "63644898a8da0bc22138abf860edaf5277b6102e" - version = "v1.1.0" - -[[projects]] - name = "github.com/spf13/cast" - packages = ["."] - revision = "8965335b8c7107321228e3e3702cab9832751bac" - version = "v1.2.0" - -[[projects]] - name = "github.com/spf13/cobra" - packages = ["."] - revision = "a1f051bc3eba734da4772d60e2d677f47cf93ef4" - version = "v0.0.2" - -[[projects]] - branch = "master" - name = "github.com/spf13/jwalterweatherman" - packages = ["."] - revision = "7c0cea34c8ece3fbeb2b27ab9b59511d360fb394" - -[[projects]] - name = "github.com/spf13/pflag" - packages = ["."] - revision = "e57e3eeb33f795204c1ca35f56c44f83227c6e66" - version = "v1.0.0" - -[[projects]] - name = "github.com/spf13/viper" - packages = ["."] - revision = "b5e8006cbee93ec955a89ab31e0e3ce3204f3736" - version = "v1.0.2" - -[[projects]] - name = "github.com/stretchr/testify" - packages = [ - "assert", - "require" - ] - revision = "12b6f73e6084dad08a7c6e575284b177ecafbc71" - version = "v1.2.1" - -[[projects]] - branch = "master" - name = "github.com/syndtr/goleveldb" - packages = [ - "leveldb", - "leveldb/cache", - "leveldb/comparer", - "leveldb/errors", - "leveldb/filter", - "leveldb/iterator", - "leveldb/journal", - "leveldb/memdb", - "leveldb/opt", - "leveldb/storage", - "leveldb/table", - "leveldb/util" - ] -<<<<<<< HEAD - revision = "169b1b37be738edb2813dab48c97a549bcf99bb5" -======= - revision = "714f901b98fdb3aa954b4193d8cbd64a28d80cad" ->>>>>>> develop - -[[projects]] - name = "github.com/tendermint/abci" - packages = [ - "client", - "example/code", - "example/counter", - "example/kvstore", - "server", - "types" - ] -<<<<<<< HEAD - revision = "c62aed95f2ce399ec815b0cafe478af002cdc4e6" - version = "v0.10.3-dev" -======= - revision = "46686763ba8ea595ede16530ed4a40fb38f49f94" - version = "v0.10.2" ->>>>>>> develop - -[[projects]] - branch = "master" - name = "github.com/tendermint/ed25519" - packages = [ - ".", - "edwards25519", - "extra25519" - ] - revision = "d8387025d2b9d158cf4efb07e7ebf814bcce2057" - -[[projects]] - name = "github.com/tendermint/go-amino" - packages = ["."] - revision = "42246108ff925a457fb709475070a03dfd3e2b5c" - version = "0.9.6" - -[[projects]] - name = "github.com/tendermint/go-crypto" - packages = ["."] - revision = "5d5f580f49ca66c13400938c64334186068c8b7c" - version = "v0.6.1" - -[[projects]] - name = "github.com/tendermint/go-wire" - packages = ["."] - revision = "fa721242b042ecd4c6ed1a934ee740db4f74e45c" - version = "v0.7.3" - -[[projects]] - name = "github.com/tendermint/tmlibs" - packages = [ - "autofile", - "cli", - "cli/flags", - "clist", - "common", - "db", - "flowrate", - "log", - "merkle", - "pubsub", - "pubsub/query", - "test" - ] - revision = "2e24b64fc121dcdf1cabceab8dc2f7257675483c" - version = "0.8.1" - -[[projects]] - branch = "master" - name = "golang.org/x/crypto" - packages = [ - "curve25519", - "nacl/box", - "nacl/secretbox", - "openpgp/armor", - "openpgp/errors", - "poly1305", - "ripemd160", - "salsa20/salsa" - ] -<<<<<<< HEAD - revision = "88942b9c40a4c9d203b82b3731787b672d6e809b" -======= - revision = "b2aa35443fbc700ab74c586ae79b81c171851023" ->>>>>>> develop - -[[projects]] - branch = "master" - name = "golang.org/x/net" - packages = [ - "context", - "http2", - "http2/hpack", - "idna", - "internal/timeseries", - "lex/httplex", - "trace" - ] -<<<<<<< HEAD - revision = "6078986fec03a1dcc236c34816c71b0e05018fda" -======= - revision = "b3c676e531a6dc479fa1b35ac961c13f5e2b4d2e" ->>>>>>> develop - -[[projects]] - branch = "master" - name = "golang.org/x/sys" - packages = ["unix"] -<<<<<<< HEAD - revision = "91ee8cde435411ca3f1cd365e8f20131aed4d0a1" -======= - revision = "1d206c9fa8975fb4cf00df1dc8bf3283dc24ba0e" ->>>>>>> develop - -[[projects]] - name = "golang.org/x/text" - packages = [ - "collate", - "collate/build", - "internal/colltab", - "internal/gen", - "internal/tag", - "internal/triegen", - "internal/ucd", - "language", - "secure/bidirule", - "transform", - "unicode/bidi", - "unicode/cldr", - "unicode/norm", - "unicode/rangetable" - ] - revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" - version = "v0.3.0" - -[[projects]] - branch = "master" - name = "google.golang.org/genproto" - packages = ["googleapis/rpc/status"] -<<<<<<< HEAD - revision = "ab0870e398d5dd054b868c0db1481ab029b9a9f2" -======= - revision = "35de2414665fc36f56b72d982c5af480d86de5ab" ->>>>>>> develop - -[[projects]] - name = "google.golang.org/grpc" - packages = [ - ".", - "balancer", - "codes", - "connectivity", - "credentials", - "grpclb/grpc_lb_v1/messages", - "grpclog", - "internal", - "keepalive", - "metadata", - "naming", - "peer", - "resolver", - "stats", - "status", - "tap", - "transport" - ] - revision = "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" - version = "v1.7.5" - -[[projects]] - name = "gopkg.in/yaml.v2" - packages = ["."] -<<<<<<< HEAD - revision = "7f97868eec74b32b0982dd158a51a446d1da7eb5" - version = "v2.1.1" -======= - revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183" - version = "v2.2.1" ->>>>>>> develop - -[solve-meta] - analyzer-name = "dep" - analyzer-version = 1 -<<<<<<< HEAD - inputs-digest = "ed1149ed5293b7b28b7505627648c6a1152aaff0ed06a3849995b29751ae00f3" -======= - inputs-digest = "b7c02a311569ec5fe2197614444fb231ea60f3e65a11a20e318421f1752054d7" ->>>>>>> develop - solver-name = "gps-cdcl" - solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index b2dcf3b33..361cc1735 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -71,7 +71,6 @@ [[constraint]] name = "github.com/tendermint/abci" -<<<<<<< HEAD version = "0.10.3-dev" [[constraint]] @@ -81,31 +80,16 @@ [[constraint]] name = "github.com/tendermint/go-amino" version = "0.9.6" -======= - version = "~0.10.2" - -[[constraint]] - name = "github.com/tendermint/go-crypto" - version = "~0.5.0" - -[[constraint]] - name = "github.com/tendermint/go-wire" - source = "github.com/tendermint/go-amino" - version = "~0.7.3" ->>>>>>> develop [[override]] # [[constraint]] name = "github.com/tendermint/tmlibs" -<<<<<<< HEAD - version = "0.8.1" -======= version = "~0.8.1" # branch = "develop" ->>>>>>> develop [[constraint]] name = "google.golang.org/grpc" + source = "github.com/grpc/grpc-go" version = "~1.7.3" [prune] diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index f205ff92a..3bcf65569 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -11,7 +11,6 @@ import ( "time" "github.com/tendermint/abci/example/kvstore" - wire "github.com/tendermint/tendermint/wire" cmn "github.com/tendermint/tmlibs/common" "github.com/tendermint/tmlibs/log" @@ -149,7 +148,7 @@ func TestReactorRecordsBlockParts(t *testing.T) { Round: 0, Part: parts.GetPart(0), } - bz, err := wire.MarshalBinary(struct{ ConsensusMessage }{msg}) + bz, err := cdc.MarshalBinary(struct{ ConsensusMessage }{msg}) require.NoError(t, err) reactor.Receive(DataChannel, peer, bz) @@ -158,7 +157,7 @@ func TestReactorRecordsBlockParts(t *testing.T) { // 2) block part with the same height, but different round msg.Round = 1 - bz, err = wire.MarshalBinary(struct{ ConsensusMessage }{msg}) + bz, err = cdc.MarshalBinary(struct{ ConsensusMessage }{msg}) require.NoError(t, err) reactor.Receive(DataChannel, peer, bz) @@ -168,7 +167,7 @@ func TestReactorRecordsBlockParts(t *testing.T) { msg.Height = 1 msg.Round = 0 - bz, err = wire.MarshalBinary(struct{ ConsensusMessage }{msg}) + bz, err = cdc.MarshalBinary(struct{ ConsensusMessage }{msg}) require.NoError(t, err) reactor.Receive(DataChannel, peer, bz) @@ -204,7 +203,7 @@ func TestReactorRecordsVotes(t *testing.T) { Type: types.VoteTypePrevote, BlockID: types.BlockID{}, } - bz, err := wire.MarshalBinary(struct{ ConsensusMessage }{&VoteMessage{vote}}) + bz, err := cdc.MarshalBinary(struct{ ConsensusMessage }{&VoteMessage{vote}}) require.NoError(t, err) reactor.Receive(VoteChannel, peer, bz) @@ -213,7 +212,7 @@ func TestReactorRecordsVotes(t *testing.T) { // 2) vote with the same height, but different round vote.Round = 1 - bz, err = wire.MarshalBinary(struct{ ConsensusMessage }{&VoteMessage{vote}}) + bz, err = cdc.MarshalBinary(struct{ ConsensusMessage }{&VoteMessage{vote}}) require.NoError(t, err) reactor.Receive(VoteChannel, peer, bz) @@ -223,7 +222,7 @@ func TestReactorRecordsVotes(t *testing.T) { vote.Height = 1 vote.Round = 0 - bz, err = wire.MarshalBinary(struct{ ConsensusMessage }{&VoteMessage{vote}}) + bz, err = cdc.MarshalBinary(struct{ ConsensusMessage }{&VoteMessage{vote}}) require.NoError(t, err) reactor.Receive(VoteChannel, peer, bz) diff --git a/p2p/dummy/peer.go b/p2p/dummy/peer.go index 61c3a8ace..97fb7e2ef 100644 --- a/p2p/dummy/peer.go +++ b/p2p/dummy/peer.go @@ -48,12 +48,12 @@ func (p *peer) Status() tmconn.ConnectionStatus { } // Send does not do anything and just returns true. -func (p *peer) Send(byte, interface{}) bool { +func (p *peer) Send(byte, []byte) bool { return true } // TrySend does not do anything and just returns true. -func (p *peer) TrySend(byte, interface{}) bool { +func (p *peer) TrySend(byte, []byte) bool { return true } diff --git a/types/priv_validator/sign_info.go b/types/priv_validator/sign_info.go deleted file mode 100644 index 8b135df63..000000000 --- a/types/priv_validator/sign_info.go +++ /dev/null @@ -1,238 +0,0 @@ -package types - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "time" - - crypto "github.com/tendermint/go-crypto" - "github.com/tendermint/tendermint/types" - cmn "github.com/tendermint/tmlibs/common" -) - -// TODO: type ? -const ( - stepNone int8 = 0 // Used to distinguish the initial state - stepPropose int8 = 1 - stepPrevote int8 = 2 - stepPrecommit int8 = 3 -) - -func voteToStep(vote *types.Vote) int8 { - switch vote.Type { - case types.VoteTypePrevote: - return stepPrevote - case types.VoteTypePrecommit: - return stepPrecommit - default: - panic("Unknown vote type") - } -} - -//------------------------------------- - -// LastSignedInfo contains information about the latest -// data signed by a validator to help prevent double signing. -type LastSignedInfo struct { - Height int64 `json:"height"` - Round int `json:"round"` - Step int8 `json:"step"` - Signature crypto.Signature `json:"signature,omitempty"` // so we dont lose signatures - SignBytes cmn.HexBytes `json:"signbytes,omitempty"` // so we dont lose signatures -} - -func NewLastSignedInfo() *LastSignedInfo { - return &LastSignedInfo{ - Step: stepNone, - } -} - -func (lsi *LastSignedInfo) String() string { - return fmt.Sprintf("LH:%v, LR:%v, LS:%v", lsi.Height, lsi.Round, lsi.Step) -} - -// Verify returns an error if there is a height/round/step regression -// or if the HRS matches but there are no LastSignBytes. -// It returns true if HRS matches exactly and the LastSignature exists. -// It panics if the HRS matches, the LastSignBytes are not empty, but the LastSignature is empty. -func (lsi LastSignedInfo) Verify(height int64, round int, step int8) (bool, error) { - if lsi.Height > height { - return false, errors.New("Height regression") - } - - if lsi.Height == height { - if lsi.Round > round { - return false, errors.New("Round regression") - } - - if lsi.Round == round { - if lsi.Step > step { - return false, errors.New("Step regression") - } else if lsi.Step == step { - if lsi.SignBytes != nil { - if lsi.Signature.Empty() { - panic("info: LastSignature is nil but LastSignBytes is not!") - } - return true, nil - } - return false, errors.New("No LastSignature found") - } - } - } - return false, nil -} - -// Set height/round/step and signature on the info -func (lsi *LastSignedInfo) Set(height int64, round int, step int8, - signBytes []byte, sig crypto.Signature) { - - lsi.Height = height - lsi.Round = round - lsi.Step = step - lsi.Signature = sig - lsi.SignBytes = signBytes -} - -// Reset resets all the values. -// XXX: Unsafe. -func (lsi *LastSignedInfo) Reset() { - lsi.Height = 0 - lsi.Round = 0 - lsi.Step = 0 - lsi.Signature = crypto.Signature{} - lsi.SignBytes = nil -} - -// SignVote checks the height/round/step (HRS) are greater than the latest state of the LastSignedInfo. -// If so, it signs the vote, updates the LastSignedInfo, and sets the signature on the vote. -// If the HRS are equal and the only thing changed is the timestamp, it sets the vote.Timestamp to the previous -// value and the Signature to the LastSignedInfo.Signature. -// Else it returns an error. -func (lsi *LastSignedInfo) SignVote(signer types.Signer, chainID string, vote *types.Vote) error { - height, round, step := vote.Height, vote.Round, voteToStep(vote) - signBytes := vote.SignBytes(chainID) - - sameHRS, err := lsi.Verify(height, round, step) - if err != nil { - return err - } - - // We might crash before writing to the wal, - // causing us to try to re-sign for the same HRS. - // If signbytes are the same, use the last signature. - // If they only differ by timestamp, use last timestamp and signature - // Otherwise, return error - if sameHRS { - if bytes.Equal(signBytes, lsi.SignBytes) { - vote.Signature = lsi.Signature - } else if timestamp, ok := checkVotesOnlyDifferByTimestamp(lsi.SignBytes, signBytes); ok { - vote.Timestamp = timestamp - vote.Signature = lsi.Signature - } else { - err = fmt.Errorf("Conflicting data") - } - return err - } - sig, err := signer.Sign(signBytes) - if err != nil { - return err - } - lsi.Set(height, round, step, signBytes, sig) - vote.Signature = sig - return nil -} - -// SignProposal checks if the height/round/step (HRS) are greater than the latest state of the LastSignedInfo. -// If so, it signs the proposal, updates the LastSignedInfo, and sets the signature on the proposal. -// If the HRS are equal and the only thing changed is the timestamp, it sets the timestamp to the previous -// value and the Signature to the LastSignedInfo.Signature. -// Else it returns an error. -func (lsi *LastSignedInfo) SignProposal(signer types.Signer, chainID string, proposal *types.Proposal) error { - height, round, step := proposal.Height, proposal.Round, stepPropose - signBytes := proposal.SignBytes(chainID) - - sameHRS, err := lsi.Verify(height, round, step) - if err != nil { - return err - } - - // We might crash before writing to the wal, - // causing us to try to re-sign for the same HRS. - // If signbytes are the same, use the last signature. - // If they only differ by timestamp, use last timestamp and signature - // Otherwise, return error - if sameHRS { - if bytes.Equal(signBytes, lsi.SignBytes) { - proposal.Signature = lsi.Signature - } else if timestamp, ok := checkProposalsOnlyDifferByTimestamp(lsi.SignBytes, signBytes); ok { - proposal.Timestamp = timestamp - proposal.Signature = lsi.Signature - } else { - err = fmt.Errorf("Conflicting data") - } - return err - } - sig, err := signer.Sign(signBytes) - if err != nil { - return err - } - lsi.Set(height, round, step, signBytes, sig) - proposal.Signature = sig - return nil -} - -//------------------------------------- - -// returns the timestamp from the lastSignBytes. -// returns true if the only difference in the votes is their timestamp. -func checkVotesOnlyDifferByTimestamp(lastSignBytes, newSignBytes []byte) (time.Time, bool) { - var lastVote, newVote types.CanonicalJSONOnceVote - if err := json.Unmarshal(lastSignBytes, &lastVote); err != nil { - panic(fmt.Sprintf("LastSignBytes cannot be unmarshalled into vote: %v", err)) - } - if err := json.Unmarshal(newSignBytes, &newVote); err != nil { - panic(fmt.Sprintf("signBytes cannot be unmarshalled into vote: %v", err)) - } - - lastTime, err := time.Parse(types.TimeFormat, lastVote.Vote.Timestamp) - if err != nil { - panic(err) - } - - // set the times to the same value and check equality - now := types.CanonicalTime(time.Now()) - lastVote.Vote.Timestamp = now - newVote.Vote.Timestamp = now - lastVoteBytes, _ := json.Marshal(lastVote) - newVoteBytes, _ := json.Marshal(newVote) - - return lastTime, bytes.Equal(newVoteBytes, lastVoteBytes) -} - -// returns the timestamp from the lastSignBytes. -// returns true if the only difference in the proposals is their timestamp -func checkProposalsOnlyDifferByTimestamp(lastSignBytes, newSignBytes []byte) (time.Time, bool) { - var lastProposal, newProposal types.CanonicalJSONOnceProposal - if err := json.Unmarshal(lastSignBytes, &lastProposal); err != nil { - panic(fmt.Sprintf("LastSignBytes cannot be unmarshalled into proposal: %v", err)) - } - if err := json.Unmarshal(newSignBytes, &newProposal); err != nil { - panic(fmt.Sprintf("signBytes cannot be unmarshalled into proposal: %v", err)) - } - - lastTime, err := time.Parse(types.TimeFormat, lastProposal.Proposal.Timestamp) - if err != nil { - panic(err) - } - - // set the times to the same value and check equality - now := types.CanonicalTime(time.Now()) - lastProposal.Proposal.Timestamp = now - newProposal.Proposal.Timestamp = now - lastProposalBytes, _ := json.Marshal(lastProposal) - newProposalBytes, _ := json.Marshal(newProposal) - - return lastTime, bytes.Equal(newProposalBytes, lastProposalBytes) -} From c68d4061952078974ea539b8f82b6e342ae99775 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sat, 7 Apr 2018 19:47:19 +0300 Subject: [PATCH 076/143] fix tests --- blockchain/store.go | 7 +++---- blockchain/store_test.go | 2 +- cmd/tendermint/commands/probe_upnp.go | 3 +-- consensus/reactor_test.go | 18 +++++++++--------- node/node.go | 5 ++--- p2p/pex/pex_reactor_test.go | 2 +- state/txindex/kv/kv_test.go | 26 ++++++++++++++++++++++++-- types/event_bus_test.go | 2 +- 8 files changed, 42 insertions(+), 23 deletions(-) diff --git a/blockchain/store.go b/blockchain/store.go index a974009e2..e7608b2cc 100644 --- a/blockchain/store.go +++ b/blockchain/store.go @@ -1,7 +1,6 @@ package blockchain import ( - "encoding/json" "fmt" "sync" @@ -218,12 +217,12 @@ func calcSeenCommitKey(height int64) []byte { var blockStoreKey = []byte("blockStore") type BlockStoreStateJSON struct { - Height int64 + Height int64 `json:"height"` } // Save persists the blockStore state to the database as JSON. func (bsj BlockStoreStateJSON) Save(db dbm.DB) { - bytes, err := json.Marshal(bsj) + bytes, err := cdc.MarshalJSON(bsj) if err != nil { cmn.PanicSanity(cmn.Fmt("Could not marshal state bytes: %v", err)) } @@ -240,7 +239,7 @@ func LoadBlockStoreStateJSON(db dbm.DB) BlockStoreStateJSON { } } bsj := BlockStoreStateJSON{} - err := json.Unmarshal(bytes, &bsj) + err := cdc.UnmarshalJSON(bytes, &bsj) if err != nil { panic(fmt.Sprintf("Could not unmarshal bytes: %X", bytes)) } diff --git a/blockchain/store_test.go b/blockchain/store_test.go index 8e690e6d2..a0d53e0cf 100644 --- a/blockchain/store_test.go +++ b/blockchain/store_test.go @@ -31,7 +31,7 @@ func TestNewBlockStore(t *testing.T) { db := db.NewMemDB() db.Set(blockStoreKey, []byte(`{"height": 10000}`)) bs := NewBlockStore(db) - assert.Equal(t, bs.Height(), int64(10000), "failed to properly parse blockstore") + require.Equal(t, int64(10000), bs.Height(), "failed to properly parse blockstore") panicCausers := []struct { data []byte diff --git a/cmd/tendermint/commands/probe_upnp.go b/cmd/tendermint/commands/probe_upnp.go index a37bc3602..35c3c354d 100644 --- a/cmd/tendermint/commands/probe_upnp.go +++ b/cmd/tendermint/commands/probe_upnp.go @@ -1,7 +1,6 @@ package commands import ( - "encoding/json" "fmt" "github.com/spf13/cobra" @@ -22,7 +21,7 @@ func probeUpnp(cmd *cobra.Command, args []string) error { fmt.Println("Probe failed: ", err) } else { fmt.Println("Probe success!") - jsonBytes, err := json.Marshal(capabilities) + jsonBytes, err := cdc.MarshalJSON(capabilities) if err != nil { return err } diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index 3bcf65569..058bf1e71 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -148,30 +148,30 @@ func TestReactorRecordsBlockParts(t *testing.T) { Round: 0, Part: parts.GetPart(0), } - bz, err := cdc.MarshalBinary(struct{ ConsensusMessage }{msg}) + bz, err := cdc.MarshalBinaryBare(msg) require.NoError(t, err) reactor.Receive(DataChannel, peer, bz) - assert.Equal(t, 1, ps.BlockPartsSent(), "number of block parts sent should have increased by 1") + require.Equal(t, 1, ps.BlockPartsSent(), "number of block parts sent should have increased by 1") // 2) block part with the same height, but different round msg.Round = 1 - bz, err = cdc.MarshalBinary(struct{ ConsensusMessage }{msg}) + bz, err = cdc.MarshalBinaryBare(msg) require.NoError(t, err) reactor.Receive(DataChannel, peer, bz) - assert.Equal(t, 1, ps.BlockPartsSent(), "number of block parts sent should stay the same") + require.Equal(t, 1, ps.BlockPartsSent(), "number of block parts sent should stay the same") // 3) block part from earlier height msg.Height = 1 msg.Round = 0 - bz, err = cdc.MarshalBinary(struct{ ConsensusMessage }{msg}) + bz, err = cdc.MarshalBinaryBare(msg) require.NoError(t, err) reactor.Receive(DataChannel, peer, bz) - assert.Equal(t, 1, ps.BlockPartsSent(), "number of block parts sent should stay the same") + require.Equal(t, 1, ps.BlockPartsSent(), "number of block parts sent should stay the same") } // Test we record votes from other peers @@ -203,7 +203,7 @@ func TestReactorRecordsVotes(t *testing.T) { Type: types.VoteTypePrevote, BlockID: types.BlockID{}, } - bz, err := cdc.MarshalBinary(struct{ ConsensusMessage }{&VoteMessage{vote}}) + bz, err := cdc.MarshalBinaryBare(&VoteMessage{vote}) require.NoError(t, err) reactor.Receive(VoteChannel, peer, bz) @@ -212,7 +212,7 @@ func TestReactorRecordsVotes(t *testing.T) { // 2) vote with the same height, but different round vote.Round = 1 - bz, err = cdc.MarshalBinary(struct{ ConsensusMessage }{&VoteMessage{vote}}) + bz, err = cdc.MarshalBinaryBare(&VoteMessage{vote}) require.NoError(t, err) reactor.Receive(VoteChannel, peer, bz) @@ -222,7 +222,7 @@ func TestReactorRecordsVotes(t *testing.T) { vote.Height = 1 vote.Round = 0 - bz, err = cdc.MarshalBinary(struct{ ConsensusMessage }{&VoteMessage{vote}}) + bz, err = cdc.MarshalBinaryBare(&VoteMessage{vote}) require.NoError(t, err) reactor.Receive(VoteChannel, peer, bz) diff --git a/node/node.go b/node/node.go index 2bcee4b3c..83d27cb48 100644 --- a/node/node.go +++ b/node/node.go @@ -2,7 +2,6 @@ package node import ( "bytes" - "encoding/json" "errors" "fmt" "net" @@ -644,7 +643,7 @@ func loadGenesisDoc(db dbm.DB) (*types.GenesisDoc, error) { return nil, errors.New("Genesis doc not found") } var genDoc *types.GenesisDoc - err := json.Unmarshal(bytes, &genDoc) + err := cdc.UnmarshalJSON(bytes, &genDoc) if err != nil { cmn.PanicCrisis(fmt.Sprintf("Failed to load genesis doc due to unmarshaling error: %v (bytes: %X)", err, bytes)) } @@ -653,7 +652,7 @@ func loadGenesisDoc(db dbm.DB) (*types.GenesisDoc, error) { // panics if failed to marshal the given genesis document func saveGenesisDoc(db dbm.DB, genDoc *types.GenesisDoc) { - bytes, err := json.Marshal(genDoc) + bytes, err := cdc.MarshalJSON(genDoc) if err != nil { cmn.PanicCrisis(fmt.Sprintf("Failed to save genesis doc due to marshaling error: %v", err)) } diff --git a/p2p/pex/pex_reactor_test.go b/p2p/pex/pex_reactor_test.go index 16136dc08..b3f29c5bd 100644 --- a/p2p/pex/pex_reactor_test.go +++ b/p2p/pex/pex_reactor_test.go @@ -297,7 +297,7 @@ func TestPEXReactorDoesNotAddPrivatePeersToAddrBook(t *testing.T) { size := book.Size() addrs := []*p2p.NetAddress{peer.NodeInfo().NetAddress()} - msg := wire.BinaryBytes(struct{ PexMessage }{&pexAddrsMessage{Addrs: addrs}}) + msg := cdc.MustMarshalBinary(&pexAddrsMessage{Addrs: addrs}) pexR.Receive(PexChannel, peer, msg) assert.Equal(t, size, book.Size()) diff --git a/state/txindex/kv/kv_test.go b/state/txindex/kv/kv_test.go index f81f8ef3a..74a2dd7cb 100644 --- a/state/txindex/kv/kv_test.go +++ b/state/txindex/kv/kv_test.go @@ -176,12 +176,34 @@ func TestIndexAllTags(t *testing.T) { func txResultWithTags(tags []cmn.KVPair) *types.TxResult { tx := types.Tx("HELLO WORLD") - return &types.TxResult{Height: 1, Index: 0, Tx: tx, Result: abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeTypeOK, Log: "", Tags: tags, Fee: cmn.KI64Pair{Key: []uint8{}, Value: 0}}} + return &types.TxResult{ + Height: 1, + Index: 0, + Tx: tx, + Result: abci.ResponseDeliverTx{ + Data: []byte{0}, + Code: abci.CodeTypeOK, + Log: "", + Tags: tags, + Fee: cmn.KI64Pair{Key: nil, Value: 0}, + }, + } } func benchmarkTxIndex(txsCount int, b *testing.B) { tx := types.Tx("HELLO WORLD") - txResult := &types.TxResult{1, 0, tx, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeTypeOK, Log: "", Tags: nil}} + txResult := &types.TxResult{ + Height: 1, + Index: 0, + Tx: tx, + Result: abci.ResponseDeliverTx{ + Data: []byte{0}, + Code: abci.CodeTypeOK, + Log: "", + Tags: []cmn.KVPair{}, + Fee: cmn.KI64Pair{Key: []uint8{}, Value: 0}, + }, + } dir, err := ioutil.TempDir("", "tx_index_db") if err != nil { diff --git a/types/event_bus_test.go b/types/event_bus_test.go index fb2e65747..70a537745 100644 --- a/types/event_bus_test.go +++ b/types/event_bus_test.go @@ -35,7 +35,7 @@ func TestEventBusPublishEventTx(t *testing.T) { done := make(chan struct{}) go func() { for e := range txEventsCh { - edt := e.(TMEventData).Unwrap().(EventDataTx) + edt := e.(EventDataTx) assert.Equal(t, int64(1), edt.Height) assert.Equal(t, uint32(0), edt.Index) assert.Equal(t, tx, edt.Tx) From 466c3ab1c79c1e9856aeea80c9e11c31219d11f0 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sat, 7 Apr 2018 19:53:29 +0300 Subject: [PATCH 077/143] forgot node/wire.go --- node/wire.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 node/wire.go diff --git a/node/wire.go b/node/wire.go new file mode 100644 index 000000000..a0d7677df --- /dev/null +++ b/node/wire.go @@ -0,0 +1,12 @@ +package node + +import ( + amino "github.com/tendermint/go-amino" + crypto "github.com/tendermint/go-crypto" +) + +var cdc = amino.NewCodec() + +func init() { + crypto.RegisterAmino(cdc) +} From df9bf60b05a8e15e46fc831bdf26dd5f15d9d65e Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sat, 7 Apr 2018 20:59:13 +0300 Subject: [PATCH 078/143] forgot Gopkg.lock --- Gopkg.lock | 389 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 389 insertions(+) create mode 100644 Gopkg.lock diff --git a/Gopkg.lock b/Gopkg.lock new file mode 100644 index 000000000..81f3c7033 --- /dev/null +++ b/Gopkg.lock @@ -0,0 +1,389 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + branch = "master" + name = "github.com/btcsuite/btcd" + packages = ["btcec"] + revision = "2be2f12b358dc57d70b8f501b00be450192efbc3" + +[[projects]] + name = "github.com/davecgh/go-spew" + packages = ["spew"] + revision = "346938d642f2ec3594ed81d874461961cd0faa76" + version = "v1.1.0" + +[[projects]] + branch = "master" + name = "github.com/ebuchman/fail-test" + packages = ["."] + revision = "95f809107225be108efcf10a3509e4ea6ceef3c4" + +[[projects]] + name = "github.com/fortytw2/leaktest" + packages = ["."] + revision = "a5ef70473c97b71626b9abeda80ee92ba2a7de9e" + version = "v1.2.0" + +[[projects]] + name = "github.com/fsnotify/fsnotify" + packages = ["."] + revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9" + version = "v1.4.7" + +[[projects]] + name = "github.com/go-kit/kit" + packages = [ + "log", + "log/level", + "log/term" + ] + revision = "4dc7be5d2d12881735283bcab7352178e190fc71" + version = "v0.6.0" + +[[projects]] + name = "github.com/go-logfmt/logfmt" + packages = ["."] + revision = "390ab7935ee28ec6b286364bba9b4dd6410cb3d5" + version = "v0.3.0" + +[[projects]] + name = "github.com/go-stack/stack" + packages = ["."] + revision = "259ab82a6cad3992b4e21ff5cac294ccb06474bc" + version = "v1.7.0" + +[[projects]] + name = "github.com/gogo/protobuf" + packages = [ + "gogoproto", + "jsonpb", + "proto", + "protoc-gen-gogo/descriptor", + "sortkeys", + "types" + ] + revision = "1adfc126b41513cc696b209667c8656ea7aac67c" + version = "v1.0.0" + +[[projects]] + name = "github.com/golang/protobuf" + packages = [ + "proto", + "ptypes", + "ptypes/any", + "ptypes/duration", + "ptypes/timestamp" + ] + revision = "925541529c1fa6821df4e44ce2723319eb2be768" + version = "v1.0.0" + +[[projects]] + branch = "master" + name = "github.com/golang/snappy" + packages = ["."] + revision = "553a641470496b2327abcac10b36396bd98e45c9" + +[[projects]] + name = "github.com/gorilla/websocket" + packages = ["."] + revision = "ea4d1f681babbce9545c9c5f3d5194a789c89f5b" + version = "v1.2.0" + +[[projects]] + branch = "master" + name = "github.com/hashicorp/hcl" + packages = [ + ".", + "hcl/ast", + "hcl/parser", + "hcl/printer", + "hcl/scanner", + "hcl/strconv", + "hcl/token", + "json/parser", + "json/scanner", + "json/token" + ] + revision = "ef8a98b0bbce4a65b5aa4c368430a80ddc533168" + +[[projects]] + name = "github.com/inconshreveable/mousetrap" + packages = ["."] + revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75" + version = "v1.0" + +[[projects]] + branch = "master" + name = "github.com/jmhodges/levigo" + packages = ["."] + revision = "c42d9e0ca023e2198120196f842701bb4c55d7b9" + +[[projects]] + branch = "master" + name = "github.com/kr/logfmt" + packages = ["."] + revision = "b84e30acd515aadc4b783ad4ff83aff3299bdfe0" + +[[projects]] + name = "github.com/magiconair/properties" + packages = ["."] + revision = "c3beff4c2358b44d0493c7dda585e7db7ff28ae6" + version = "v1.7.6" + +[[projects]] + branch = "master" + name = "github.com/mitchellh/mapstructure" + packages = ["."] + revision = "00c29f56e2386353d58c599509e8dc3801b0d716" + +[[projects]] + name = "github.com/pelletier/go-toml" + packages = ["."] + revision = "acdc4509485b587f5e675510c4f2c63e90ff68a8" + version = "v1.1.0" + +[[projects]] + name = "github.com/pkg/errors" + packages = ["."] + revision = "645ef00459ed84a119197bfb8d8205042c6df63d" + version = "v0.8.0" + +[[projects]] + name = "github.com/pmezard/go-difflib" + packages = ["difflib"] + revision = "792786c7400a136282c1664665ae0a8db921c6c2" + version = "v1.0.0" + +[[projects]] + branch = "master" + name = "github.com/rcrowley/go-metrics" + packages = ["."] + revision = "d932a24a8ccb8fcadc993e5c6c58f93dac168294" + +[[projects]] + name = "github.com/spf13/afero" + packages = [ + ".", + "mem" + ] + revision = "63644898a8da0bc22138abf860edaf5277b6102e" + version = "v1.1.0" + +[[projects]] + name = "github.com/spf13/cast" + packages = ["."] + revision = "8965335b8c7107321228e3e3702cab9832751bac" + version = "v1.2.0" + +[[projects]] + name = "github.com/spf13/cobra" + packages = ["."] + revision = "a1f051bc3eba734da4772d60e2d677f47cf93ef4" + version = "v0.0.2" + +[[projects]] + branch = "master" + name = "github.com/spf13/jwalterweatherman" + packages = ["."] + revision = "7c0cea34c8ece3fbeb2b27ab9b59511d360fb394" + +[[projects]] + name = "github.com/spf13/pflag" + packages = ["."] + revision = "e57e3eeb33f795204c1ca35f56c44f83227c6e66" + version = "v1.0.0" + +[[projects]] + name = "github.com/spf13/viper" + packages = ["."] + revision = "b5e8006cbee93ec955a89ab31e0e3ce3204f3736" + version = "v1.0.2" + +[[projects]] + name = "github.com/stretchr/testify" + packages = [ + "assert", + "require" + ] + revision = "12b6f73e6084dad08a7c6e575284b177ecafbc71" + version = "v1.2.1" + +[[projects]] + branch = "master" + name = "github.com/syndtr/goleveldb" + packages = [ + "leveldb", + "leveldb/cache", + "leveldb/comparer", + "leveldb/errors", + "leveldb/filter", + "leveldb/iterator", + "leveldb/journal", + "leveldb/memdb", + "leveldb/opt", + "leveldb/storage", + "leveldb/table", + "leveldb/util" + ] + revision = "714f901b98fdb3aa954b4193d8cbd64a28d80cad" + +[[projects]] + name = "github.com/tendermint/abci" + packages = [ + "client", + "example/code", + "example/counter", + "example/kvstore", + "server", + "types" + ] + revision = "c62aed95f2ce399ec815b0cafe478af002cdc4e6" + version = "v0.10.3-dev" + +[[projects]] + branch = "master" + name = "github.com/tendermint/ed25519" + packages = [ + ".", + "edwards25519", + "extra25519" + ] + revision = "d8387025d2b9d158cf4efb07e7ebf814bcce2057" + +[[projects]] + name = "github.com/tendermint/go-amino" + packages = ["."] + revision = "42246108ff925a457fb709475070a03dfd3e2b5c" + version = "0.9.6" + +[[projects]] + name = "github.com/tendermint/go-crypto" + packages = ["."] + revision = "5d5f580f49ca66c13400938c64334186068c8b7c" + version = "v0.6.1" + +[[projects]] + name = "github.com/tendermint/go-wire" + packages = ["."] + revision = "fa721242b042ecd4c6ed1a934ee740db4f74e45c" + version = "v0.7.3" + +[[projects]] + name = "github.com/tendermint/tmlibs" + packages = [ + "autofile", + "cli", + "cli/flags", + "clist", + "common", + "db", + "flowrate", + "log", + "merkle", + "pubsub", + "pubsub/query", + "test" + ] + revision = "2e24b64fc121dcdf1cabceab8dc2f7257675483c" + version = "0.8.1" + +[[projects]] + branch = "master" + name = "golang.org/x/crypto" + packages = [ + "curve25519", + "nacl/box", + "nacl/secretbox", + "openpgp/armor", + "openpgp/errors", + "poly1305", + "ripemd160", + "salsa20/salsa" + ] + revision = "b2aa35443fbc700ab74c586ae79b81c171851023" + +[[projects]] + branch = "master" + name = "golang.org/x/net" + packages = [ + "context", + "http2", + "http2/hpack", + "idna", + "internal/timeseries", + "lex/httplex", + "trace" + ] + revision = "61147c48b25b599e5b561d2e9c4f3e1ef489ca41" + +[[projects]] + branch = "master" + name = "golang.org/x/sys" + packages = ["unix"] + revision = "3b87a42e500a6dc65dae1a55d0b641295971163e" + +[[projects]] + name = "golang.org/x/text" + packages = [ + "collate", + "collate/build", + "internal/colltab", + "internal/gen", + "internal/tag", + "internal/triegen", + "internal/ucd", + "language", + "secure/bidirule", + "transform", + "unicode/bidi", + "unicode/cldr", + "unicode/norm", + "unicode/rangetable" + ] + revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" + version = "v0.3.0" + +[[projects]] + branch = "master" + name = "google.golang.org/genproto" + packages = ["googleapis/rpc/status"] + revision = "ce84044298496ef4b54b4a0a0909ba593cc60e30" + +[[projects]] + name = "google.golang.org/grpc" + packages = [ + ".", + "balancer", + "codes", + "connectivity", + "credentials", + "grpclb/grpc_lb_v1/messages", + "grpclog", + "internal", + "keepalive", + "metadata", + "naming", + "peer", + "resolver", + "stats", + "status", + "tap", + "transport" + ] + revision = "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + source = "github.com/grpc/grpc-go" + version = "v1.7.5" + +[[projects]] + name = "gopkg.in/yaml.v2" + packages = ["."] + revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183" + version = "v2.2.1" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + inputs-digest = "e5e6c7942710846bdb5589fba10c90a869773f2685da825b955510afe0d7c5a4" + solver-name = "gps-cdcl" + solver-version = 1 From 767521ac5214a49410e72e3b813c08303fa69dac Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sat, 7 Apr 2018 22:03:48 +0300 Subject: [PATCH 079/143] update test/p2p/data for amino --- test/p2p/data/app/init.sh | 9 --- test/p2p/data/chain_config.json | 53 ------------- test/p2p/data/mach1/core/config/genesis.json | 74 +++++++++---------- test/p2p/data/mach1/core/config/node_key.json | 7 +- .../mach1/core/config/priv_validator.json | 20 ++--- test/p2p/data/mach2/core/config/genesis.json | 74 +++++++++---------- test/p2p/data/mach2/core/config/node_key.json | 7 +- .../mach2/core/config/priv_validator.json | 20 ++--- test/p2p/data/mach3/core/config/genesis.json | 74 +++++++++---------- test/p2p/data/mach3/core/config/node_key.json | 7 +- .../mach3/core/config/priv_validator.json | 20 ++--- test/p2p/data/mach4/core/config/genesis.json | 74 +++++++++---------- test/p2p/data/mach4/core/config/node_key.json | 7 +- .../mach4/core/config/priv_validator.json | 20 ++--- 14 files changed, 212 insertions(+), 254 deletions(-) delete mode 100755 test/p2p/data/app/init.sh delete mode 100644 test/p2p/data/chain_config.json diff --git a/test/p2p/data/app/init.sh b/test/p2p/data/app/init.sh deleted file mode 100755 index 24f017630..000000000 --- a/test/p2p/data/app/init.sh +++ /dev/null @@ -1,9 +0,0 @@ -#! /bin/bash -# This is a sample bash script for a ABCI application - -cd app/ -git clone https://github.com/tendermint/nomnomcoin.git -cd nomnomcoin -npm install . - -node app.js --eyes="unix:///data/tendermint/data/data.sock" \ No newline at end of file diff --git a/test/p2p/data/chain_config.json b/test/p2p/data/chain_config.json deleted file mode 100644 index 4221ba8da..000000000 --- a/test/p2p/data/chain_config.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "id": "", - "val_set_id": "anon", - "validators": [ - { - "validator": { - "id": "mach1", - "pub_key": { - "type": "ed25519", - "data": "BE8933DFF1600C026E34718F1785A4CDEAB90C35698B394E38B6947AE91DE116" - } - }, - "p2p_addr": "", - "rpc_addr": "" - }, - { - "validator": { - "id": "mach2", - "pub_key": { - "type": "ed25519", - "data": "6DC534465323126587D2A2A93B59D689B717073B1DE968A25A6EF13D595318AD" - } - }, - "p2p_addr": "", - "rpc_addr": "", - "index": 1 - }, - { - "validator": { - "id": "mach3", - "pub_key": { - "type": "ed25519", - "data": "AE67AC697D135AA0B4601EA57EAAB3FEBF4BAA4F229C45A598C2985B12FCD1A1" - } - }, - "p2p_addr": "", - "rpc_addr": "", - "index": 2 - }, - { - "validator": { - "id": "mach4", - "pub_key": { - "type": "ed25519", - "data": "9EBC8F58CED4B46DCD5AB8ABA591DD253CD7CB5037273FDA32BC0B6461C4EFD9" - } - }, - "p2p_addr": "", - "rpc_addr": "", - "index": 3 - } - ] -} diff --git a/test/p2p/data/mach1/core/config/genesis.json b/test/p2p/data/mach1/core/config/genesis.json index 8d5c6c7b9..19577db00 100644 --- a/test/p2p/data/mach1/core/config/genesis.json +++ b/test/p2p/data/mach1/core/config/genesis.json @@ -1,39 +1,39 @@ { - "app_hash": "", - "chain_id": "chain-9ujDWI", - "genesis_time": "2016-06-24T20:01:19.322Z", - "validators": [ - { - "power": 1, - "name": "mach1", - "pub_key": { - "type": "ed25519", - "data": "BE8933DFF1600C026E34718F1785A4CDEAB90C35698B394E38B6947AE91DE116" - } - }, - { - "power": 1, - "name": "mach2", - "pub_key": { - "type": "ed25519", - "data": "6DC534465323126587D2A2A93B59D689B717073B1DE968A25A6EF13D595318AD" - } - }, - { - "power": 1, - "name": "mach3", - "pub_key": { - "type": "ed25519", - "data": "AE67AC697D135AA0B4601EA57EAAB3FEBF4BAA4F229C45A598C2985B12FCD1A1" - } - }, - { - "power": 1, - "name": "mach4", - "pub_key": { - "type": "ed25519", - "data": "9EBC8F58CED4B46DCD5AB8ABA591DD253CD7CB5037273FDA32BC0B6461C4EFD9" - } - } - ] + "genesis_time": "2016-06-24T20:01:19.322Z", + "chain_id": "chain-9ujDWI", + "validators": [ + { + "pub_key": { + "type": "AC26791624DE60", + "value": "vokz3/FgDAJuNHGPF4Wkzeq5DDVpizlOOLaUeukd4RY=" + }, + "power": 1, + "name": "mach1" + }, + { + "pub_key": { + "type": "AC26791624DE60", + "value": "bcU0RlMjEmWH0qKpO1nWibcXBzsd6WiiWm7xPVlTGK0=" + }, + "power": 1, + "name": "mach2" + }, + { + "pub_key": { + "type": "AC26791624DE60", + "value": "rmesaX0TWqC0YB6lfqqz/r9Lqk8inEWlmMKYWxL80aE=" + }, + "power": 1, + "name": "mach3" + }, + { + "pub_key": { + "type": "AC26791624DE60", + "value": "nryPWM7UtG3NWrirpZHdJTzXy1A3Jz/aMrwLZGHE79k=" + }, + "power": 1, + "name": "mach4" + } + ], + "app_hash": "" } diff --git a/test/p2p/data/mach1/core/config/node_key.json b/test/p2p/data/mach1/core/config/node_key.json index 7540cb5f4..c6d65008f 100644 --- a/test/p2p/data/mach1/core/config/node_key.json +++ b/test/p2p/data/mach1/core/config/node_key.json @@ -1 +1,6 @@ -{"priv_key":{"type":"ed25519","data":"06962D169F314ABB9D05AE5A04B46E48F0FBD8F1830149B47493910CBDCA7796096E5B94CD179F545AE3C281D9BF5C9E0E3D8FF719048B62F7849094CFFA8591"}} \ No newline at end of file +{ + "priv_key": { + "type": "954568A3288910", + "value": "BpYtFp8xSrudBa5aBLRuSPD72PGDAUm0dJORDL3Kd5YJbluUzRefVFrjwoHZv1yeDj2P9xkEi2L3hJCUz/qFkQ==" + } +} diff --git a/test/p2p/data/mach1/core/config/priv_validator.json b/test/p2p/data/mach1/core/config/priv_validator.json index 6538281b4..4a7d3ade9 100644 --- a/test/p2p/data/mach1/core/config/priv_validator.json +++ b/test/p2p/data/mach1/core/config/priv_validator.json @@ -1,14 +1,14 @@ { - "address": "0E6925C3EE4C599DFF1536A5071AF4A26DF33635", - "last_height": 0, - "last_round": 0, - "last_step": 0, - "priv_key": { - "type": "ed25519", - "data": "547AA07C7A8CE16C5CB2A40C6C26D15B0A32960410A9F1EA6E50B636F1AB389ABE8933DFF1600C026E34718F1785A4CDEAB90C35698B394E38B6947AE91DE116" + "address": "0E6925C3EE4C599DFF1536A5071AF4A26DF33635", + "pub_key": { + "type": "AC26791624DE60", + "value": "vokz3/FgDAJuNHGPF4Wkzeq5DDVpizlOOLaUeukd4RY=" }, - "pub_key": { - "type": "ed25519", - "data": "BE8933DFF1600C026E34718F1785A4CDEAB90C35698B394E38B6947AE91DE116" + "last_height": 0, + "last_round": 0, + "last_step": 0, + "priv_key": { + "type": "954568A3288910", + "value": "VHqgfHqM4WxcsqQMbCbRWwoylgQQqfHqblC2NvGrOJq+iTPf8WAMAm40cY8XhaTN6rkMNWmLOU44tpR66R3hFg==" } } diff --git a/test/p2p/data/mach2/core/config/genesis.json b/test/p2p/data/mach2/core/config/genesis.json index 8d5c6c7b9..19577db00 100644 --- a/test/p2p/data/mach2/core/config/genesis.json +++ b/test/p2p/data/mach2/core/config/genesis.json @@ -1,39 +1,39 @@ { - "app_hash": "", - "chain_id": "chain-9ujDWI", - "genesis_time": "2016-06-24T20:01:19.322Z", - "validators": [ - { - "power": 1, - "name": "mach1", - "pub_key": { - "type": "ed25519", - "data": "BE8933DFF1600C026E34718F1785A4CDEAB90C35698B394E38B6947AE91DE116" - } - }, - { - "power": 1, - "name": "mach2", - "pub_key": { - "type": "ed25519", - "data": "6DC534465323126587D2A2A93B59D689B717073B1DE968A25A6EF13D595318AD" - } - }, - { - "power": 1, - "name": "mach3", - "pub_key": { - "type": "ed25519", - "data": "AE67AC697D135AA0B4601EA57EAAB3FEBF4BAA4F229C45A598C2985B12FCD1A1" - } - }, - { - "power": 1, - "name": "mach4", - "pub_key": { - "type": "ed25519", - "data": "9EBC8F58CED4B46DCD5AB8ABA591DD253CD7CB5037273FDA32BC0B6461C4EFD9" - } - } - ] + "genesis_time": "2016-06-24T20:01:19.322Z", + "chain_id": "chain-9ujDWI", + "validators": [ + { + "pub_key": { + "type": "AC26791624DE60", + "value": "vokz3/FgDAJuNHGPF4Wkzeq5DDVpizlOOLaUeukd4RY=" + }, + "power": 1, + "name": "mach1" + }, + { + "pub_key": { + "type": "AC26791624DE60", + "value": "bcU0RlMjEmWH0qKpO1nWibcXBzsd6WiiWm7xPVlTGK0=" + }, + "power": 1, + "name": "mach2" + }, + { + "pub_key": { + "type": "AC26791624DE60", + "value": "rmesaX0TWqC0YB6lfqqz/r9Lqk8inEWlmMKYWxL80aE=" + }, + "power": 1, + "name": "mach3" + }, + { + "pub_key": { + "type": "AC26791624DE60", + "value": "nryPWM7UtG3NWrirpZHdJTzXy1A3Jz/aMrwLZGHE79k=" + }, + "power": 1, + "name": "mach4" + } + ], + "app_hash": "" } diff --git a/test/p2p/data/mach2/core/config/node_key.json b/test/p2p/data/mach2/core/config/node_key.json index efad2f0b3..146a1328d 100644 --- a/test/p2p/data/mach2/core/config/node_key.json +++ b/test/p2p/data/mach2/core/config/node_key.json @@ -1 +1,6 @@ -{"priv_key":{"type":"ed25519","data":"B8CE8B0D5138C10208526ABDADCE91C735FCCC4186E06E0972EC35E64973428A45EBC61F24CE1B91B3D26AFBAB11C2789EF04CBAC28183619C01116B66A9C528"}} \ No newline at end of file +{ + "priv_key": { + "type": "954568A3288910", + "value": "uM6LDVE4wQIIUmq9rc6RxzX8zEGG4G4Jcuw15klzQopF68YfJM4bkbPSavurEcJ4nvBMusKBg2GcARFrZqnFKA==" + } +} diff --git a/test/p2p/data/mach2/core/config/priv_validator.json b/test/p2p/data/mach2/core/config/priv_validator.json index 3602454f5..b44541742 100644 --- a/test/p2p/data/mach2/core/config/priv_validator.json +++ b/test/p2p/data/mach2/core/config/priv_validator.json @@ -1,14 +1,14 @@ { - "address": "99DBBD2AFC28FB5BAC5574AFAF0D9C806CED3B55", - "last_height": 0, - "last_round": 0, - "last_step": 0, - "priv_key": { - "type": "ed25519", - "data": "D047889E60502FC3129D0AB7F334B1838ED9ED1ECD99CBB96B71AD5ABF5A81436DC534465323126587D2A2A93B59D689B717073B1DE968A25A6EF13D595318AD" + "address": "99DBBD2AFC28FB5BAC5574AFAF0D9C806CED3B55", + "pub_key": { + "type": "AC26791624DE60", + "value": "bcU0RlMjEmWH0qKpO1nWibcXBzsd6WiiWm7xPVlTGK0=" }, - "pub_key": { - "type": "ed25519", - "data": "6DC534465323126587D2A2A93B59D689B717073B1DE968A25A6EF13D595318AD" + "last_height": 0, + "last_round": 0, + "last_step": 0, + "priv_key": { + "type": "954568A3288910", + "value": "0EeInmBQL8MSnQq38zSxg47Z7R7Nmcu5a3GtWr9agUNtxTRGUyMSZYfSoqk7WdaJtxcHOx3paKJabvE9WVMYrQ==" } } diff --git a/test/p2p/data/mach3/core/config/genesis.json b/test/p2p/data/mach3/core/config/genesis.json index 8d5c6c7b9..19577db00 100644 --- a/test/p2p/data/mach3/core/config/genesis.json +++ b/test/p2p/data/mach3/core/config/genesis.json @@ -1,39 +1,39 @@ { - "app_hash": "", - "chain_id": "chain-9ujDWI", - "genesis_time": "2016-06-24T20:01:19.322Z", - "validators": [ - { - "power": 1, - "name": "mach1", - "pub_key": { - "type": "ed25519", - "data": "BE8933DFF1600C026E34718F1785A4CDEAB90C35698B394E38B6947AE91DE116" - } - }, - { - "power": 1, - "name": "mach2", - "pub_key": { - "type": "ed25519", - "data": "6DC534465323126587D2A2A93B59D689B717073B1DE968A25A6EF13D595318AD" - } - }, - { - "power": 1, - "name": "mach3", - "pub_key": { - "type": "ed25519", - "data": "AE67AC697D135AA0B4601EA57EAAB3FEBF4BAA4F229C45A598C2985B12FCD1A1" - } - }, - { - "power": 1, - "name": "mach4", - "pub_key": { - "type": "ed25519", - "data": "9EBC8F58CED4B46DCD5AB8ABA591DD253CD7CB5037273FDA32BC0B6461C4EFD9" - } - } - ] + "genesis_time": "2016-06-24T20:01:19.322Z", + "chain_id": "chain-9ujDWI", + "validators": [ + { + "pub_key": { + "type": "AC26791624DE60", + "value": "vokz3/FgDAJuNHGPF4Wkzeq5DDVpizlOOLaUeukd4RY=" + }, + "power": 1, + "name": "mach1" + }, + { + "pub_key": { + "type": "AC26791624DE60", + "value": "bcU0RlMjEmWH0qKpO1nWibcXBzsd6WiiWm7xPVlTGK0=" + }, + "power": 1, + "name": "mach2" + }, + { + "pub_key": { + "type": "AC26791624DE60", + "value": "rmesaX0TWqC0YB6lfqqz/r9Lqk8inEWlmMKYWxL80aE=" + }, + "power": 1, + "name": "mach3" + }, + { + "pub_key": { + "type": "AC26791624DE60", + "value": "nryPWM7UtG3NWrirpZHdJTzXy1A3Jz/aMrwLZGHE79k=" + }, + "power": 1, + "name": "mach4" + } + ], + "app_hash": "" } diff --git a/test/p2p/data/mach3/core/config/node_key.json b/test/p2p/data/mach3/core/config/node_key.json index 58880f6f3..82689b8e0 100644 --- a/test/p2p/data/mach3/core/config/node_key.json +++ b/test/p2p/data/mach3/core/config/node_key.json @@ -1 +1,6 @@ -{"priv_key":{"type":"ed25519","data":"913DE8AC6D18922A53F6B0196EF023B4693FECFBB565E084F0B4941768F3DAE892B35ADD954562FE071C465BC244B2AFAED4A270EC849269341473CE192DE682"}} \ No newline at end of file +{ + "priv_key": { + "type": "954568A3288910", + "value": "kT3orG0YkipT9rAZbvAjtGk/7Pu1ZeCE8LSUF2jz2uiSs1rdlUVi/gccRlvCRLKvrtSicOyEkmk0FHPOGS3mgg==" + } +} diff --git a/test/p2p/data/mach3/core/config/priv_validator.json b/test/p2p/data/mach3/core/config/priv_validator.json index 743a931f5..8ca68d563 100644 --- a/test/p2p/data/mach3/core/config/priv_validator.json +++ b/test/p2p/data/mach3/core/config/priv_validator.json @@ -1,14 +1,14 @@ { - "address": "4C5F061DAC28660853904A66705B12CA2B317572", - "last_height": 0, - "last_round": 0, - "last_step": 0, - "priv_key": { - "type": "ed25519", - "data": "C1A4E47F349FC5F556F4A9A27BA776B94424C312BAA6CF6EE44B867348D7C3F2AE67AC697D135AA0B4601EA57EAAB3FEBF4BAA4F229C45A598C2985B12FCD1A1" + "address": "4C5F061DAC28660853904A66705B12CA2B317572", + "pub_key": { + "type": "AC26791624DE60", + "value": "rmesaX0TWqC0YB6lfqqz/r9Lqk8inEWlmMKYWxL80aE=" }, - "pub_key": { - "type": "ed25519", - "data": "AE67AC697D135AA0B4601EA57EAAB3FEBF4BAA4F229C45A598C2985B12FCD1A1" + "last_height": 0, + "last_round": 0, + "last_step": 0, + "priv_key": { + "type": "954568A3288910", + "value": "waTkfzSfxfVW9Kmie6d2uUQkwxK6ps9u5EuGc0jXw/KuZ6xpfRNaoLRgHqV+qrP+v0uqTyKcRaWYwphbEvzRoQ==" } } diff --git a/test/p2p/data/mach4/core/config/genesis.json b/test/p2p/data/mach4/core/config/genesis.json index 8d5c6c7b9..19577db00 100644 --- a/test/p2p/data/mach4/core/config/genesis.json +++ b/test/p2p/data/mach4/core/config/genesis.json @@ -1,39 +1,39 @@ { - "app_hash": "", - "chain_id": "chain-9ujDWI", - "genesis_time": "2016-06-24T20:01:19.322Z", - "validators": [ - { - "power": 1, - "name": "mach1", - "pub_key": { - "type": "ed25519", - "data": "BE8933DFF1600C026E34718F1785A4CDEAB90C35698B394E38B6947AE91DE116" - } - }, - { - "power": 1, - "name": "mach2", - "pub_key": { - "type": "ed25519", - "data": "6DC534465323126587D2A2A93B59D689B717073B1DE968A25A6EF13D595318AD" - } - }, - { - "power": 1, - "name": "mach3", - "pub_key": { - "type": "ed25519", - "data": "AE67AC697D135AA0B4601EA57EAAB3FEBF4BAA4F229C45A598C2985B12FCD1A1" - } - }, - { - "power": 1, - "name": "mach4", - "pub_key": { - "type": "ed25519", - "data": "9EBC8F58CED4B46DCD5AB8ABA591DD253CD7CB5037273FDA32BC0B6461C4EFD9" - } - } - ] + "genesis_time": "2016-06-24T20:01:19.322Z", + "chain_id": "chain-9ujDWI", + "validators": [ + { + "pub_key": { + "type": "AC26791624DE60", + "value": "vokz3/FgDAJuNHGPF4Wkzeq5DDVpizlOOLaUeukd4RY=" + }, + "power": 1, + "name": "mach1" + }, + { + "pub_key": { + "type": "AC26791624DE60", + "value": "bcU0RlMjEmWH0qKpO1nWibcXBzsd6WiiWm7xPVlTGK0=" + }, + "power": 1, + "name": "mach2" + }, + { + "pub_key": { + "type": "AC26791624DE60", + "value": "rmesaX0TWqC0YB6lfqqz/r9Lqk8inEWlmMKYWxL80aE=" + }, + "power": 1, + "name": "mach3" + }, + { + "pub_key": { + "type": "AC26791624DE60", + "value": "nryPWM7UtG3NWrirpZHdJTzXy1A3Jz/aMrwLZGHE79k=" + }, + "power": 1, + "name": "mach4" + } + ], + "app_hash": "" } diff --git a/test/p2p/data/mach4/core/config/node_key.json b/test/p2p/data/mach4/core/config/node_key.json index 72e7b25df..a0c8d3917 100644 --- a/test/p2p/data/mach4/core/config/node_key.json +++ b/test/p2p/data/mach4/core/config/node_key.json @@ -1 +1,6 @@ -{"priv_key":{"type":"ed25519","data":"408226F3F40411AC22262DD9A33BFE27D6FED42B9F084906B3797118C951CB82F81552170A85C94F0608AE8B59B70A0CA8B604A9057585B28A266140DC615E97"}} \ No newline at end of file +{ + "priv_key": { + "type": "954568A3288910", + "value": "QIIm8/QEEawiJi3Zozv+J9b+1CufCEkGs3lxGMlRy4L4FVIXCoXJTwYIrotZtwoMqLYEqQV1hbKKJmFA3GFelw==" + } +} diff --git a/test/p2p/data/mach4/core/config/priv_validator.json b/test/p2p/data/mach4/core/config/priv_validator.json index 1c10eb8a6..7ef32a6f3 100644 --- a/test/p2p/data/mach4/core/config/priv_validator.json +++ b/test/p2p/data/mach4/core/config/priv_validator.json @@ -1,14 +1,14 @@ { - "address": "86F6DA4B34F16D743D2D992B5ACB12F5E724CC2D", - "last_height": 0, - "last_round": 0, - "last_step": 0, - "priv_key": { - "type": "ed25519", - "data": "C4CC3ED28F020C2DBDA98BCDBF08C3CED370470E74F25E938D5D295E8E3D2B0C9EBC8F58CED4B46DCD5AB8ABA591DD253CD7CB5037273FDA32BC0B6461C4EFD9" + "address": "86F6DA4B34F16D743D2D992B5ACB12F5E724CC2D", + "pub_key": { + "type": "AC26791624DE60", + "value": "nryPWM7UtG3NWrirpZHdJTzXy1A3Jz/aMrwLZGHE79k=" }, - "pub_key": { - "type": "ed25519", - "data": "9EBC8F58CED4B46DCD5AB8ABA591DD253CD7CB5037273FDA32BC0B6461C4EFD9" + "last_height": 0, + "last_round": 0, + "last_step": 0, + "priv_key": { + "type": "954568A3288910", + "value": "xMw+0o8CDC29qYvNvwjDztNwRw508l6TjV0pXo49KwyevI9YztS0bc1auKulkd0lPNfLUDcnP9oyvAtkYcTv2Q==" } } From bb9b12d67a1e53bb78e282f25299c910196fa09f Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sat, 7 Apr 2018 22:04:28 +0300 Subject: [PATCH 080/143] add scripts/wire2amino.go --- scripts/wire2amino.go | 181 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100644 scripts/wire2amino.go diff --git a/scripts/wire2amino.go b/scripts/wire2amino.go new file mode 100644 index 000000000..dd81ad2d8 --- /dev/null +++ b/scripts/wire2amino.go @@ -0,0 +1,181 @@ +package main + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "time" + + "github.com/tendermint/go-amino" + crypto "github.com/tendermint/go-crypto" + cmn "github.com/tendermint/tmlibs/common" + + "github.com/tendermint/tendermint/p2p" + "github.com/tendermint/tendermint/types" + priv_val "github.com/tendermint/tendermint/types/priv_validator" +) + +type GenesisValidator struct { + PubKey Data `json:"pub_key"` + Power int64 `json:"power"` + Name string `json:"name"` +} + +type Genesis struct { + GenesisTime time.Time `json:"genesis_time"` + ChainID string `json:"chain_id"` + ConsensusParams *types.ConsensusParams `json:"consensus_params,omitempty"` + Validators []GenesisValidator `json:"validators"` + AppHash cmn.HexBytes `json:"app_hash"` + AppStateJSON json.RawMessage `json:"app_state,omitempty"` + AppOptions json.RawMessage `json:"app_options,omitempty"` // DEPRECATED + +} + +type NodeKey struct { + PrivKey Data `json:"priv_key"` +} + +type PrivVal struct { + Address cmn.HexBytes `json:"address"` + LastHeight int64 `json:"last_height"` + LastRound int `json:"last_round"` + LastStep int8 `json:"last_step"` + PubKey Data `json:"pub_key"` + PrivKey Data `json:"priv_key"` +} + +type Data struct { + Type string `json:"type"` + Data cmn.HexBytes `json:"data"` +} + +func convertNodeKey(cdc *amino.Codec, jsonBytes []byte) ([]byte, error) { + var nodeKey NodeKey + err := json.Unmarshal(jsonBytes, &nodeKey) + if err != nil { + return nil, err + } + + var privKey crypto.PrivKeyEd25519 + copy(privKey[:], nodeKey.PrivKey.Data) + + nodeKeyNew := p2p.NodeKey{privKey} + + bz, err := cdc.MarshalJSON(nodeKeyNew) + if err != nil { + return nil, err + } + return bz, nil +} + +func convertPrivVal(cdc *amino.Codec, jsonBytes []byte) ([]byte, error) { + var privVal PrivVal + err := json.Unmarshal(jsonBytes, &privVal) + if err != nil { + return nil, err + } + + var privKey crypto.PrivKeyEd25519 + copy(privKey[:], privVal.PrivKey.Data) + + var pubKey crypto.PubKeyEd25519 + copy(pubKey[:], privVal.PubKey.Data) + + privValNew := priv_val.FilePV{ + Address: privVal.Address, + PubKey: pubKey, + LastHeight: privVal.LastHeight, + LastRound: privVal.LastRound, + LastStep: privVal.LastStep, + PrivKey: privKey, + } + + bz, err := cdc.MarshalJSON(privValNew) + if err != nil { + return nil, err + } + return bz, nil +} + +func convertGenesis(cdc *amino.Codec, jsonBytes []byte) ([]byte, error) { + var genesis Genesis + err := json.Unmarshal(jsonBytes, &genesis) + if err != nil { + return nil, err + } + + genesisNew := types.GenesisDoc{ + GenesisTime: genesis.GenesisTime, + ChainID: genesis.ChainID, + ConsensusParams: genesis.ConsensusParams, + // Validators + AppHash: genesis.AppHash, + AppStateJSON: genesis.AppStateJSON, + } + + if genesis.AppOptions != nil { + genesisNew.AppStateJSON = genesis.AppOptions + } + + for _, v := range genesis.Validators { + var pubKey crypto.PubKeyEd25519 + copy(pubKey[:], v.PubKey.Data) + genesisNew.Validators = append( + genesisNew.Validators, + types.GenesisValidator{ + PubKey: pubKey, + Power: v.Power, + Name: v.Name, + }, + ) + + } + + bz, err := cdc.MarshalJSON(genesisNew) + if err != nil { + return nil, err + } + return bz, nil +} + +func main() { + cdc := amino.NewCodec() + crypto.RegisterAmino(cdc) + + args := os.Args[1:] + if len(args) != 1 { + fmt.Println("Please specify a file to convert") + os.Exit(1) + } + + filePath := args[0] + fileName := filepath.Base(filePath) + + fileBytes, err := ioutil.ReadFile(filePath) + if err != nil { + panic(err) + } + + var bz []byte + + switch fileName { + case "node_key.json": + bz, err = convertNodeKey(cdc, fileBytes) + case "priv_validator.json": + bz, err = convertPrivVal(cdc, fileBytes) + case "genesis.json": + bz, err = convertGenesis(cdc, fileBytes) + default: + fmt.Println("Expected file name to be in (node_key.json, priv_validator.json, genesis.json)") + os.Exit(1) + } + + if err != nil { + panic(err) + } + fmt.Println(string(bz)) + +} From c778d7f5d1f9b5cdd24efe0d555e21eca5cc1389 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sat, 7 Apr 2018 23:13:41 +0300 Subject: [PATCH 081/143] fix addresses --- scripts/wire2amino.go | 2 +- test/p2p/data/mach1/core/config/priv_validator.json | 2 +- test/p2p/data/mach2/core/config/priv_validator.json | 2 +- test/p2p/data/mach3/core/config/priv_validator.json | 2 +- test/p2p/data/mach4/core/config/priv_validator.json | 2 +- test/p2p/peer.sh | 5 +---- 6 files changed, 6 insertions(+), 9 deletions(-) diff --git a/scripts/wire2amino.go b/scripts/wire2amino.go index dd81ad2d8..949082877 100644 --- a/scripts/wire2amino.go +++ b/scripts/wire2amino.go @@ -85,7 +85,7 @@ func convertPrivVal(cdc *amino.Codec, jsonBytes []byte) ([]byte, error) { copy(pubKey[:], privVal.PubKey.Data) privValNew := priv_val.FilePV{ - Address: privVal.Address, + Address: pubKey.Address(), PubKey: pubKey, LastHeight: privVal.LastHeight, LastRound: privVal.LastRound, diff --git a/test/p2p/data/mach1/core/config/priv_validator.json b/test/p2p/data/mach1/core/config/priv_validator.json index 4a7d3ade9..08c7c503b 100644 --- a/test/p2p/data/mach1/core/config/priv_validator.json +++ b/test/p2p/data/mach1/core/config/priv_validator.json @@ -1,5 +1,5 @@ { - "address": "0E6925C3EE4C599DFF1536A5071AF4A26DF33635", + "address": "7E9D1FB08EDBAFCF116638D4C8FAFAEE2ABE1AAA", "pub_key": { "type": "AC26791624DE60", "value": "vokz3/FgDAJuNHGPF4Wkzeq5DDVpizlOOLaUeukd4RY=" diff --git a/test/p2p/data/mach2/core/config/priv_validator.json b/test/p2p/data/mach2/core/config/priv_validator.json index b44541742..8e813dff4 100644 --- a/test/p2p/data/mach2/core/config/priv_validator.json +++ b/test/p2p/data/mach2/core/config/priv_validator.json @@ -1,5 +1,5 @@ { - "address": "99DBBD2AFC28FB5BAC5574AFAF0D9C806CED3B55", + "address": "8893D14FE09F1157E39CD34B98036048D51B4985", "pub_key": { "type": "AC26791624DE60", "value": "bcU0RlMjEmWH0qKpO1nWibcXBzsd6WiiWm7xPVlTGK0=" diff --git a/test/p2p/data/mach3/core/config/priv_validator.json b/test/p2p/data/mach3/core/config/priv_validator.json index 8ca68d563..84c98b98f 100644 --- a/test/p2p/data/mach3/core/config/priv_validator.json +++ b/test/p2p/data/mach3/core/config/priv_validator.json @@ -1,5 +1,5 @@ { - "address": "4C5F061DAC28660853904A66705B12CA2B317572", + "address": "7C747D7E002932B3864E3FBE9AC04287043F66A0", "pub_key": { "type": "AC26791624DE60", "value": "rmesaX0TWqC0YB6lfqqz/r9Lqk8inEWlmMKYWxL80aE=" diff --git a/test/p2p/data/mach4/core/config/priv_validator.json b/test/p2p/data/mach4/core/config/priv_validator.json index 7ef32a6f3..4f88045b5 100644 --- a/test/p2p/data/mach4/core/config/priv_validator.json +++ b/test/p2p/data/mach4/core/config/priv_validator.json @@ -1,5 +1,5 @@ { - "address": "86F6DA4B34F16D743D2D992B5ACB12F5E724CC2D", + "address": "CEBEFE3CA1363D425643EF63FC179E77A50A1E9A", "pub_key": { "type": "AC26791624DE60", "value": "nryPWM7UtG3NWrirpZHdJTzXy1A3Jz/aMrwLZGHE79k=" diff --git a/test/p2p/peer.sh b/test/p2p/peer.sh index d5ede231a..15d44ff33 100644 --- a/test/p2p/peer.sh +++ b/test/p2p/peer.sh @@ -10,13 +10,10 @@ set +u NODE_FLAGS=$5 set -u -set +eu - echo "starting tendermint peer ID=$ID" # start tendermint container on the network # NOTE: $NODE_FLAGS should be unescaped (no quotes). otherwise it will be # treated as one flag. -set -u docker run -d \ --net="$NETWORK_NAME" \ --ip=$(test/p2p/ip.sh "$ID") \ @@ -27,4 +24,4 @@ docker run -d \ --log-opt syslog-address=udp://127.0.0.1:5514 \ --log-opt syslog-facility=daemon \ --log-opt tag="{{.Name}}" \ - "$DOCKER_IMAGE" node $NODE_FLAGS --log_level=debug --proxy_app="$APP_PROXY" + "$DOCKER_IMAGE" node $NODE_FLAGS --log_level=debug --proxy_app="$APP_PROXY" From bb1b249e8a2d2569206c14e3f174d4dd175e20be Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 9 Apr 2018 15:04:59 +0300 Subject: [PATCH 082/143] types: lock block on MakePartSet --- types/block.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/types/block.go b/types/block.go index da5be2b32..34d2cb85d 100644 --- a/types/block.go +++ b/types/block.go @@ -108,6 +108,12 @@ func (b *Block) Hash() cmn.HexBytes { // MakePartSet returns a PartSet containing parts of a serialized block. // This is the form in which the block is gossipped to peers. func (b *Block) MakePartSet(partSize int) *PartSet { + if b == nil { + return nil + } + b.mtx.Lock() + defer b.mtx.Unlock() + // We prefix the byte length, so that unmarshaling // can easily happen via a reader. bz, err := cdc.MarshalBinary(b) From 1a1e4e767bec82e09a7cc6b02b4feb30787028f2 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 9 Apr 2018 15:14:33 +0300 Subject: [PATCH 083/143] check max msg size in DecodeMessage --- blockchain/reactor.go | 8 ++++++-- consensus/reactor.go | 14 +++++++++----- evidence/reactor.go | 11 ++++++++--- mempool/reactor.go | 6 +++++- p2p/pex/pex_reactor.go | 6 +++++- types/priv_validator.go | 1 - 6 files changed, 33 insertions(+), 13 deletions(-) diff --git a/blockchain/reactor.go b/blockchain/reactor.go index 64f4445a9..33dfdd288 100644 --- a/blockchain/reactor.go +++ b/blockchain/reactor.go @@ -30,7 +30,7 @@ const ( // NOTE: keep up to date with bcBlockResponseMessage bcBlockResponseMessagePrefixSize = 4 bcBlockResponseMessageFieldKeySize = 1 - maxMessageSize = types.MaxBlockSizeBytes + + maxMsgSize = types.MaxBlockSizeBytes + bcBlockResponseMessagePrefixSize + bcBlockResponseMessageFieldKeySize ) @@ -133,7 +133,7 @@ func (bcR *BlockchainReactor) GetChannels() []*p2p.ChannelDescriptor { Priority: 10, SendQueueCapacity: 1000, RecvBufferCapacity: 50 * 4096, - RecvMessageCapacity: maxMessageSize, + RecvMessageCapacity: maxMsgSize, }, } } @@ -345,6 +345,10 @@ func RegisterBlockchainMessages(cdc *amino.Codec) { // DecodeMessage decodes BlockchainMessage. // TODO: ensure that bz is completely read. func DecodeMessage(bz []byte) (msg BlockchainMessage, err error) { + if len(bz) > maxMsgSize { + return msg, fmt.Errorf("Msg exceeds max size (%d > %d)", + len(bz), maxMsgSize) + } err = cdc.UnmarshalBinaryBare(bz, &msg) if err != nil { err = cmn.ErrorWrap(err, "DecodeMessage() had bytes left over") diff --git a/consensus/reactor.go b/consensus/reactor.go index 2c8d4d85a..8f341184e 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -25,7 +25,7 @@ const ( VoteChannel = byte(0x22) VoteSetBitsChannel = byte(0x23) - maxConsensusMessageSize = 1048576 // 1MB; NOTE/TODO: keep in sync with types.PartSet sizes. + maxMsgSize = 1048576 // 1MB; NOTE/TODO: keep in sync with types.PartSet sizes. blocksToContributeToBecomeGoodPeer = 10000 ) @@ -112,28 +112,28 @@ func (conR *ConsensusReactor) GetChannels() []*p2p.ChannelDescriptor { ID: StateChannel, Priority: 5, SendQueueCapacity: 100, - RecvMessageCapacity: maxConsensusMessageSize, + RecvMessageCapacity: maxMsgSize, }, { ID: DataChannel, // maybe split between gossiping current block and catchup stuff Priority: 10, // once we gossip the whole block there's nothing left to send until next height or round SendQueueCapacity: 100, RecvBufferCapacity: 50 * 4096, - RecvMessageCapacity: maxConsensusMessageSize, + RecvMessageCapacity: maxMsgSize, }, { ID: VoteChannel, Priority: 5, SendQueueCapacity: 100, RecvBufferCapacity: 100 * 100, - RecvMessageCapacity: maxConsensusMessageSize, + RecvMessageCapacity: maxMsgSize, }, { ID: VoteSetBitsChannel, Priority: 1, SendQueueCapacity: 2, RecvBufferCapacity: 1024, - RecvMessageCapacity: maxConsensusMessageSize, + RecvMessageCapacity: maxMsgSize, }, } } @@ -1314,6 +1314,10 @@ func RegisterConsensusMessages(cdc *amino.Codec) { // DecodeMessage decodes the given bytes into a ConsensusMessage. func DecodeMessage(bz []byte) (msg ConsensusMessage, err error) { + if len(bz) > maxMsgSize { + return msg, fmt.Errorf("Msg exceeds max size (%d > %d)", + len(bz), maxMsgSize) + } err = cdc.UnmarshalBinaryBare(bz, &msg) return } diff --git a/evidence/reactor.go b/evidence/reactor.go index 27cbdbf31..a6aa66b1f 100644 --- a/evidence/reactor.go +++ b/evidence/reactor.go @@ -2,11 +2,12 @@ package evidence import ( "fmt" - "github.com/tendermint/go-amino" - "github.com/tendermint/tmlibs/log" "reflect" "time" + "github.com/tendermint/go-amino" + "github.com/tendermint/tmlibs/log" + "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/types" ) @@ -14,7 +15,7 @@ import ( const ( EvidenceChannel = byte(0x38) - maxEvidenceMessageSize = 1048576 // 1MB TODO make it configurable + maxMsgSize = 1048576 // 1MB TODO make it configurable broadcastEvidenceIntervalS = 60 // broadcast uncommitted evidence this often ) @@ -146,6 +147,10 @@ func RegisterEvidenceMessages(cdc *amino.Codec) { // DecodeMessage decodes a byte-array into a EvidenceMessage. func DecodeMessage(bz []byte) (msg EvidenceMessage, err error) { + if len(bz) > maxMsgSize { + return msg, fmt.Errorf("Msg exceeds max size (%d > %d)", + len(bz), maxMsgSize) + } err = cdc.UnmarshalBinaryBare(bz, &msg) return } diff --git a/mempool/reactor.go b/mempool/reactor.go index acd693e42..54a3c32fe 100644 --- a/mempool/reactor.go +++ b/mempool/reactor.go @@ -18,7 +18,7 @@ import ( const ( MempoolChannel = byte(0x30) - maxMempoolMessageSize = 1048576 // 1MB TODO make it configurable + maxMsgSize = 1048576 // 1MB TODO make it configurable peerCatchupSleepIntervalMS = 100 // If peer is behind, sleep this amount ) @@ -167,6 +167,10 @@ func RegisterMempoolMessages(cdc *amino.Codec) { // DecodeMessage decodes a byte-array into a MempoolMessage. func DecodeMessage(bz []byte) (msg MempoolMessage, err error) { + if len(bz) > maxMsgSize { + return msg, fmt.Errorf("Msg exceeds max size (%d > %d)", + len(bz), maxMsgSize) + } err = cdc.UnmarshalBinaryBare(bz, &msg) return } diff --git a/p2p/pex/pex_reactor.go b/p2p/pex/pex_reactor.go index db4bb9f08..00144c35f 100644 --- a/p2p/pex/pex_reactor.go +++ b/p2p/pex/pex_reactor.go @@ -21,7 +21,7 @@ const ( // PexChannel is a channel for PEX messages PexChannel = byte(0x00) - maxPexMessageSize = 1048576 // 1MB + maxMsgSize = 1048576 // 1MB // ensure we have enough peers defaultEnsurePeersPeriod = 30 * time.Second @@ -616,6 +616,10 @@ func RegisterPexMessage(cdc *amino.Codec) { // DecodeMessage implements interface registered above. func DecodeMessage(bz []byte) (msg PexMessage, err error) { + if len(bz) > maxMsgSize { + return msg, fmt.Errorf("Msg exceeds max size (%d > %d)", + len(bz), maxMsgSize) + } err = cdc.UnmarshalBinary(bz, &msg) return } diff --git a/types/priv_validator.go b/types/priv_validator.go index ecba4456f..8759d3f99 100644 --- a/types/priv_validator.go +++ b/types/priv_validator.go @@ -92,5 +92,4 @@ func (pv *MockPV) String() string { func (pv *MockPV) DisableChecks() { // Currently this does nothing, // as MockPV has no safety checks at all. - return } From dcd00b0e688f133b9634aee513f8bf856e659356 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 9 Apr 2018 16:36:28 +0300 Subject: [PATCH 084/143] update deps and changelog --- CHANGELOG.md | 12 ++++++++++++ Gopkg.lock | 13 ++++++------- Gopkg.toml | 8 +++----- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c5dc06753..f54c9805d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,18 @@ BUG FIXES: - Graceful handling/recovery for apps that have non-determinism or fail to halt - Graceful handling/recovery for violations of safety, or liveness +## 0.19.0 (TBD) + +BREAKING: + +Upgrade from go-wire to go-amino. This is a sweeping change that breaks everything that is +serialized to disk or over the network. + +See github.com/tendermint/go-amino for details on the new format. + +See `scripts/wire2amino.go` for a tool to upgrade +genesis/priv_validator/node_key JSON files. + ## 0.18.0 (April 6th, 2018) BREAKING: diff --git a/Gopkg.lock b/Gopkg.lock index 81f3c7033..a634a1ad3 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -238,8 +238,8 @@ "server", "types" ] - revision = "c62aed95f2ce399ec815b0cafe478af002cdc4e6" - version = "v0.10.3-dev" + revision = "78a8905690ef54f9d57e3b2b0ee7ad3a04ef3f1f" + version = "v0.10.3" [[projects]] branch = "master" @@ -260,8 +260,8 @@ [[projects]] name = "github.com/tendermint/go-crypto" packages = ["."] - revision = "5d5f580f49ca66c13400938c64334186068c8b7c" - version = "v0.6.1" + revision = "915416979bf70efa4bcbf1c6cd5d64c5fff9fc19" + version = "v0.6.2" [[projects]] name = "github.com/tendermint/go-wire" @@ -286,7 +286,7 @@ "test" ] revision = "2e24b64fc121dcdf1cabceab8dc2f7257675483c" - version = "0.8.1" + version = "v0.8.1" [[projects]] branch = "master" @@ -372,7 +372,6 @@ "transport" ] revision = "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" - source = "github.com/grpc/grpc-go" version = "v1.7.5" [[projects]] @@ -384,6 +383,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "e5e6c7942710846bdb5589fba10c90a869773f2685da825b955510afe0d7c5a4" + inputs-digest = "9b49dde42e615372db85e13d7952cb547b8b1833458a4906f908af988cf0bd4f" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 361cc1735..47b6460f5 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -71,25 +71,23 @@ [[constraint]] name = "github.com/tendermint/abci" - version = "0.10.3-dev" + version = "~0.10.3" [[constraint]] name = "github.com/tendermint/go-crypto" - version = "0.6.1" + version = "~0.6.2" [[constraint]] name = "github.com/tendermint/go-amino" - version = "0.9.6" + version = "~0.9.6" [[override]] # [[constraint]] name = "github.com/tendermint/tmlibs" version = "~0.8.1" - # branch = "develop" [[constraint]] name = "google.golang.org/grpc" - source = "github.com/grpc/grpc-go" version = "~1.7.3" [prune] From 2b732bc11aaa1ace7f9b61727ddcaa0ea11e0c5c Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Mon, 9 Apr 2018 15:41:26 +0200 Subject: [PATCH 085/143] generate node_key when running tendermint init --- cmd/tendermint/commands/init.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/cmd/tendermint/commands/init.go b/cmd/tendermint/commands/init.go index 224452646..2faff20fe 100644 --- a/cmd/tendermint/commands/init.go +++ b/cmd/tendermint/commands/init.go @@ -3,6 +3,7 @@ package commands import ( "github.com/spf13/cobra" + "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/types" pvm "github.com/tendermint/tendermint/types/priv_validator" cmn "github.com/tendermint/tmlibs/common" @@ -28,6 +29,16 @@ func initFiles(cmd *cobra.Command, args []string) { logger.Info("Generated private validator", "path", privValFile) } + nodeKeyFile := config.NodeKeyFile() + if cmn.FileExists(nodeKeyFile) { + logger.Info("Found node key", "path", nodeKeyFile) + } else { + if _, err := p2p.LoadOrGenNodeKey(nodeKeyFile); err != nil { + panic(err) + } + logger.Info("Generated node key", "path", nodeKeyFile) + } + // genesis file genFile := config.GenesisFile() if cmn.FileExists(genFile) { From 9be16d56ba4081213435d918b2ec7a709095e16e Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Mon, 9 Apr 2018 15:42:19 +0200 Subject: [PATCH 086/143] [docs] prefix IPs with node IDs Refs #1429 --- docs/deploy-testnets.rst | 16 ++++++++-------- docs/examples/getting-started.md | 25 ++++++++++++++++--------- docs/examples/install_tendermint.sh | 2 +- docs/examples/node1/node_key.json | 6 ++++++ docs/examples/node2/node_key.json | 6 ++++++ docs/examples/node3/node_key.json | 6 ++++++ docs/examples/node4/node_key.json | 6 ++++++ docs/using-tendermint.rst | 14 +++++++------- 8 files changed, 56 insertions(+), 25 deletions(-) create mode 100644 docs/examples/node1/node_key.json create mode 100644 docs/examples/node2/node_key.json create mode 100644 docs/examples/node3/node_key.json create mode 100644 docs/examples/node4/node_key.json diff --git a/docs/deploy-testnets.rst b/docs/deploy-testnets.rst index 5740ca56f..32355e4ae 100644 --- a/docs/deploy-testnets.rst +++ b/docs/deploy-testnets.rst @@ -11,26 +11,26 @@ Manual Deployments It's relatively easy to setup a Tendermint cluster manually. The only requirements for a particular Tendermint node are a private key for the -validator, stored as ``priv_validator.json``, and a list of the public -keys of all validators, stored as ``genesis.json``. These files should -be stored in ``~/.tendermint/config``, or wherever the ``$TMHOME`` variable -might be set to. +validator, stored as ``priv_validator.json``, a node key, stored as +``node_key.json`` and a list of the public keys of all validators, stored as +``genesis.json``. These files should be stored in ``~/.tendermint/config``, or +wherever the ``$TMHOME`` variable might be set to. Here are the steps to setting up a testnet manually: 1) Provision nodes on your cloud provider of choice 2) Install Tendermint and the application of interest on all nodes -3) Generate a private key for each validator using - ``tendermint gen_validator`` +3) Generate a private key and a node key for each validator using + ``tendermint init`` 4) Compile a list of public keys for each validator into a - ``genesis.json`` file. + ``genesis.json`` file and replace the existing file with it. 5) Run ``tendermint node --p2p.persistent_peers=< peer addresses >`` on each node, where ``< peer addresses >`` is a comma separated list of the IP:PORT combination for each node. The default port for Tendermint is ``46656``. Thus, if the IP addresses of your nodes were ``192.168.0.1, 192.168.0.2, 192.168.0.3, 192.168.0.4``, the command would look like: - ``tendermint node --p2p.persistent_peers=192.168.0.1:46656,192.168.0.2:46656,192.168.0.3:46656,192.168.0.4:46656``. + ``tendermint node --p2p.persistent_peers=96663a3dd0d7b9d17d4c8211b191af259621c693@192.168.0.1:46656, 429fcf25974313b95673f58d77eacdd434402665@192.168.0.2:46656, 0491d373a8e0fcf1023aaf18c51d6a1d0d4f31bd@192.168.0.3:46656, f9baeaa15fedf5e1ef7448dd60f46c01f1a9e9c4@192.168.0.4:46656``. After a few seconds, all the nodes should connect to each other and start making blocks! For more information, see the Tendermint Networks section diff --git a/docs/examples/getting-started.md b/docs/examples/getting-started.md index 1675b55b2..59015b79f 100644 --- a/docs/examples/getting-started.md +++ b/docs/examples/getting-started.md @@ -2,8 +2,9 @@ ## Overview -This is a quick start guide. If you have a vague idea about how Tendermint works -and want to get started right away, continue. Otherwise, [review the documentation](http://tendermint.readthedocs.io/en/master/) +This is a quick start guide. If you have a vague idea about how Tendermint +works and want to get started right away, continue. Otherwise, [review the +documentation](http://tendermint.readthedocs.io/en/master/). ## Install @@ -42,7 +43,7 @@ Confirm installation: ``` $ tendermint version -0.15.0-381fe19 +0.18.0-XXXXXXX ``` ## Initialization @@ -117,7 +118,9 @@ where the value is returned in hex. ## Cluster of Nodes -First create four Ubuntu cloud machines. The following was tested on Digital Ocean Ubuntu 16.04 x64 (3GB/1CPU, 20GB SSD). We'll refer to their respective IP addresses below as IP1, IP2, IP3, IP4. +First create four Ubuntu cloud machines. The following was tested on Digital +Ocean Ubuntu 16.04 x64 (3GB/1CPU, 20GB SSD). We'll refer to their respective IP +addresses below as IP1, IP2, IP3, IP4. Then, `ssh` into each machine, and execute [this script](https://git.io/vNLfY): @@ -131,12 +134,16 @@ This will install `go` and other dependencies, get the Tendermint source code, t Next, `cd` into `docs/examples`. Each command below should be run from each node, in sequence: ``` -tendermint node --home ./node1 --proxy_app=kvstore --p2p.seeds IP1:46656,IP2:46656,IP3:46656,IP4:46656 -tendermint node --home ./node2 --proxy_app=kvstore --p2p.seeds IP1:46656,IP2:46656,IP3:46656,IP4:46656 -tendermint node --home ./node3 --proxy_app=kvstore --p2p.seeds IP1:46656,IP2:46656,IP3:46656,IP4:46656 -tendermint node --home ./node4 --proxy_app=kvstore --p2p.seeds IP1:46656,IP2:46656,IP3:46656,IP4:46656 +tendermint node --home ./node1 --proxy_app=kvstore --p2p.persistent_peers="3a558bd6f8c97453aa6c2372bb800e8b6ed8e6db@IP1:46656,ccf30d873fddda10a495f42687c8f33472a6569f@IP2:46656,9a4c3de5d6788a76c6ee3cd9ff41e3b45b4cfd14@IP3:46656,58e6f2ab297b3ceae107ba4c8c2898da5c009ff4@IP4:46656" +tendermint node --home ./node2 --proxy_app=kvstore --p2p.persistent_peers="3a558bd6f8c97453aa6c2372bb800e8b6ed8e6db@IP1:46656,ccf30d873fddda10a495f42687c8f33472a6569f@IP2:46656,9a4c3de5d6788a76c6ee3cd9ff41e3b45b4cfd14@IP3:46656,58e6f2ab297b3ceae107ba4c8c2898da5c009ff4@IP4:46656" +tendermint node --home ./node3 --proxy_app=kvstore --p2p.persistent_peers="3a558bd6f8c97453aa6c2372bb800e8b6ed8e6db@IP1:46656,ccf30d873fddda10a495f42687c8f33472a6569f@IP2:46656,9a4c3de5d6788a76c6ee3cd9ff41e3b45b4cfd14@IP3:46656,58e6f2ab297b3ceae107ba4c8c2898da5c009ff4@IP4:46656" +tendermint node --home ./node4 --proxy_app=kvstore --p2p.persistent_peers="3a558bd6f8c97453aa6c2372bb800e8b6ed8e6db@IP1:46656,ccf30d873fddda10a495f42687c8f33472a6569f@IP2:46656,9a4c3de5d6788a76c6ee3cd9ff41e3b45b4cfd14@IP3:46656,58e6f2ab297b3ceae107ba4c8c2898da5c009ff4@IP4:46656" ``` -Note that after the third node is started, blocks will start to stream in because >2/3 of validators (defined in the `genesis.json`) have come online. Seeds can also be specified in the `config.toml`. See [this PR](https://github.com/tendermint/tendermint/pull/792) for more information about configuration options. +Note that after the third node is started, blocks will start to stream in +because >2/3 of validators (defined in the `genesis.json`) have come online. +Seeds can also be specified in the `config.toml`. See [this +PR](https://github.com/tendermint/tendermint/pull/792) for more information +about configuration options. Transactions can then be sent as covered in the single, local node example above. diff --git a/docs/examples/install_tendermint.sh b/docs/examples/install_tendermint.sh index d58b84d04..aeb87db5f 100644 --- a/docs/examples/install_tendermint.sh +++ b/docs/examples/install_tendermint.sh @@ -26,7 +26,7 @@ go get $REPO cd $GOPATH/src/$REPO ## build -git checkout v0.17.0 +git checkout v0.18.0 make get_tools make get_vendor_deps make install diff --git a/docs/examples/node1/node_key.json b/docs/examples/node1/node_key.json new file mode 100644 index 000000000..de1c41718 --- /dev/null +++ b/docs/examples/node1/node_key.json @@ -0,0 +1,6 @@ +{ + "priv_key" : { + "data" : "DA9BAABEA7211A6D93D9A1986B4279EAB3021FAA1653D459D53E6AB4D1CFB4C69BF7D52E48CF00AC5779AA0A6D3C368955D5636A677F72370B8ED19989714CFC", + "type" : "ed25519" + } +} diff --git a/docs/examples/node2/node_key.json b/docs/examples/node2/node_key.json new file mode 100644 index 000000000..4e8b0b100 --- /dev/null +++ b/docs/examples/node2/node_key.json @@ -0,0 +1,6 @@ +{ + "priv_key" : { + "data" : "F7BCABA165DFC0DDD50AE563EFB285BAA236EA805D35612504238A36EFA105958756442B1D9F942D7ABD259F2D59671657B6378E9C7194342A7AAA47A66D1E95", + "type" : "ed25519" + } +} diff --git a/docs/examples/node3/node_key.json b/docs/examples/node3/node_key.json new file mode 100644 index 000000000..32fdeee9d --- /dev/null +++ b/docs/examples/node3/node_key.json @@ -0,0 +1,6 @@ +{ + "priv_key" : { + "data" : "95136FCC97E4446B3141EDF9841078107ECE755E99925D79CCBF91085492680B3CA1034D9917DF1DED4E4AB2D9BC225919F6CB2176F210D2368697CC339DF4E7", + "type" : "ed25519" + } +} diff --git a/docs/examples/node4/node_key.json b/docs/examples/node4/node_key.json new file mode 100644 index 000000000..955fc989a --- /dev/null +++ b/docs/examples/node4/node_key.json @@ -0,0 +1,6 @@ +{ + "priv_key" : { + "data" : "8895D6C9A1B46AB83A8E2BAE2121B8C3E245B9E9126EBD797FEAC5058285F2F64FDE2E8182C88AD5185A49D837C581465D57BD478C41865A66D7D9742D8AEF57", + "type" : "ed25519" + } +} diff --git a/docs/using-tendermint.rst b/docs/using-tendermint.rst index e07534c96..13c2d882f 100644 --- a/docs/using-tendermint.rst +++ b/docs/using-tendermint.rst @@ -81,13 +81,13 @@ We can see the chain's status at the ``/status`` end-point: :: - curl http://localhost:46657/status | jsonpp + curl http://localhost:46657/status | jsonpp and the ``latest_app_hash`` in particular: :: - curl http://localhost:46657/status | jsonpp | grep app_hash + curl http://localhost:46657/status | jsonpp | grep app_hash Visit http://localhost:46657 in your browser to see the list of other endpoints. Some take no arguments (like ``/status``), while others @@ -185,7 +185,7 @@ once per second, it is possible to disable empty blocks or set a block creation interval. In the former case, blocks will be created when there are new transactions or when the AppHash changes. -To configure Tendermint to not produce empty blocks unless there are +To configure Tendermint to not produce empty blocks unless there are transactions or the app hash changes, run Tendermint with this additional flag: :: @@ -334,14 +334,14 @@ For instance, :: - tendermint node --p2p.seeds "1.2.3.4:46656,5.6.7.8:46656" + tendermint node --p2p.seeds "f9baeaa15fedf5e1ef7448dd60f46c01f1a9e9c4@1.2.3.4:46656,0491d373a8e0fcf1023aaf18c51d6a1d0d4f31bd@5.6.7.8:46656" Alternatively, you can use the ``/dial_seeds`` endpoint of the RPC to specify seeds for a running node to connect to: :: - curl 'localhost:46657/dial_seeds?seeds=\["1.2.3.4:46656","5.6.7.8:46656"\]' + curl 'localhost:46657/dial_seeds?seeds=\["f9baeaa15fedf5e1ef7448dd60f46c01f1a9e9c4@1.2.3.4:46656","0491d373a8e0fcf1023aaf18c51d6a1d0d4f31bd@5.6.7.8:46656"\]' Note, if the peer-exchange protocol (PEX) is enabled (default), you should not normally need seeds after the first start. Peers will be gossipping about known @@ -355,8 +355,8 @@ core instance. :: - tendermint node --p2p.persistent_peers "10.11.12.13:46656,10.11.12.14:46656" - curl 'localhost:46657/dial_peers?persistent=true&peers=\["1.2.3.4:46656","5.6.7.8:46656"\]' + tendermint node --p2p.persistent_peers "429fcf25974313b95673f58d77eacdd434402665@10.11.12.13:46656,96663a3dd0d7b9d17d4c8211b191af259621c693@10.11.12.14:46656" + curl 'localhost:46657/dial_peers?persistent=true&peers=\["429fcf25974313b95673f58d77eacdd434402665@10.11.12.13:46656","96663a3dd0d7b9d17d4c8211b191af259621c693@10.11.12.14:46656"\]' Adding a Non-Validator ~~~~~~~~~~~~~~~~~~~~~~ From c954fca376420c867864f1cb0b68b561409eb22b Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Mon, 9 Apr 2018 15:55:56 +0200 Subject: [PATCH 087/143] gen_node_key cmd --- cmd/tendermint/commands/gen_node_key.go | 32 +++++++++++++++++++++++++ cmd/tendermint/main.go | 1 + 2 files changed, 33 insertions(+) create mode 100644 cmd/tendermint/commands/gen_node_key.go diff --git a/cmd/tendermint/commands/gen_node_key.go b/cmd/tendermint/commands/gen_node_key.go new file mode 100644 index 000000000..4990be477 --- /dev/null +++ b/cmd/tendermint/commands/gen_node_key.go @@ -0,0 +1,32 @@ +package commands + +import ( + "fmt" + + "github.com/spf13/cobra" + + "github.com/tendermint/tendermint/p2p" + cmn "github.com/tendermint/tmlibs/common" +) + +// GenNodeKeyCmd allows the generation of a node key. It prints node's ID to +// the standard output. +var GenNodeKeyCmd = &cobra.Command{ + Use: "gen_node_key", + Short: "Generate a node key for this node and print its ID", + RunE: genNodeKey, +} + +func genNodeKey(cmd *cobra.Command, args []string) error { + nodeKeyFile := config.NodeKeyFile() + if cmn.FileExists(nodeKeyFile) { + return fmt.Errorf("node key at %s already exists", nodeKeyFile) + } + + nodeKey, err := p2p.LoadOrGenNodeKey(nodeKeyFile) + if err != nil { + return err + } + fmt.Println(nodeKey.ID()) + return nil +} diff --git a/cmd/tendermint/main.go b/cmd/tendermint/main.go index fd6287115..8c7f0cd17 100644 --- a/cmd/tendermint/main.go +++ b/cmd/tendermint/main.go @@ -25,6 +25,7 @@ func main() { cmd.ShowValidatorCmd, cmd.TestnetFilesCmd, cmd.ShowNodeIDCmd, + cmd.GenNodeKeyCmd, cmd.VersionCmd) // NOTE: From 609452958c7b1a5add06f021de427f515e3d610c Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Mon, 9 Apr 2018 16:12:35 +0200 Subject: [PATCH 088/143] [docs/specification/secure-p2p] add a note about config --- docs/specification/secure-p2p.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/specification/secure-p2p.rst b/docs/specification/secure-p2p.rst index c48ddb590..2110c835b 100644 --- a/docs/specification/secure-p2p.rst +++ b/docs/specification/secure-p2p.rst @@ -62,6 +62,13 @@ such as the Web-of-Trust or Certificate Authorities. In our case, we can use the blockchain itself as a certificate authority to ensure that we are connected to at least one validator. +Config +------ + +Authenticated encryption is enabled by default. If you wish to use another +authentication scheme or your peers are connected via VPN, you can turn it off +by setting ``auth_enc`` to ``false`` in the config file. + Additional Reading ------------------ From 26c38e770ebdac15556922f95de1cf9eb4e70b93 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Tue, 10 Apr 2018 11:15:16 +0200 Subject: [PATCH 089/143] fix data race Closes #1442 ``` WARNING: DATA RACE Write at 0x00c4209de7c8 by goroutine 23: github.com/tendermint/tendermint/types.(*Block).fillHeader() /home/vagrant/go/src/github.com/tendermint/tendermint/types/block.go:88 +0x157 github.com/tendermint/tendermint/types.(*Block).Hash() /home/vagrant/go/src/github.com/tendermint/tendermint/types/block.go:104 +0x121 github.com/tendermint/tendermint/types.(*Block).HashesTo() /home/vagrant/go/src/github.com/tendermint/tendermint/types/block.go:135 +0x4f github.com/tendermint/tendermint/consensus.(*ConsensusState).enterPrecommit() /home/vagrant/go/src/github.com/tendermint/tendermint/consensus/state.go:1037 +0x182d github.com/tendermint/tendermint/consensus.(*ConsensusState).addVote() /home/vagrant/go/src/github.com/tendermint/tendermint/consensus/state.go:1425 +0x1a6c github.com/tendermint/tendermint/consensus.(*ConsensusState).tryAddVote() /home/vagrant/go/src/github.com/tendermint/tendermint/consensus/state.go:1318 +0x77 github.com/tendermint/tendermint/consensus.(*ConsensusState).handleMsg() /home/vagrant/go/src/github.com/tendermint/tendermint/consensus/state.go:581 +0x7a9 github.com/tendermint/tendermint/consensus.(*ConsensusState).receiveRoutine() /home/vagrant/go/src/github.com/tendermint/tendermint/consensus/state.go:539 +0x6c3 Previous read at 0x00c4209de7c8 by goroutine 47: github.com/tendermint/tendermint/vendor/github.com/tendermint/tmlibs/common.(*HexBytes).MarshalJSON() :1 +0x52 github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino.invokeMarshalJSON() /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino/json-encode.go:433 +0x88 github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino.(*Codec)._encodeReflectJSON() /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino/json-encode.go:82 +0x8d2 github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino.(*Codec).encodeReflectJSON() /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino/json-encode.go:50 +0x10e github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino.(*Codec).encodeReflectJSONStruct() /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino/json-encode.go:348 +0x539 github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino.(*Codec)._encodeReflectJSON() /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino/json-encode.go:119 +0x83f github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino.(*Codec).encodeReflectJSON() /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino/json-encode.go:50 +0x10e github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino.(*Codec).encodeReflectJSONStruct() /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino/json-encode.go:348 +0x539 github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino.(*Codec)._encodeReflectJSON() /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino/json-encode.go:119 +0x83f github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino.(*Codec).encodeReflectJSON() /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino/json-encode.go:50 +0x10e github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino.(*Codec).encodeReflectJSONStruct() /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino/json-encode.go:348 +0x539 github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino.(*Codec)._encodeReflectJSON() /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino/json-encode.go:119 +0x83f github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino.(*Codec).encodeReflectJSON() /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino/json-encode.go:50 +0x10e github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino.(*Codec).encodeReflectJSONStruct() /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino/json-encode.go:348 +0x539 github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino.(*Codec)._encodeReflectJSON() /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino/json-encode.go:119 +0x83f github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino.(*Codec).encodeReflectJSON() /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino/json-encode.go:50 +0x10e github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino.(*Codec).MarshalJSON() /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino/amino.go:296 +0x182 github.com/tendermint/tendermint/rpc/lib/types.NewRPCSuccessResponse() /home/vagrant/go/src/github.com/tendermint/tendermint/rpc/lib/types/types.go:100 +0x12c github.com/tendermint/tendermint/rpc/lib/server.makeJSONRPCHandler.func1() /home/vagrant/go/src/github.com/tendermint/tendermint/rpc/lib/server/handlers.go:152 +0xab7 net/http.HandlerFunc.ServeHTTP() /usr/lib/go-1.9/src/net/http/server.go:1918 +0x51 net/http.(*ServeMux).ServeHTTP() /usr/lib/go-1.9/src/net/http/server.go:2254 +0xa2 github.com/tendermint/tendermint/rpc/lib/server.RecoverAndLogHandler.func1() /home/vagrant/go/src/github.com/tendermint/tendermint/rpc/lib/server/http_server.go:138 +0x4fa net/http.HandlerFunc.ServeHTTP() /usr/lib/go-1.9/src/net/http/server.go:1918 +0x51 net/http.serverHandler.ServeHTTP() /usr/lib/go-1.9/src/net/http/server.go:2619 +0xbc net/http.(*conn).serve() /usr/lib/go-1.9/src/net/http/server.go:1801 +0x83b Goroutine 23 (running) created at: github.com/tendermint/tendermint/consensus.(*ConsensusState).OnStart() /home/vagrant/go/src/github.com/tendermint/tendermint/consensus/state.go:250 +0x35b github.com/tendermint/tendermint/vendor/github.com/tendermint/tmlibs/common.(*BaseService).Start() /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/tmlibs/common/service.go:130 +0x5fc github.com/tendermint/tendermint/consensus.(*ConsensusReactor).OnStart() /home/vagrant/go/src/github.com/tendermint/tendermint/consensus/reactor.go:69 +0x1b4 github.com/tendermint/tendermint/vendor/github.com/tendermint/tmlibs/common.(*BaseService).Start() /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/tmlibs/common/service.go:130 +0x5fc github.com/tendermint/tendermint/consensus.(*ConsensusReactor).Start() :1 +0x43 github.com/tendermint/tendermint/p2p.(*Switch).OnStart() /home/vagrant/go/src/github.com/tendermint/tendermint/p2p/switch.go:177 +0x124 github.com/tendermint/tendermint/vendor/github.com/tendermint/tmlibs/common.(*BaseService).Start() /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/tmlibs/common/service.go:130 +0x5fc github.com/tendermint/tendermint/node.(*Node).OnStart() /home/vagrant/go/src/github.com/tendermint/tendermint/node/node.go:416 +0xa1b github.com/tendermint/tendermint/vendor/github.com/tendermint/tmlibs/common.(*BaseService).Start() /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/tmlibs/common/service.go:130 +0x5fc github.com/tendermint/tendermint/rpc/test.StartTendermint() /home/vagrant/go/src/github.com/tendermint/tendermint/rpc/test/helpers.go:100 +0x5b github.com/tendermint/tendermint/rpc/client_test.TestMain() /home/vagrant/go/src/github.com/tendermint/tendermint/rpc/client/main_test.go:17 +0x4c main.main() github.com/tendermint/tendermint/rpc/client/_test/_testmain.go:76 +0x1cd Goroutine 47 (running) created at: net/http.(*Server).Serve() /usr/lib/go-1.9/src/net/http/server.go:2720 +0x37c net/http.Serve() /usr/lib/go-1.9/src/net/http/server.go:2323 +0xe2 github.com/tendermint/tendermint/rpc/lib/server.StartHTTPServer.func1() /home/vagrant/go/src/github.com/tendermint/tendermint/rpc/lib/server/http_server.go:35 +0xb3 ``` --- consensus/reactor.go | 20 +++++++++++++++----- consensus/state.go | 13 +++++++++---- consensus/types/state.go | 2 +- rpc/core/consensus.go | 18 +++++++++++++----- rpc/core/pipe.go | 5 ++--- rpc/core/types/responses.go | 6 +++--- 6 files changed, 43 insertions(+), 21 deletions(-) diff --git a/consensus/reactor.go b/consensus/reactor.go index 8f341184e..385e8d867 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -9,7 +9,7 @@ import ( "github.com/pkg/errors" - "github.com/tendermint/go-amino" + amino "github.com/tendermint/go-amino" cmn "github.com/tendermint/tmlibs/common" "github.com/tendermint/tmlibs/log" @@ -838,8 +838,8 @@ var ( ErrPeerStateInvalidStartTime = errors.New("Error peer state invalid startTime") ) -// PeerState contains the known state of a peer, including its connection -// and threadsafe access to its PeerRoundState. +// PeerState contains the known state of a peer, including its connection and +// threadsafe access to its PeerRoundState. type PeerState struct { Peer p2p.Peer logger log.Logger @@ -878,12 +878,14 @@ func NewPeerState(peer p2p.Peer) *PeerState { } } +// SetLogger allows to set a logger on the peer state. Returns the peer state +// itself. func (ps *PeerState) SetLogger(logger log.Logger) *PeerState { ps.logger = logger return ps } -// GetRoundState returns an atomic snapshot of the PeerRoundState. +// GetRoundState returns an shallow copy of the PeerRoundState. // There's no point in mutating it since it won't change PeerState. func (ps *PeerState) GetRoundState() *cstypes.PeerRoundState { ps.mtx.Lock() @@ -893,6 +895,14 @@ func (ps *PeerState) GetRoundState() *cstypes.PeerRoundState { return &prs } +// GetRoundStateJSON returns a json of PeerRoundState, marshalled using go-amino. +func (ps *PeerState) GetRoundStateJSON() ([]byte, error) { + ps.mtx.Lock() + defer ps.mtx.Unlock() + + return cdc.MarshalJSON(ps.PeerRoundState) +} + // GetHeight returns an atomic snapshot of the PeerRoundState's height // used by the mempool to ensure peers are caught up before broadcasting new txs func (ps *PeerState) GetHeight() int64 { @@ -1055,7 +1065,7 @@ func (ps *PeerState) ensureCatchupCommitRound(height int64, round int, numValida } } -// EnsureVoteVitArrays ensures the bit-arrays have been allocated for tracking +// EnsureVoteBitArrays ensures the bit-arrays have been allocated for tracking // what votes this peer has received. // NOTE: It's important to make sure that numValidators actually matches // what the node sees as the number of validators for height. diff --git a/consensus/state.go b/consensus/state.go index c6532e369..2e97c4953 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -168,18 +168,23 @@ func (cs *ConsensusState) GetState() sm.State { return cs.state.Copy() } -// GetRoundState returns a copy of the internal consensus state. +// GetRoundState returns a shallow copy of the internal consensus state. func (cs *ConsensusState) GetRoundState() *cstypes.RoundState { cs.mtx.Lock() defer cs.mtx.Unlock() - return cs.getRoundState() -} -func (cs *ConsensusState) getRoundState() *cstypes.RoundState { rs := cs.RoundState // copy return &rs } +// GetRoundStateJSON returns a json of RoundState, marshalled using go-amino. +func (cs *ConsensusState) GetRoundStateJSON() ([]byte, error) { + cs.mtx.Lock() + defer cs.mtx.Unlock() + + return cdc.MarshalJSON(cs.RoundState) +} + // GetValidators returns a copy of the current validators. func (cs *ConsensusState) GetValidators() (int64, []*types.Validator) { cs.mtx.Lock() diff --git a/consensus/types/state.go b/consensus/types/state.go index 8e79f10d2..b18fbd7c0 100644 --- a/consensus/types/state.go +++ b/consensus/types/state.go @@ -52,7 +52,7 @@ func (rs RoundStepType) String() string { //----------------------------------------------------------------------------- // RoundState defines the internal consensus state. -// It is Immutable when returned from ConsensusState.GetRoundState() +// It should be immutable when returned from ConsensusState.GetRoundState(), but it's not. // TODO: Actually, only the top pointer is copied, // so access to field pointers is still racey // NOTE: Not thread safe. Should only be manipulated by functions downstream diff --git a/rpc/core/consensus.go b/rpc/core/consensus.go index 25b67925c..7647aef7d 100644 --- a/rpc/core/consensus.go +++ b/rpc/core/consensus.go @@ -1,8 +1,9 @@ package core import ( + "encoding/json" + cm "github.com/tendermint/tendermint/consensus" - cstypes "github.com/tendermint/tendermint/consensus/types" p2p "github.com/tendermint/tendermint/p2p" ctypes "github.com/tendermint/tendermint/rpc/core/types" sm "github.com/tendermint/tendermint/state" @@ -58,7 +59,7 @@ func Validators(heightPtr *int64) (*ctypes.ResultValidators, error) { return &ctypes.ResultValidators{height, validators.Validators}, nil } -// Dump consensus state. +// DumpConsensusState dumps consensus state. // // ```shell // curl 'localhost:46657/dump_consensus_state' @@ -83,11 +84,18 @@ func Validators(heightPtr *int64) (*ctypes.ResultValidators, error) { // } // ``` func DumpConsensusState() (*ctypes.ResultDumpConsensusState, error) { - peerRoundStates := make(map[p2p.ID]*cstypes.PeerRoundState) + peerRoundStates := make(map[p2p.ID]json.RawMessage) for _, peer := range p2pSwitch.Peers().List() { peerState := peer.Get(types.PeerStateKey).(*cm.PeerState) - peerRoundState := peerState.GetRoundState() + peerRoundState, err := peerState.GetRoundStateJSON() + if err != nil { + return nil, err + } peerRoundStates[peer.ID()] = peerRoundState } - return &ctypes.ResultDumpConsensusState{consensusState.GetRoundState(), peerRoundStates}, nil + roundState, err := consensusState.GetRoundStateJSON() + if err != nil { + return nil, err + } + return &ctypes.ResultDumpConsensusState{roundState, peerRoundStates}, nil } diff --git a/rpc/core/pipe.go b/rpc/core/pipe.go index 1eb00ceeb..0bbead943 100644 --- a/rpc/core/pipe.go +++ b/rpc/core/pipe.go @@ -3,9 +3,8 @@ package core import ( "time" - "github.com/tendermint/go-crypto" + crypto "github.com/tendermint/go-crypto" "github.com/tendermint/tendermint/consensus" - cstypes "github.com/tendermint/tendermint/consensus/types" "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/proxy" sm "github.com/tendermint/tendermint/state" @@ -23,7 +22,7 @@ var subscribeTimeout = 5 * time.Second type Consensus interface { GetState() sm.State GetValidators() (int64, []*types.Validator) - GetRoundState() *cstypes.RoundState + GetRoundStateJSON() ([]byte, error) } type P2P interface { diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index 8a6fff63c..7a31ec5d8 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -1,6 +1,7 @@ package core_types import ( + "encoding/json" "strings" "time" @@ -8,7 +9,6 @@ import ( crypto "github.com/tendermint/go-crypto" cmn "github.com/tendermint/tmlibs/common" - cstypes "github.com/tendermint/tendermint/consensus/types" "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" @@ -109,8 +109,8 @@ type ResultValidators struct { } type ResultDumpConsensusState struct { - RoundState *cstypes.RoundState `json:"round_state"` - PeerRoundStates map[p2p.ID]*cstypes.PeerRoundState `json:"peer_round_states"` + RoundState json.RawMessage `json:"round_state"` + PeerRoundStates map[p2p.ID]json.RawMessage `json:"peer_round_states"` } type ResultBroadcastTx struct { From cca1dd8e3ec50024d4912f655960f811123e0cd0 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Tue, 10 Apr 2018 11:36:31 +0200 Subject: [PATCH 090/143] removed excessive comment Refs https://github.com/tendermint/tendermint/pull/1446#discussion_r180353446 --- consensus/types/state.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/consensus/types/state.go b/consensus/types/state.go index b18fbd7c0..0c3626cc7 100644 --- a/consensus/types/state.go +++ b/consensus/types/state.go @@ -52,9 +52,6 @@ func (rs RoundStepType) String() string { //----------------------------------------------------------------------------- // RoundState defines the internal consensus state. -// It should be immutable when returned from ConsensusState.GetRoundState(), but it's not. -// TODO: Actually, only the top pointer is copied, -// so access to field pointers is still racey // NOTE: Not thread safe. Should only be manipulated by functions downstream // of the cs.receiveRoutine type RoundState struct { From 6a48bd0c88e7a858d6ec44ea68e953bf41913433 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Corbi=C3=A8re?= Date: Tue, 10 Apr 2018 16:03:03 +0200 Subject: [PATCH 091/143] use the tag interface for pubsub. (#1438) * use the tag interface for pubsub. * update tmlibs. * Fix unresolved conflict. --- Gopkg.lock | 6 +++--- Gopkg.toml | 3 ++- types/event_bus.go | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index a634a1ad3..c4d798709 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -270,6 +270,7 @@ version = "v0.7.3" [[projects]] + branch = "develop" name = "github.com/tendermint/tmlibs" packages = [ "autofile", @@ -285,8 +286,7 @@ "pubsub/query", "test" ] - revision = "2e24b64fc121dcdf1cabceab8dc2f7257675483c" - version = "v0.8.1" + revision = "357648b8d63732df77fb1dd0f5ad813800912854" [[projects]] branch = "master" @@ -383,6 +383,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "9b49dde42e615372db85e13d7952cb547b8b1833458a4906f908af988cf0bd4f" + inputs-digest = "794125e6cc5926371340043eed0ce8731a2212367cc6d4b845e4618ef2dd1996" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 47b6460f5..4bbcefa20 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -84,7 +84,8 @@ [[override]] # [[constraint]] name = "github.com/tendermint/tmlibs" - version = "~0.8.1" + # version = "~0.8.1" + branch = "develop" [[constraint]] name = "google.golang.org/grpc" diff --git a/types/event_bus.go b/types/event_bus.go index 7c82d7e45..460a3e294 100644 --- a/types/event_bus.go +++ b/types/event_bus.go @@ -67,7 +67,7 @@ func (b *EventBus) UnsubscribeAll(ctx context.Context, subscriber string) error func (b *EventBus) Publish(eventType string, eventData TMEventData) error { // no explicit deadline for publishing events ctx := context.Background() - b.pubsub.PublishWithTags(ctx, eventData, map[string]interface{}{EventTypeKey: eventType}) + b.pubsub.PublishWithTags(ctx, eventData, tmpubsub.NewTagMap(map[string]interface{}{EventTypeKey: eventType})) return nil } @@ -114,7 +114,7 @@ func (b *EventBus) PublishEventTx(event EventDataTx) error { logIfTagExists(TxHeightKey, tags, b.Logger) tags[TxHeightKey] = event.Height - b.pubsub.PublishWithTags(ctx, event, tags) + b.pubsub.PublishWithTags(ctx, event, tmpubsub.NewTagMap(tags)) return nil } From 384b3ea0652e30cc0b650d754611eae66243d80a Mon Sep 17 00:00:00 2001 From: suyuhuang Date: Tue, 10 Apr 2018 22:03:51 +0800 Subject: [PATCH 092/143] improve `show_node_id` (#1433) * fix show_node_id * make LoadNodeKey public * make LoadNodeKey public * remove if * remove if --- cmd/tendermint/commands/show_node_id.go | 5 ++++- p2p/key.go | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cmd/tendermint/commands/show_node_id.go b/cmd/tendermint/commands/show_node_id.go index 92af49417..1d94933ef 100644 --- a/cmd/tendermint/commands/show_node_id.go +++ b/cmd/tendermint/commands/show_node_id.go @@ -6,6 +6,7 @@ import ( "github.com/spf13/cobra" "github.com/tendermint/tendermint/p2p" + ) // ShowNodeIDCmd dumps node's ID to the standard output. @@ -16,10 +17,12 @@ var ShowNodeIDCmd = &cobra.Command{ } func showNodeID(cmd *cobra.Command, args []string) error { - nodeKey, err := p2p.LoadOrGenNodeKey(config.NodeKeyFile()) + + nodeKey, err := p2p.LoadNodeKey(config.NodeKeyFile()) if err != nil { return err } fmt.Println(nodeKey.ID()) + return nil } diff --git a/p2p/key.go b/p2p/key.go index 9b8087cd4..73103ebd4 100644 --- a/p2p/key.go +++ b/p2p/key.go @@ -47,7 +47,7 @@ func PubKeyToID(pubKey crypto.PubKey) ID { // If the file does not exist, it generates and saves a new NodeKey. func LoadOrGenNodeKey(filePath string) (*NodeKey, error) { if cmn.FileExists(filePath) { - nodeKey, err := loadNodeKey(filePath) + nodeKey, err := LoadNodeKey(filePath) if err != nil { return nil, err } @@ -56,7 +56,7 @@ func LoadOrGenNodeKey(filePath string) (*NodeKey, error) { return genNodeKey(filePath) } -func loadNodeKey(filePath string) (*NodeKey, error) { +func LoadNodeKey(filePath string) (*NodeKey, error) { jsonBytes, err := ioutil.ReadFile(filePath) if err != nil { return nil, err From e88f74bb9bb9edb9c311f256037fcca217b45ab6 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Mon, 9 Apr 2018 16:32:43 +0200 Subject: [PATCH 093/143] remove wal_light setting Closes #1428 --- CHANGELOG.md | 2 ++ config/config.go | 8 +++----- config/toml.go | 1 - consensus/replay_test.go | 4 ++-- consensus/state.go | 2 +- consensus/wal.go | 15 ++------------- consensus/wal_test.go | 2 +- docs/specification/configuration.rst | 1 - 8 files changed, 11 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f54c9805d..e96dd4cdc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,8 @@ See github.com/tendermint/go-amino for details on the new format. See `scripts/wire2amino.go` for a tool to upgrade genesis/priv_validator/node_key JSON files. +- [config] removed `wal_light` setting + ## 0.18.0 (April 6th, 2018) BREAKING: diff --git a/config/config.go b/config/config.go index e83e9e51d..b76f5ed19 100644 --- a/config/config.go +++ b/config/config.go @@ -367,10 +367,9 @@ func (cfg *MempoolConfig) WalDir() string { // ConsensusConfig defines the confuguration for the Tendermint consensus service, // including timeouts and details about the WAL and the block structure. type ConsensusConfig struct { - RootDir string `mapstructure:"home"` - WalPath string `mapstructure:"wal_file"` - WalLight bool `mapstructure:"wal_light"` - walFile string // overrides WalPath if set + RootDir string `mapstructure:"home"` + WalPath string `mapstructure:"wal_file"` + walFile string // overrides WalPath if set // All timeouts are in milliseconds TimeoutPropose int `mapstructure:"timeout_propose"` @@ -401,7 +400,6 @@ type ConsensusConfig struct { func DefaultConsensusConfig() *ConsensusConfig { return &ConsensusConfig{ WalPath: filepath.Join(defaultDataDir, "cs.wal", "wal"), - WalLight: false, TimeoutPropose: 3000, TimeoutProposeDelta: 500, TimeoutPrevote: 1000, diff --git a/config/toml.go b/config/toml.go index f10c08ab3..a084cc592 100644 --- a/config/toml.go +++ b/config/toml.go @@ -178,7 +178,6 @@ wal_dir = "{{ .Mempool.WalPath }}" [consensus] wal_file = "{{ .Consensus.WalPath }}" -wal_light = {{ .Consensus.WalLight }} # All timeouts are in milliseconds timeout_propose = {{ .Consensus.TimeoutPropose }} diff --git a/consensus/replay_test.go b/consensus/replay_test.go index 75f208bb6..c706cef0c 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -17,7 +17,7 @@ import ( "github.com/tendermint/abci/example/kvstore" abci "github.com/tendermint/abci/types" - "github.com/tendermint/go-crypto" + crypto "github.com/tendermint/go-crypto" auto "github.com/tendermint/tmlibs/autofile" cmn "github.com/tendermint/tmlibs/common" dbm "github.com/tendermint/tmlibs/db" @@ -327,7 +327,7 @@ func testHandshakeReplay(t *testing.T, nBlocks int, mode uint) { privVal := pvm.LoadFilePV(config.PrivValidatorFile()) - wal, err := NewWAL(walFile, false) + wal, err := NewWAL(walFile) if err != nil { t.Fatal(err) } diff --git a/consensus/state.go b/consensus/state.go index 2e97c4953..bd069f70c 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -293,7 +293,7 @@ func (cs *ConsensusState) Wait() { // OpenWAL opens a file to log all consensus messages and timeouts for deterministic accountability func (cs *ConsensusState) OpenWAL(walFile string) (WAL, error) { - wal, err := NewWAL(walFile, cs.config.WalLight) + wal, err := NewWAL(walFile) if err != nil { cs.Logger.Error("Failed to open WAL for consensus state", "wal", walFile, "err", err) return nil, err diff --git a/consensus/wal.go b/consensus/wal.go index 694e2516e..d22c3ea1f 100644 --- a/consensus/wal.go +++ b/consensus/wal.go @@ -10,7 +10,7 @@ import ( "github.com/pkg/errors" - "github.com/tendermint/go-amino" + amino "github.com/tendermint/go-amino" "github.com/tendermint/tendermint/types" auto "github.com/tendermint/tmlibs/autofile" cmn "github.com/tendermint/tmlibs/common" @@ -67,12 +67,11 @@ type baseWAL struct { cmn.BaseService group *auto.Group - light bool // ignore block parts enc *WALEncoder } -func NewWAL(walFile string, light bool) (*baseWAL, error) { +func NewWAL(walFile string) (*baseWAL, error) { err := cmn.EnsureDir(filepath.Dir(walFile), 0700) if err != nil { return nil, errors.Wrap(err, "failed to ensure WAL directory is in place") @@ -84,7 +83,6 @@ func NewWAL(walFile string, light bool) (*baseWAL, error) { } wal := &baseWAL{ group: group, - light: light, enc: NewWALEncoder(group), } wal.BaseService = *cmn.NewBaseService(nil, "baseWAL", wal) @@ -117,15 +115,6 @@ func (wal *baseWAL) Save(msg WALMessage) { return } - if wal.light { - // in light mode we only write new steps, timeouts, and our own votes (no proposals, block parts) - if mi, ok := msg.(msgInfo); ok { - if mi.PeerID != "" { - return - } - } - } - // Write the wal message if err := wal.enc.Encode(&TimedWALMessage{time.Now(), msg}); err != nil { cmn.PanicQ(cmn.Fmt("Error writing msg to consensus wal: %v \n\nMessage: %v", err, msg)) diff --git a/consensus/wal_test.go b/consensus/wal_test.go index 45af36e12..eebbc85a2 100644 --- a/consensus/wal_test.go +++ b/consensus/wal_test.go @@ -47,7 +47,7 @@ func TestWALSearchForEndHeight(t *testing.T) { } walFile := tempWALWithData(walBody) - wal, err := NewWAL(walFile, false) + wal, err := NewWAL(walFile) if err != nil { t.Fatal(err) } diff --git a/docs/specification/configuration.rst b/docs/specification/configuration.rst index 2a9ad794e..6b52dbd1e 100644 --- a/docs/specification/configuration.rst +++ b/docs/specification/configuration.rst @@ -140,7 +140,6 @@ like the file below, however, double check by inspecting the [consensus] wal_file = "data/cs.wal/wal" - wal_light = false # All timeouts are in milliseconds timeout_propose = 3000 From 7c22e47629909f48d2050a711a74d964af58e5c3 Mon Sep 17 00:00:00 2001 From: Vladislav Dmitriyev Date: Wed, 11 Apr 2018 11:11:11 +0300 Subject: [PATCH 094/143] Replaced NodeInfo's pubkey to ID (#1443) * Replaced NodeInfo PubKey to NodeID * Fixed tests and replaced NodeID with ID * Removed unnecessary method ID() * Fixed codec_test.go * Fixed codec_test.go * Removed unnecessary bracket * Fixed all tests * Fixed peer_set_test.go * Fixed peer_test.go * Fixed common_test.go * Fixed common_test.go * Renamed node_id to id * Removed peer.ID() from RPC net.go * Replaced NodeInfo pubKey to ID * Fixed codec_test.go * Fixed peer_set_test.go * Fix pex_reactor_test.go * Refactored code for privateKey initiali * Fixed peer_set_test.go * Fixed test.proto and removed orphan string in codec_test.go * Fixed pointer to a string * generate node_key when running tendermint init * [docs] prefix IPs with node IDs Refs #1429 * gen_node_key cmd * [docs/specification/secure-p2p] add a note about config * fix data race Closes #1442 ``` WARNING: DATA RACE Write at 0x00c4209de7c8 by goroutine 23: github.com/tendermint/tendermint/types.(*Block).fillHeader() /home/vagrant/go/src/github.com/tendermint/tendermint/types/block.go:88 +0x157 github.com/tendermint/tendermint/types.(*Block).Hash() /home/vagrant/go/src/github.com/tendermint/tendermint/types/block.go:104 +0x121 github.com/tendermint/tendermint/types.(*Block).HashesTo() /home/vagrant/go/src/github.com/tendermint/tendermint/types/block.go:135 +0x4f github.com/tendermint/tendermint/consensus.(*ConsensusState).enterPrecommit() /home/vagrant/go/src/github.com/tendermint/tendermint/consensus/state.go:1037 +0x182d github.com/tendermint/tendermint/consensus.(*ConsensusState).addVote() /home/vagrant/go/src/github.com/tendermint/tendermint/consensus/state.go:1425 +0x1a6c github.com/tendermint/tendermint/consensus.(*ConsensusState).tryAddVote() /home/vagrant/go/src/github.com/tendermint/tendermint/consensus/state.go:1318 +0x77 github.com/tendermint/tendermint/consensus.(*ConsensusState).handleMsg() /home/vagrant/go/src/github.com/tendermint/tendermint/consensus/state.go:581 +0x7a9 github.com/tendermint/tendermint/consensus.(*ConsensusState).receiveRoutine() /home/vagrant/go/src/github.com/tendermint/tendermint/consensus/state.go:539 +0x6c3 Previous read at 0x00c4209de7c8 by goroutine 47: github.com/tendermint/tendermint/vendor/github.com/tendermint/tmlibs/common.(*HexBytes).MarshalJSON() :1 +0x52 github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino.invokeMarshalJSON() /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino/json-encode.go:433 +0x88 github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino.(*Codec)._encodeReflectJSON() /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino/json-encode.go:82 +0x8d2 github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino.(*Codec).encodeReflectJSON() /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino/json-encode.go:50 +0x10e github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino.(*Codec).encodeReflectJSONStruct() /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino/json-encode.go:348 +0x539 github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino.(*Codec)._encodeReflectJSON() /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino/json-encode.go:119 +0x83f github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino.(*Codec).encodeReflectJSON() /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino/json-encode.go:50 +0x10e github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino.(*Codec).encodeReflectJSONStruct() /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino/json-encode.go:348 +0x539 github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino.(*Codec)._encodeReflectJSON() /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino/json-encode.go:119 +0x83f github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino.(*Codec).encodeReflectJSON() /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino/json-encode.go:50 +0x10e github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino.(*Codec).encodeReflectJSONStruct() /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino/json-encode.go:348 +0x539 github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino.(*Codec)._encodeReflectJSON() /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino/json-encode.go:119 +0x83f github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino.(*Codec).encodeReflectJSON() /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino/json-encode.go:50 +0x10e github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino.(*Codec).encodeReflectJSONStruct() /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino/json-encode.go:348 +0x539 github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino.(*Codec)._encodeReflectJSON() /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino/json-encode.go:119 +0x83f github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino.(*Codec).encodeReflectJSON() /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino/json-encode.go:50 +0x10e github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino.(*Codec).MarshalJSON() /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/go-amino/amino.go:296 +0x182 github.com/tendermint/tendermint/rpc/lib/types.NewRPCSuccessResponse() /home/vagrant/go/src/github.com/tendermint/tendermint/rpc/lib/types/types.go:100 +0x12c github.com/tendermint/tendermint/rpc/lib/server.makeJSONRPCHandler.func1() /home/vagrant/go/src/github.com/tendermint/tendermint/rpc/lib/server/handlers.go:152 +0xab7 net/http.HandlerFunc.ServeHTTP() /usr/lib/go-1.9/src/net/http/server.go:1918 +0x51 net/http.(*ServeMux).ServeHTTP() /usr/lib/go-1.9/src/net/http/server.go:2254 +0xa2 github.com/tendermint/tendermint/rpc/lib/server.RecoverAndLogHandler.func1() /home/vagrant/go/src/github.com/tendermint/tendermint/rpc/lib/server/http_server.go:138 +0x4fa net/http.HandlerFunc.ServeHTTP() /usr/lib/go-1.9/src/net/http/server.go:1918 +0x51 net/http.serverHandler.ServeHTTP() /usr/lib/go-1.9/src/net/http/server.go:2619 +0xbc net/http.(*conn).serve() /usr/lib/go-1.9/src/net/http/server.go:1801 +0x83b Goroutine 23 (running) created at: github.com/tendermint/tendermint/consensus.(*ConsensusState).OnStart() /home/vagrant/go/src/github.com/tendermint/tendermint/consensus/state.go:250 +0x35b github.com/tendermint/tendermint/vendor/github.com/tendermint/tmlibs/common.(*BaseService).Start() /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/tmlibs/common/service.go:130 +0x5fc github.com/tendermint/tendermint/consensus.(*ConsensusReactor).OnStart() /home/vagrant/go/src/github.com/tendermint/tendermint/consensus/reactor.go:69 +0x1b4 github.com/tendermint/tendermint/vendor/github.com/tendermint/tmlibs/common.(*BaseService).Start() /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/tmlibs/common/service.go:130 +0x5fc github.com/tendermint/tendermint/consensus.(*ConsensusReactor).Start() :1 +0x43 github.com/tendermint/tendermint/p2p.(*Switch).OnStart() /home/vagrant/go/src/github.com/tendermint/tendermint/p2p/switch.go:177 +0x124 github.com/tendermint/tendermint/vendor/github.com/tendermint/tmlibs/common.(*BaseService).Start() /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/tmlibs/common/service.go:130 +0x5fc github.com/tendermint/tendermint/node.(*Node).OnStart() /home/vagrant/go/src/github.com/tendermint/tendermint/node/node.go:416 +0xa1b github.com/tendermint/tendermint/vendor/github.com/tendermint/tmlibs/common.(*BaseService).Start() /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/tmlibs/common/service.go:130 +0x5fc github.com/tendermint/tendermint/rpc/test.StartTendermint() /home/vagrant/go/src/github.com/tendermint/tendermint/rpc/test/helpers.go:100 +0x5b github.com/tendermint/tendermint/rpc/client_test.TestMain() /home/vagrant/go/src/github.com/tendermint/tendermint/rpc/client/main_test.go:17 +0x4c main.main() github.com/tendermint/tendermint/rpc/client/_test/_testmain.go:76 +0x1cd Goroutine 47 (running) created at: net/http.(*Server).Serve() /usr/lib/go-1.9/src/net/http/server.go:2720 +0x37c net/http.Serve() /usr/lib/go-1.9/src/net/http/server.go:2323 +0xe2 github.com/tendermint/tendermint/rpc/lib/server.StartHTTPServer.func1() /home/vagrant/go/src/github.com/tendermint/tendermint/rpc/lib/server/http_server.go:35 +0xb3 ``` * removed excessive comment Refs https://github.com/tendermint/tendermint/pull/1446#discussion_r180353446 * use the tag interface for pubsub. (#1438) * use the tag interface for pubsub. * update tmlibs. * Fix unresolved conflict. * improve `show_node_id` (#1433) * fix show_node_id * make LoadNodeKey public * make LoadNodeKey public * remove if * remove if --- benchmarks/codec_test.go | 35 +- benchmarks/proto/test.pb.go | 942 ++++++++++++------------ benchmarks/proto/test.proto | 6 +- consensus/common_test.go | 2 +- docs/how-to-read-logs.rst | 2 +- docs/specification/new-spec/p2p/peer.md | 4 +- node/node.go | 6 +- p2p/node_info.go | 18 +- p2p/peer.go | 2 +- p2p/peer_set_test.go | 4 +- p2p/peer_test.go | 4 +- p2p/pex/pex_reactor_test.go | 6 +- p2p/switch.go | 2 +- p2p/switch_test.go | 4 +- p2p/test_util.go | 4 +- rpc/core/net.go | 1 - rpc/core/types/responses.go | 1 - 17 files changed, 526 insertions(+), 517 deletions(-) diff --git a/benchmarks/codec_test.go b/benchmarks/codec_test.go index 83e25e212..2f24b776d 100644 --- a/benchmarks/codec_test.go +++ b/benchmarks/codec_test.go @@ -16,17 +16,17 @@ func BenchmarkEncodeStatusWire(b *testing.B) { b.StopTimer() cdc := amino.NewCodec() ctypes.RegisterAmino(cdc) - pubKey := crypto.GenPrivKeyEd25519().PubKey() + nodeKey := p2p.NodeKey{PrivKey: crypto.GenPrivKeyEd25519()} status := &ctypes.ResultStatus{ NodeInfo: p2p.NodeInfo{ - PubKey: pubKey, + ID: nodeKey.ID(), Moniker: "SOMENAME", Network: "SOMENAME", ListenAddr: "SOMEADDR", Version: "SOMEVER", Other: []string{"SOMESTRING", "OTHERSTRING"}, }, - PubKey: pubKey, + PubKey: nodeKey.PubKey(), LatestBlockHash: []byte("SOMEBYTES"), LatestBlockHeight: 123, LatestBlockTime: time.Unix(0, 1234), @@ -48,9 +48,9 @@ func BenchmarkEncodeNodeInfoWire(b *testing.B) { b.StopTimer() cdc := amino.NewCodec() ctypes.RegisterAmino(cdc) - pubKey := crypto.GenPrivKeyEd25519().PubKey() + nodeKey := p2p.NodeKey{PrivKey: crypto.GenPrivKeyEd25519()} nodeInfo := p2p.NodeInfo{ - PubKey: pubKey, + ID: nodeKey.ID(), Moniker: "SOMENAME", Network: "SOMENAME", ListenAddr: "SOMEADDR", @@ -73,9 +73,9 @@ func BenchmarkEncodeNodeInfoBinary(b *testing.B) { b.StopTimer() cdc := amino.NewCodec() ctypes.RegisterAmino(cdc) - pubKey := crypto.GenPrivKeyEd25519().PubKey() + nodeKey := p2p.NodeKey{PrivKey: crypto.GenPrivKeyEd25519()} nodeInfo := p2p.NodeInfo{ - PubKey: pubKey, + ID: nodeKey.ID(), Moniker: "SOMENAME", Network: "SOMENAME", ListenAddr: "SOMEADDR", @@ -94,15 +94,20 @@ func BenchmarkEncodeNodeInfoBinary(b *testing.B) { func BenchmarkEncodeNodeInfoProto(b *testing.B) { b.StopTimer() - pubKey := crypto.GenPrivKeyEd25519().PubKey().(crypto.PubKeyEd25519) - pubKey2 := &proto.PubKey{Ed25519: &proto.PubKeyEd25519{Bytes: pubKey[:]}} + nodeKey := p2p.NodeKey{PrivKey: crypto.GenPrivKeyEd25519()} + nodeID := string(nodeKey.ID()) + someName := "SOMENAME" + someAddr := "SOMEADDR" + someVer := "SOMEVER" + someString := "SOMESTRING" + otherString := "OTHERSTRING" nodeInfo := proto.NodeInfo{ - PubKey: pubKey2, - Moniker: "SOMENAME", - Network: "SOMENAME", - ListenAddr: "SOMEADDR", - Version: "SOMEVER", - Other: []string{"SOMESTRING", "OTHERSTRING"}, + Id: &proto.ID{Id: &nodeID}, + Moniker: &someName, + Network: &someName, + ListenAddr: &someAddr, + Version: &someVer, + Other: []string{someString, otherString}, } b.StartTimer() diff --git a/benchmarks/proto/test.pb.go b/benchmarks/proto/test.pb.go index dc21a2a82..d430eeb08 100644 --- a/benchmarks/proto/test.pb.go +++ b/benchmarks/proto/test.pb.go @@ -1,6 +1,5 @@ -// Code generated by protoc-gen-gogo. +// Code generated by protoc-gen-gogo. DO NOT EDIT. // source: test.proto -// DO NOT EDIT! /* Package test is a generated protocol buffer package. @@ -11,6 +10,7 @@ It has these top-level messages: ResultStatus NodeInfo + ID PubKey PubKeyEd25519 */ @@ -20,11 +20,6 @@ import proto "github.com/gogo/protobuf/proto" import fmt "fmt" import math "math" -import bytes "bytes" - -import strings "strings" -import github_com_gogo_protobuf_proto "github.com/gogo/protobuf/proto" - import io "io" // Reference imports to suppress errors if they are not otherwise used. @@ -32,16 +27,25 @@ var _ = proto.Marshal var _ = fmt.Errorf var _ = math.Inf +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package + type ResultStatus struct { NodeInfo *NodeInfo `protobuf:"bytes,1,opt,name=nodeInfo" json:"nodeInfo,omitempty"` PubKey *PubKey `protobuf:"bytes,2,req,name=pubKey" json:"pubKey,omitempty"` - LatestBlockHash []byte `protobuf:"bytes,3,req,name=latestBlockHash" json:"latestBlockHash"` - LatestBlockHeight int64 `protobuf:"varint,4,req,name=latestBlockHeight" json:"latestBlockHeight"` - LatestBlocktime int64 `protobuf:"varint,5,req,name=latestBlocktime" json:"latestBlocktime"` + LatestBlockHash []byte `protobuf:"bytes,3,req,name=latestBlockHash" json:"latestBlockHash,omitempty"` + LatestBlockHeight *int64 `protobuf:"varint,4,req,name=latestBlockHeight" json:"latestBlockHeight,omitempty"` + LatestBlocktime *int64 `protobuf:"varint,5,req,name=latestBlocktime" json:"latestBlocktime,omitempty"` + XXX_unrecognized []byte `json:"-"` } -func (m *ResultStatus) Reset() { *m = ResultStatus{} } -func (*ResultStatus) ProtoMessage() {} +func (m *ResultStatus) Reset() { *m = ResultStatus{} } +func (m *ResultStatus) String() string { return proto.CompactTextString(m) } +func (*ResultStatus) ProtoMessage() {} +func (*ResultStatus) Descriptor() ([]byte, []int) { return fileDescriptorTest, []int{0} } func (m *ResultStatus) GetNodeInfo() *NodeInfo { if m != nil { @@ -65,70 +69,73 @@ func (m *ResultStatus) GetLatestBlockHash() []byte { } func (m *ResultStatus) GetLatestBlockHeight() int64 { - if m != nil { - return m.LatestBlockHeight + if m != nil && m.LatestBlockHeight != nil { + return *m.LatestBlockHeight } return 0 } func (m *ResultStatus) GetLatestBlocktime() int64 { - if m != nil { - return m.LatestBlocktime + if m != nil && m.LatestBlocktime != nil { + return *m.LatestBlocktime } return 0 } type NodeInfo struct { - PubKey *PubKey `protobuf:"bytes,1,req,name=pubKey" json:"pubKey,omitempty"` - Moniker string `protobuf:"bytes,2,req,name=moniker" json:"moniker"` - Network string `protobuf:"bytes,3,req,name=network" json:"network"` - RemoteAddr string `protobuf:"bytes,4,req,name=remoteAddr" json:"remoteAddr"` - ListenAddr string `protobuf:"bytes,5,req,name=listenAddr" json:"listenAddr"` - Version string `protobuf:"bytes,6,req,name=version" json:"version"` - Other []string `protobuf:"bytes,7,rep,name=other" json:"other,omitempty"` + Id *ID `protobuf:"bytes,1,req,name=id" json:"id,omitempty"` + Moniker *string `protobuf:"bytes,2,req,name=moniker" json:"moniker,omitempty"` + Network *string `protobuf:"bytes,3,req,name=network" json:"network,omitempty"` + RemoteAddr *string `protobuf:"bytes,4,req,name=remoteAddr" json:"remoteAddr,omitempty"` + ListenAddr *string `protobuf:"bytes,5,req,name=listenAddr" json:"listenAddr,omitempty"` + Version *string `protobuf:"bytes,6,req,name=version" json:"version,omitempty"` + Other []string `protobuf:"bytes,7,rep,name=other" json:"other,omitempty"` + XXX_unrecognized []byte `json:"-"` } -func (m *NodeInfo) Reset() { *m = NodeInfo{} } -func (*NodeInfo) ProtoMessage() {} +func (m *NodeInfo) Reset() { *m = NodeInfo{} } +func (m *NodeInfo) String() string { return proto.CompactTextString(m) } +func (*NodeInfo) ProtoMessage() {} +func (*NodeInfo) Descriptor() ([]byte, []int) { return fileDescriptorTest, []int{1} } -func (m *NodeInfo) GetPubKey() *PubKey { +func (m *NodeInfo) GetId() *ID { if m != nil { - return m.PubKey + return m.Id } return nil } func (m *NodeInfo) GetMoniker() string { - if m != nil { - return m.Moniker + if m != nil && m.Moniker != nil { + return *m.Moniker } return "" } func (m *NodeInfo) GetNetwork() string { - if m != nil { - return m.Network + if m != nil && m.Network != nil { + return *m.Network } return "" } func (m *NodeInfo) GetRemoteAddr() string { - if m != nil { - return m.RemoteAddr + if m != nil && m.RemoteAddr != nil { + return *m.RemoteAddr } return "" } func (m *NodeInfo) GetListenAddr() string { - if m != nil { - return m.ListenAddr + if m != nil && m.ListenAddr != nil { + return *m.ListenAddr } return "" } func (m *NodeInfo) GetVersion() string { - if m != nil { - return m.Version + if m != nil && m.Version != nil { + return *m.Version } return "" } @@ -140,12 +147,32 @@ func (m *NodeInfo) GetOther() []string { return nil } -type PubKey struct { - Ed25519 *PubKeyEd25519 `protobuf:"bytes,1,opt,name=ed25519" json:"ed25519,omitempty"` +type ID struct { + Id *string `protobuf:"bytes,1,req,name=id" json:"id,omitempty"` + XXX_unrecognized []byte `json:"-"` } -func (m *PubKey) Reset() { *m = PubKey{} } -func (*PubKey) ProtoMessage() {} +func (m *ID) Reset() { *m = ID{} } +func (m *ID) String() string { return proto.CompactTextString(m) } +func (*ID) ProtoMessage() {} +func (*ID) Descriptor() ([]byte, []int) { return fileDescriptorTest, []int{2} } + +func (m *ID) GetId() string { + if m != nil && m.Id != nil { + return *m.Id + } + return "" +} + +type PubKey struct { + Ed25519 *PubKeyEd25519 `protobuf:"bytes,1,opt,name=ed25519" json:"ed25519,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *PubKey) Reset() { *m = PubKey{} } +func (m *PubKey) String() string { return proto.CompactTextString(m) } +func (*PubKey) ProtoMessage() {} +func (*PubKey) Descriptor() ([]byte, []int) { return fileDescriptorTest, []int{3} } func (m *PubKey) GetEd25519() *PubKeyEd25519 { if m != nil { @@ -155,11 +182,14 @@ func (m *PubKey) GetEd25519() *PubKeyEd25519 { } type PubKeyEd25519 struct { - Bytes []byte `protobuf:"bytes,1,req,name=bytes" json:"bytes"` + Bytes []byte `protobuf:"bytes,1,req,name=bytes" json:"bytes,omitempty"` + XXX_unrecognized []byte `json:"-"` } -func (m *PubKeyEd25519) Reset() { *m = PubKeyEd25519{} } -func (*PubKeyEd25519) ProtoMessage() {} +func (m *PubKeyEd25519) Reset() { *m = PubKeyEd25519{} } +func (m *PubKeyEd25519) String() string { return proto.CompactTextString(m) } +func (*PubKeyEd25519) ProtoMessage() {} +func (*PubKeyEd25519) Descriptor() ([]byte, []int) { return fileDescriptorTest, []int{4} } func (m *PubKeyEd25519) GetBytes() []byte { if m != nil { @@ -171,400 +201,259 @@ func (m *PubKeyEd25519) GetBytes() []byte { func init() { proto.RegisterType((*ResultStatus)(nil), "ResultStatus") proto.RegisterType((*NodeInfo)(nil), "NodeInfo") + proto.RegisterType((*ID)(nil), "ID") proto.RegisterType((*PubKey)(nil), "PubKey") proto.RegisterType((*PubKeyEd25519)(nil), "PubKeyEd25519") } -func (this *ResultStatus) Equal(that interface{}) bool { - if that == nil { - if this == nil { - return true - } - return false - } - - that1, ok := that.(*ResultStatus) - if !ok { - that2, ok := that.(ResultStatus) - if ok { - that1 = &that2 - } else { - return false - } - } - if that1 == nil { - if this == nil { - return true - } - return false - } else if this == nil { - return false - } - if !this.NodeInfo.Equal(that1.NodeInfo) { - return false - } - if !this.PubKey.Equal(that1.PubKey) { - return false - } - if !bytes.Equal(this.LatestBlockHash, that1.LatestBlockHash) { - return false - } - if this.LatestBlockHeight != that1.LatestBlockHeight { - return false - } - if this.LatestBlocktime != that1.LatestBlocktime { - return false - } - return true -} -func (this *NodeInfo) Equal(that interface{}) bool { - if that == nil { - if this == nil { - return true - } - return false - } - - that1, ok := that.(*NodeInfo) - if !ok { - that2, ok := that.(NodeInfo) - if ok { - that1 = &that2 - } else { - return false - } - } - if that1 == nil { - if this == nil { - return true - } - return false - } else if this == nil { - return false - } - if !this.PubKey.Equal(that1.PubKey) { - return false - } - if this.Moniker != that1.Moniker { - return false - } - if this.Network != that1.Network { - return false - } - if this.RemoteAddr != that1.RemoteAddr { - return false - } - if this.ListenAddr != that1.ListenAddr { - return false - } - if this.Version != that1.Version { - return false - } - if len(this.Other) != len(that1.Other) { - return false - } - for i := range this.Other { - if this.Other[i] != that1.Other[i] { - return false - } - } - return true -} -func (this *PubKey) Equal(that interface{}) bool { - if that == nil { - if this == nil { - return true - } - return false - } - - that1, ok := that.(*PubKey) - if !ok { - that2, ok := that.(PubKey) - if ok { - that1 = &that2 - } else { - return false - } - } - if that1 == nil { - if this == nil { - return true - } - return false - } else if this == nil { - return false - } - if !this.Ed25519.Equal(that1.Ed25519) { - return false - } - return true -} -func (this *PubKeyEd25519) Equal(that interface{}) bool { - if that == nil { - if this == nil { - return true - } - return false - } - - that1, ok := that.(*PubKeyEd25519) - if !ok { - that2, ok := that.(PubKeyEd25519) - if ok { - that1 = &that2 - } else { - return false - } - } - if that1 == nil { - if this == nil { - return true - } - return false - } else if this == nil { - return false - } - if !bytes.Equal(this.Bytes, that1.Bytes) { - return false - } - return true -} -func (this *ResultStatus) GoString() string { - if this == nil { - return "nil" - } - s := make([]string, 0, 9) - s = append(s, "&test.ResultStatus{") - if this.NodeInfo != nil { - s = append(s, "NodeInfo: "+fmt.Sprintf("%#v", this.NodeInfo)+",\n") - } - if this.PubKey != nil { - s = append(s, "PubKey: "+fmt.Sprintf("%#v", this.PubKey)+",\n") - } - s = append(s, "LatestBlockHash: "+fmt.Sprintf("%#v", this.LatestBlockHash)+",\n") - s = append(s, "LatestBlockHeight: "+fmt.Sprintf("%#v", this.LatestBlockHeight)+",\n") - s = append(s, "LatestBlocktime: "+fmt.Sprintf("%#v", this.LatestBlocktime)+",\n") - s = append(s, "}") - return strings.Join(s, "") -} -func (this *NodeInfo) GoString() string { - if this == nil { - return "nil" - } - s := make([]string, 0, 11) - s = append(s, "&test.NodeInfo{") - if this.PubKey != nil { - s = append(s, "PubKey: "+fmt.Sprintf("%#v", this.PubKey)+",\n") - } - s = append(s, "Moniker: "+fmt.Sprintf("%#v", this.Moniker)+",\n") - s = append(s, "Network: "+fmt.Sprintf("%#v", this.Network)+",\n") - s = append(s, "RemoteAddr: "+fmt.Sprintf("%#v", this.RemoteAddr)+",\n") - s = append(s, "ListenAddr: "+fmt.Sprintf("%#v", this.ListenAddr)+",\n") - s = append(s, "Version: "+fmt.Sprintf("%#v", this.Version)+",\n") - if this.Other != nil { - s = append(s, "Other: "+fmt.Sprintf("%#v", this.Other)+",\n") - } - s = append(s, "}") - return strings.Join(s, "") -} -func (this *PubKey) GoString() string { - if this == nil { - return "nil" - } - s := make([]string, 0, 5) - s = append(s, "&test.PubKey{") - if this.Ed25519 != nil { - s = append(s, "Ed25519: "+fmt.Sprintf("%#v", this.Ed25519)+",\n") - } - s = append(s, "}") - return strings.Join(s, "") -} -func (this *PubKeyEd25519) GoString() string { - if this == nil { - return "nil" - } - s := make([]string, 0, 5) - s = append(s, "&test.PubKeyEd25519{") - s = append(s, "Bytes: "+fmt.Sprintf("%#v", this.Bytes)+",\n") - s = append(s, "}") - return strings.Join(s, "") -} -func (m *ResultStatus) Marshal() (data []byte, err error) { +func (m *ResultStatus) Marshal() (dAtA []byte, err error) { size := m.Size() - data = make([]byte, size) - n, err := m.MarshalTo(data) + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } - return data[:n], nil + return dAtA[:n], nil } -func (m *ResultStatus) MarshalTo(data []byte) (int, error) { +func (m *ResultStatus) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int _ = l if m.NodeInfo != nil { - data[i] = 0xa + dAtA[i] = 0xa i++ - i = encodeVarintTest(data, i, uint64(m.NodeInfo.Size())) - n1, err := m.NodeInfo.MarshalTo(data[i:]) + i = encodeVarintTest(dAtA, i, uint64(m.NodeInfo.Size())) + n1, err := m.NodeInfo.MarshalTo(dAtA[i:]) if err != nil { return 0, err } i += n1 } if m.PubKey == nil { - return 0, github_com_gogo_protobuf_proto.NewRequiredNotSetError("pubKey") + return 0, proto.NewRequiredNotSetError("pubKey") } else { - data[i] = 0x12 + dAtA[i] = 0x12 i++ - i = encodeVarintTest(data, i, uint64(m.PubKey.Size())) - n2, err := m.PubKey.MarshalTo(data[i:]) + i = encodeVarintTest(dAtA, i, uint64(m.PubKey.Size())) + n2, err := m.PubKey.MarshalTo(dAtA[i:]) if err != nil { return 0, err } i += n2 } - if m.LatestBlockHash != nil { - data[i] = 0x1a + if m.LatestBlockHash == nil { + return 0, proto.NewRequiredNotSetError("latestBlockHash") + } else { + dAtA[i] = 0x1a i++ - i = encodeVarintTest(data, i, uint64(len(m.LatestBlockHash))) - i += copy(data[i:], m.LatestBlockHash) + i = encodeVarintTest(dAtA, i, uint64(len(m.LatestBlockHash))) + i += copy(dAtA[i:], m.LatestBlockHash) + } + if m.LatestBlockHeight == nil { + return 0, proto.NewRequiredNotSetError("latestBlockHeight") + } else { + dAtA[i] = 0x20 + i++ + i = encodeVarintTest(dAtA, i, uint64(*m.LatestBlockHeight)) + } + if m.LatestBlocktime == nil { + return 0, proto.NewRequiredNotSetError("latestBlocktime") + } else { + dAtA[i] = 0x28 + i++ + i = encodeVarintTest(dAtA, i, uint64(*m.LatestBlocktime)) + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) } - data[i] = 0x20 - i++ - i = encodeVarintTest(data, i, uint64(m.LatestBlockHeight)) - data[i] = 0x28 - i++ - i = encodeVarintTest(data, i, uint64(m.LatestBlocktime)) return i, nil } -func (m *NodeInfo) Marshal() (data []byte, err error) { +func (m *NodeInfo) Marshal() (dAtA []byte, err error) { size := m.Size() - data = make([]byte, size) - n, err := m.MarshalTo(data) + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } - return data[:n], nil + return dAtA[:n], nil } -func (m *NodeInfo) MarshalTo(data []byte) (int, error) { +func (m *NodeInfo) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int _ = l - if m.PubKey == nil { - return 0, github_com_gogo_protobuf_proto.NewRequiredNotSetError("pubKey") + if m.Id == nil { + return 0, proto.NewRequiredNotSetError("id") } else { - data[i] = 0xa + dAtA[i] = 0xa i++ - i = encodeVarintTest(data, i, uint64(m.PubKey.Size())) - n3, err := m.PubKey.MarshalTo(data[i:]) + i = encodeVarintTest(dAtA, i, uint64(m.Id.Size())) + n3, err := m.Id.MarshalTo(dAtA[i:]) if err != nil { return 0, err } i += n3 } - data[i] = 0x12 - i++ - i = encodeVarintTest(data, i, uint64(len(m.Moniker))) - i += copy(data[i:], m.Moniker) - data[i] = 0x1a - i++ - i = encodeVarintTest(data, i, uint64(len(m.Network))) - i += copy(data[i:], m.Network) - data[i] = 0x22 - i++ - i = encodeVarintTest(data, i, uint64(len(m.RemoteAddr))) - i += copy(data[i:], m.RemoteAddr) - data[i] = 0x2a - i++ - i = encodeVarintTest(data, i, uint64(len(m.ListenAddr))) - i += copy(data[i:], m.ListenAddr) - data[i] = 0x32 - i++ - i = encodeVarintTest(data, i, uint64(len(m.Version))) - i += copy(data[i:], m.Version) + if m.Moniker == nil { + return 0, proto.NewRequiredNotSetError("moniker") + } else { + dAtA[i] = 0x12 + i++ + i = encodeVarintTest(dAtA, i, uint64(len(*m.Moniker))) + i += copy(dAtA[i:], *m.Moniker) + } + if m.Network == nil { + return 0, proto.NewRequiredNotSetError("network") + } else { + dAtA[i] = 0x1a + i++ + i = encodeVarintTest(dAtA, i, uint64(len(*m.Network))) + i += copy(dAtA[i:], *m.Network) + } + if m.RemoteAddr == nil { + return 0, proto.NewRequiredNotSetError("remoteAddr") + } else { + dAtA[i] = 0x22 + i++ + i = encodeVarintTest(dAtA, i, uint64(len(*m.RemoteAddr))) + i += copy(dAtA[i:], *m.RemoteAddr) + } + if m.ListenAddr == nil { + return 0, proto.NewRequiredNotSetError("listenAddr") + } else { + dAtA[i] = 0x2a + i++ + i = encodeVarintTest(dAtA, i, uint64(len(*m.ListenAddr))) + i += copy(dAtA[i:], *m.ListenAddr) + } + if m.Version == nil { + return 0, proto.NewRequiredNotSetError("version") + } else { + dAtA[i] = 0x32 + i++ + i = encodeVarintTest(dAtA, i, uint64(len(*m.Version))) + i += copy(dAtA[i:], *m.Version) + } if len(m.Other) > 0 { for _, s := range m.Other { - data[i] = 0x3a + dAtA[i] = 0x3a i++ l = len(s) for l >= 1<<7 { - data[i] = uint8(uint64(l)&0x7f | 0x80) + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) l >>= 7 i++ } - data[i] = uint8(l) + dAtA[i] = uint8(l) i++ - i += copy(data[i:], s) + i += copy(dAtA[i:], s) } } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } return i, nil } -func (m *PubKey) Marshal() (data []byte, err error) { +func (m *ID) Marshal() (dAtA []byte, err error) { size := m.Size() - data = make([]byte, size) - n, err := m.MarshalTo(data) + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } - return data[:n], nil + return dAtA[:n], nil } -func (m *PubKey) MarshalTo(data []byte) (int, error) { +func (m *ID) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Id == nil { + return 0, proto.NewRequiredNotSetError("id") + } else { + dAtA[i] = 0xa + i++ + i = encodeVarintTest(dAtA, i, uint64(len(*m.Id))) + i += copy(dAtA[i:], *m.Id) + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil +} + +func (m *PubKey) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *PubKey) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int _ = l if m.Ed25519 != nil { - data[i] = 0xa + dAtA[i] = 0xa i++ - i = encodeVarintTest(data, i, uint64(m.Ed25519.Size())) - n4, err := m.Ed25519.MarshalTo(data[i:]) + i = encodeVarintTest(dAtA, i, uint64(m.Ed25519.Size())) + n4, err := m.Ed25519.MarshalTo(dAtA[i:]) if err != nil { return 0, err } i += n4 } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } return i, nil } -func (m *PubKeyEd25519) Marshal() (data []byte, err error) { +func (m *PubKeyEd25519) Marshal() (dAtA []byte, err error) { size := m.Size() - data = make([]byte, size) - n, err := m.MarshalTo(data) + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } - return data[:n], nil + return dAtA[:n], nil } -func (m *PubKeyEd25519) MarshalTo(data []byte) (int, error) { +func (m *PubKeyEd25519) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int _ = l - if m.Bytes != nil { - data[i] = 0xa + if m.Bytes == nil { + return 0, proto.NewRequiredNotSetError("bytes") + } else { + dAtA[i] = 0xa i++ - i = encodeVarintTest(data, i, uint64(len(m.Bytes))) - i += copy(data[i:], m.Bytes) + i = encodeVarintTest(dAtA, i, uint64(len(m.Bytes))) + i += copy(dAtA[i:], m.Bytes) + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) } return i, nil } -func encodeVarintTest(data []byte, offset int, v uint64) int { +func encodeVarintTest(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { - data[offset] = uint8(v&0x7f | 0x80) + dAtA[offset] = uint8(v&0x7f | 0x80) v >>= 7 offset++ } - data[offset] = uint8(v) + dAtA[offset] = uint8(v) return offset + 1 } func (m *ResultStatus) Size() (n int) { @@ -582,34 +471,67 @@ func (m *ResultStatus) Size() (n int) { l = len(m.LatestBlockHash) n += 1 + l + sovTest(uint64(l)) } - n += 1 + sovTest(uint64(m.LatestBlockHeight)) - n += 1 + sovTest(uint64(m.LatestBlocktime)) + if m.LatestBlockHeight != nil { + n += 1 + sovTest(uint64(*m.LatestBlockHeight)) + } + if m.LatestBlocktime != nil { + n += 1 + sovTest(uint64(*m.LatestBlocktime)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } return n } func (m *NodeInfo) Size() (n int) { var l int _ = l - if m.PubKey != nil { - l = m.PubKey.Size() + if m.Id != nil { + l = m.Id.Size() + n += 1 + l + sovTest(uint64(l)) + } + if m.Moniker != nil { + l = len(*m.Moniker) + n += 1 + l + sovTest(uint64(l)) + } + if m.Network != nil { + l = len(*m.Network) + n += 1 + l + sovTest(uint64(l)) + } + if m.RemoteAddr != nil { + l = len(*m.RemoteAddr) + n += 1 + l + sovTest(uint64(l)) + } + if m.ListenAddr != nil { + l = len(*m.ListenAddr) + n += 1 + l + sovTest(uint64(l)) + } + if m.Version != nil { + l = len(*m.Version) n += 1 + l + sovTest(uint64(l)) } - l = len(m.Moniker) - n += 1 + l + sovTest(uint64(l)) - l = len(m.Network) - n += 1 + l + sovTest(uint64(l)) - l = len(m.RemoteAddr) - n += 1 + l + sovTest(uint64(l)) - l = len(m.ListenAddr) - n += 1 + l + sovTest(uint64(l)) - l = len(m.Version) - n += 1 + l + sovTest(uint64(l)) if len(m.Other) > 0 { for _, s := range m.Other { l = len(s) n += 1 + l + sovTest(uint64(l)) } } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *ID) Size() (n int) { + var l int + _ = l + if m.Id != nil { + l = len(*m.Id) + n += 1 + l + sovTest(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } return n } @@ -620,6 +542,9 @@ func (m *PubKey) Size() (n int) { l = m.Ed25519.Size() n += 1 + l + sovTest(uint64(l)) } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } return n } @@ -630,6 +555,9 @@ func (m *PubKeyEd25519) Size() (n int) { l = len(m.Bytes) n += 1 + l + sovTest(uint64(l)) } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } return n } @@ -643,59 +571,10 @@ func sovTest(x uint64) (n int) { } return n } -func (this *ResultStatus) String() string { - if this == nil { - return "nil" - } - s := strings.Join([]string{`&ResultStatus{`, - `NodeInfo:` + strings.Replace(fmt.Sprintf("%v", this.NodeInfo), "NodeInfo", "NodeInfo", 1) + `,`, - `PubKey:` + strings.Replace(fmt.Sprintf("%v", this.PubKey), "PubKey", "PubKey", 1) + `,`, - `LatestBlockHash:` + fmt.Sprintf("%v", this.LatestBlockHash) + `,`, - `LatestBlockHeight:` + fmt.Sprintf("%v", this.LatestBlockHeight) + `,`, - `LatestBlocktime:` + fmt.Sprintf("%v", this.LatestBlocktime) + `,`, - `}`, - }, "") - return s -} -func (this *NodeInfo) String() string { - if this == nil { - return "nil" - } - s := strings.Join([]string{`&NodeInfo{`, - `PubKey:` + strings.Replace(fmt.Sprintf("%v", this.PubKey), "PubKey", "PubKey", 1) + `,`, - `Moniker:` + fmt.Sprintf("%v", this.Moniker) + `,`, - `Network:` + fmt.Sprintf("%v", this.Network) + `,`, - `RemoteAddr:` + fmt.Sprintf("%v", this.RemoteAddr) + `,`, - `ListenAddr:` + fmt.Sprintf("%v", this.ListenAddr) + `,`, - `Version:` + fmt.Sprintf("%v", this.Version) + `,`, - `Other:` + fmt.Sprintf("%v", this.Other) + `,`, - `}`, - }, "") - return s -} -func (this *PubKey) String() string { - if this == nil { - return "nil" - } - s := strings.Join([]string{`&PubKey{`, - `Ed25519:` + strings.Replace(fmt.Sprintf("%v", this.Ed25519), "PubKeyEd25519", "PubKeyEd25519", 1) + `,`, - `}`, - }, "") - return s -} -func (this *PubKeyEd25519) String() string { - if this == nil { - return "nil" - } - s := strings.Join([]string{`&PubKeyEd25519{`, - `Bytes:` + fmt.Sprintf("%v", this.Bytes) + `,`, - `}`, - }, "") - return s -} -func (m *ResultStatus) Unmarshal(data []byte) error { + +func (m *ResultStatus) Unmarshal(dAtA []byte) error { var hasFields [1]uint64 - l := len(data) + l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx @@ -707,7 +586,7 @@ func (m *ResultStatus) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { @@ -735,7 +614,7 @@ func (m *ResultStatus) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ msglen |= (int(b) & 0x7F) << shift if b < 0x80 { @@ -752,7 +631,7 @@ func (m *ResultStatus) Unmarshal(data []byte) error { if m.NodeInfo == nil { m.NodeInfo = &NodeInfo{} } - if err := m.NodeInfo.Unmarshal(data[iNdEx:postIndex]); err != nil { + if err := m.NodeInfo.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -768,7 +647,7 @@ func (m *ResultStatus) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ msglen |= (int(b) & 0x7F) << shift if b < 0x80 { @@ -785,7 +664,7 @@ func (m *ResultStatus) Unmarshal(data []byte) error { if m.PubKey == nil { m.PubKey = &PubKey{} } - if err := m.PubKey.Unmarshal(data[iNdEx:postIndex]); err != nil { + if err := m.PubKey.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -802,7 +681,7 @@ func (m *ResultStatus) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ byteLen |= (int(b) & 0x7F) << shift if b < 0x80 { @@ -816,14 +695,17 @@ func (m *ResultStatus) Unmarshal(data []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.LatestBlockHash = append([]byte{}, data[iNdEx:postIndex]...) + m.LatestBlockHash = append(m.LatestBlockHash[:0], dAtA[iNdEx:postIndex]...) + if m.LatestBlockHash == nil { + m.LatestBlockHash = []byte{} + } iNdEx = postIndex hasFields[0] |= uint64(0x00000002) case 4: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field LatestBlockHeight", wireType) } - m.LatestBlockHeight = 0 + var v int64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowTest @@ -831,19 +713,20 @@ func (m *ResultStatus) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ - m.LatestBlockHeight |= (int64(b) & 0x7F) << shift + v |= (int64(b) & 0x7F) << shift if b < 0x80 { break } } + m.LatestBlockHeight = &v hasFields[0] |= uint64(0x00000004) case 5: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field LatestBlocktime", wireType) } - m.LatestBlocktime = 0 + var v int64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowTest @@ -851,17 +734,18 @@ func (m *ResultStatus) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ - m.LatestBlocktime |= (int64(b) & 0x7F) << shift + v |= (int64(b) & 0x7F) << shift if b < 0x80 { break } } + m.LatestBlocktime = &v hasFields[0] |= uint64(0x00000008) default: iNdEx = preIndex - skippy, err := skipTest(data[iNdEx:]) + skippy, err := skipTest(dAtA[iNdEx:]) if err != nil { return err } @@ -871,20 +755,21 @@ func (m *ResultStatus) Unmarshal(data []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if hasFields[0]&uint64(0x00000001) == 0 { - return github_com_gogo_protobuf_proto.NewRequiredNotSetError("pubKey") + return proto.NewRequiredNotSetError("pubKey") } if hasFields[0]&uint64(0x00000002) == 0 { - return github_com_gogo_protobuf_proto.NewRequiredNotSetError("latestBlockHash") + return proto.NewRequiredNotSetError("latestBlockHash") } if hasFields[0]&uint64(0x00000004) == 0 { - return github_com_gogo_protobuf_proto.NewRequiredNotSetError("latestBlockHeight") + return proto.NewRequiredNotSetError("latestBlockHeight") } if hasFields[0]&uint64(0x00000008) == 0 { - return github_com_gogo_protobuf_proto.NewRequiredNotSetError("latestBlocktime") + return proto.NewRequiredNotSetError("latestBlocktime") } if iNdEx > l { @@ -892,9 +777,9 @@ func (m *ResultStatus) Unmarshal(data []byte) error { } return nil } -func (m *NodeInfo) Unmarshal(data []byte) error { +func (m *NodeInfo) Unmarshal(dAtA []byte) error { var hasFields [1]uint64 - l := len(data) + l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx @@ -906,7 +791,7 @@ func (m *NodeInfo) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { @@ -924,7 +809,7 @@ func (m *NodeInfo) Unmarshal(data []byte) error { switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field PubKey", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Id", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -934,7 +819,7 @@ func (m *NodeInfo) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ msglen |= (int(b) & 0x7F) << shift if b < 0x80 { @@ -948,10 +833,10 @@ func (m *NodeInfo) Unmarshal(data []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - if m.PubKey == nil { - m.PubKey = &PubKey{} + if m.Id == nil { + m.Id = &ID{} } - if err := m.PubKey.Unmarshal(data[iNdEx:postIndex]); err != nil { + if err := m.Id.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -968,7 +853,7 @@ func (m *NodeInfo) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { @@ -983,7 +868,8 @@ func (m *NodeInfo) Unmarshal(data []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Moniker = string(data[iNdEx:postIndex]) + s := string(dAtA[iNdEx:postIndex]) + m.Moniker = &s iNdEx = postIndex hasFields[0] |= uint64(0x00000002) case 3: @@ -998,7 +884,7 @@ func (m *NodeInfo) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { @@ -1013,7 +899,8 @@ func (m *NodeInfo) Unmarshal(data []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Network = string(data[iNdEx:postIndex]) + s := string(dAtA[iNdEx:postIndex]) + m.Network = &s iNdEx = postIndex hasFields[0] |= uint64(0x00000004) case 4: @@ -1028,7 +915,7 @@ func (m *NodeInfo) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { @@ -1043,7 +930,8 @@ func (m *NodeInfo) Unmarshal(data []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.RemoteAddr = string(data[iNdEx:postIndex]) + s := string(dAtA[iNdEx:postIndex]) + m.RemoteAddr = &s iNdEx = postIndex hasFields[0] |= uint64(0x00000008) case 5: @@ -1058,7 +946,7 @@ func (m *NodeInfo) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { @@ -1073,7 +961,8 @@ func (m *NodeInfo) Unmarshal(data []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.ListenAddr = string(data[iNdEx:postIndex]) + s := string(dAtA[iNdEx:postIndex]) + m.ListenAddr = &s iNdEx = postIndex hasFields[0] |= uint64(0x00000010) case 6: @@ -1088,7 +977,7 @@ func (m *NodeInfo) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { @@ -1103,7 +992,8 @@ func (m *NodeInfo) Unmarshal(data []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Version = string(data[iNdEx:postIndex]) + s := string(dAtA[iNdEx:postIndex]) + m.Version = &s iNdEx = postIndex hasFields[0] |= uint64(0x00000020) case 7: @@ -1118,7 +1008,7 @@ func (m *NodeInfo) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { @@ -1133,11 +1023,11 @@ func (m *NodeInfo) Unmarshal(data []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Other = append(m.Other, string(data[iNdEx:postIndex])) + m.Other = append(m.Other, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex default: iNdEx = preIndex - skippy, err := skipTest(data[iNdEx:]) + skippy, err := skipTest(dAtA[iNdEx:]) if err != nil { return err } @@ -1147,26 +1037,27 @@ func (m *NodeInfo) Unmarshal(data []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if hasFields[0]&uint64(0x00000001) == 0 { - return github_com_gogo_protobuf_proto.NewRequiredNotSetError("pubKey") + return proto.NewRequiredNotSetError("id") } if hasFields[0]&uint64(0x00000002) == 0 { - return github_com_gogo_protobuf_proto.NewRequiredNotSetError("moniker") + return proto.NewRequiredNotSetError("moniker") } if hasFields[0]&uint64(0x00000004) == 0 { - return github_com_gogo_protobuf_proto.NewRequiredNotSetError("network") + return proto.NewRequiredNotSetError("network") } if hasFields[0]&uint64(0x00000008) == 0 { - return github_com_gogo_protobuf_proto.NewRequiredNotSetError("remoteAddr") + return proto.NewRequiredNotSetError("remoteAddr") } if hasFields[0]&uint64(0x00000010) == 0 { - return github_com_gogo_protobuf_proto.NewRequiredNotSetError("listenAddr") + return proto.NewRequiredNotSetError("listenAddr") } if hasFields[0]&uint64(0x00000020) == 0 { - return github_com_gogo_protobuf_proto.NewRequiredNotSetError("version") + return proto.NewRequiredNotSetError("version") } if iNdEx > l { @@ -1174,8 +1065,9 @@ func (m *NodeInfo) Unmarshal(data []byte) error { } return nil } -func (m *PubKey) Unmarshal(data []byte) error { - l := len(data) +func (m *ID) Unmarshal(dAtA []byte) error { + var hasFields [1]uint64 + l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx @@ -1187,7 +1079,92 @@ func (m *PubKey) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ID: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ID: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Id", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTest + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthTest + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + s := string(dAtA[iNdEx:postIndex]) + m.Id = &s + iNdEx = postIndex + hasFields[0] |= uint64(0x00000001) + default: + iNdEx = preIndex + skippy, err := skipTest(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthTest + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + if hasFields[0]&uint64(0x00000001) == 0 { + return proto.NewRequiredNotSetError("id") + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *PubKey) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTest + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] iNdEx++ wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { @@ -1215,7 +1192,7 @@ func (m *PubKey) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ msglen |= (int(b) & 0x7F) << shift if b < 0x80 { @@ -1232,13 +1209,13 @@ func (m *PubKey) Unmarshal(data []byte) error { if m.Ed25519 == nil { m.Ed25519 = &PubKeyEd25519{} } - if err := m.Ed25519.Unmarshal(data[iNdEx:postIndex]); err != nil { + if err := m.Ed25519.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex - skippy, err := skipTest(data[iNdEx:]) + skippy, err := skipTest(dAtA[iNdEx:]) if err != nil { return err } @@ -1248,6 +1225,7 @@ func (m *PubKey) Unmarshal(data []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -1257,9 +1235,9 @@ func (m *PubKey) Unmarshal(data []byte) error { } return nil } -func (m *PubKeyEd25519) Unmarshal(data []byte) error { +func (m *PubKeyEd25519) Unmarshal(dAtA []byte) error { var hasFields [1]uint64 - l := len(data) + l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx @@ -1271,7 +1249,7 @@ func (m *PubKeyEd25519) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { @@ -1299,7 +1277,7 @@ func (m *PubKeyEd25519) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ byteLen |= (int(b) & 0x7F) << shift if b < 0x80 { @@ -1313,12 +1291,15 @@ func (m *PubKeyEd25519) Unmarshal(data []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Bytes = append([]byte{}, data[iNdEx:postIndex]...) + m.Bytes = append(m.Bytes[:0], dAtA[iNdEx:postIndex]...) + if m.Bytes == nil { + m.Bytes = []byte{} + } iNdEx = postIndex hasFields[0] |= uint64(0x00000001) default: iNdEx = preIndex - skippy, err := skipTest(data[iNdEx:]) + skippy, err := skipTest(dAtA[iNdEx:]) if err != nil { return err } @@ -1328,11 +1309,12 @@ func (m *PubKeyEd25519) Unmarshal(data []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if hasFields[0]&uint64(0x00000001) == 0 { - return github_com_gogo_protobuf_proto.NewRequiredNotSetError("bytes") + return proto.NewRequiredNotSetError("bytes") } if iNdEx > l { @@ -1340,8 +1322,8 @@ func (m *PubKeyEd25519) Unmarshal(data []byte) error { } return nil } -func skipTest(data []byte) (n int, err error) { - l := len(data) +func skipTest(dAtA []byte) (n int, err error) { + l := len(dAtA) iNdEx := 0 for iNdEx < l { var wire uint64 @@ -1352,7 +1334,7 @@ func skipTest(data []byte) (n int, err error) { if iNdEx >= l { return 0, io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { @@ -1370,7 +1352,7 @@ func skipTest(data []byte) (n int, err error) { return 0, io.ErrUnexpectedEOF } iNdEx++ - if data[iNdEx-1] < 0x80 { + if dAtA[iNdEx-1] < 0x80 { break } } @@ -1387,7 +1369,7 @@ func skipTest(data []byte) (n int, err error) { if iNdEx >= l { return 0, io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ length |= (int(b) & 0x7F) << shift if b < 0x80 { @@ -1410,7 +1392,7 @@ func skipTest(data []byte) (n int, err error) { if iNdEx >= l { return 0, io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ innerWire |= (uint64(b) & 0x7F) << shift if b < 0x80 { @@ -1421,7 +1403,7 @@ func skipTest(data []byte) (n int, err error) { if innerWireType == 4 { break } - next, err := skipTest(data[start:]) + next, err := skipTest(dAtA[start:]) if err != nil { return 0, err } @@ -1444,3 +1426,31 @@ var ( ErrInvalidLengthTest = fmt.Errorf("proto: negative length found during unmarshaling") ErrIntOverflowTest = fmt.Errorf("proto: integer overflow") ) + +func init() { proto.RegisterFile("test.proto", fileDescriptorTest) } + +var fileDescriptorTest = []byte{ + // 342 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x64, 0x91, 0x41, 0x4b, 0xf3, 0x30, + 0x1c, 0xc6, 0x49, 0xfb, 0x6e, 0x5d, 0xff, 0xeb, 0x3b, 0x31, 0xee, 0x90, 0x53, 0x2d, 0x85, 0x41, + 0x0f, 0x52, 0xb0, 0xb0, 0x83, 0x47, 0xc7, 0x04, 0x87, 0x20, 0x23, 0x7e, 0x82, 0xcd, 0xfe, 0x75, + 0x65, 0x5d, 0x33, 0x92, 0x4c, 0xd9, 0xe7, 0xf3, 0xe2, 0xd1, 0xa3, 0x47, 0xd9, 0x27, 0x91, 0xa6, + 0xed, 0x9c, 0xf3, 0xf8, 0xfc, 0x7e, 0xe5, 0xc9, 0xd3, 0x04, 0x40, 0xa3, 0xd2, 0xf1, 0x5a, 0x0a, + 0x2d, 0xc2, 0x4f, 0x02, 0x1e, 0x47, 0xb5, 0xc9, 0xf5, 0x83, 0x9e, 0xe9, 0x8d, 0xa2, 0x03, 0xe8, + 0x14, 0x22, 0xc5, 0x49, 0xf1, 0x24, 0x18, 0x09, 0x48, 0xd4, 0x4d, 0xdc, 0xf8, 0xbe, 0x06, 0x7c, + 0xaf, 0xe8, 0x39, 0xb4, 0xd7, 0x9b, 0xf9, 0x1d, 0x6e, 0x99, 0x15, 0x58, 0x51, 0x37, 0x71, 0xe2, + 0xa9, 0x89, 0xbc, 0xc6, 0x34, 0x82, 0x93, 0x7c, 0x56, 0x1e, 0x34, 0xca, 0xc5, 0xe3, 0xf2, 0x76, + 0xa6, 0x16, 0xcc, 0x0e, 0xac, 0xc8, 0xe3, 0xc7, 0x98, 0x5e, 0xc0, 0xe9, 0x21, 0xc2, 0xec, 0x79, + 0xa1, 0xd9, 0xbf, 0xc0, 0x8a, 0x6c, 0xfe, 0x57, 0x1c, 0xf5, 0xea, 0x6c, 0x85, 0xac, 0x65, 0xbe, + 0x3d, 0xc6, 0xe1, 0x1b, 0x81, 0x4e, 0xb3, 0x9c, 0x9e, 0x81, 0x95, 0xa5, 0x8c, 0x98, 0xad, 0x76, + 0x3c, 0x19, 0x73, 0x2b, 0x4b, 0x29, 0x03, 0x67, 0x25, 0x8a, 0x6c, 0x89, 0xd2, 0xfc, 0x85, 0xcb, + 0x9b, 0x58, 0x9a, 0x02, 0xf5, 0xab, 0x90, 0x4b, 0xb3, 0xda, 0xe5, 0x4d, 0xa4, 0x3e, 0x80, 0xc4, + 0x95, 0xd0, 0x78, 0x9d, 0xa6, 0xd2, 0xcc, 0x74, 0xf9, 0x01, 0x29, 0x7d, 0x9e, 0x29, 0x8d, 0x85, + 0xf1, 0xad, 0xca, 0xff, 0x90, 0xb2, 0xf9, 0x05, 0xa5, 0xca, 0x44, 0xc1, 0xda, 0x55, 0x73, 0x1d, + 0x69, 0x1f, 0x5a, 0x42, 0x2f, 0x50, 0x32, 0x27, 0xb0, 0x23, 0x97, 0x57, 0x21, 0xec, 0x83, 0x35, + 0x19, 0xd3, 0xde, 0x7e, 0xbe, 0x5b, 0x2e, 0x0f, 0x13, 0x68, 0x4f, 0x9b, 0x7b, 0x76, 0x30, 0x4d, + 0x86, 0xc3, 0xcb, 0xab, 0xfa, 0xb9, 0x7a, 0xf5, 0x4b, 0xdc, 0x54, 0x94, 0x37, 0x3a, 0x1c, 0xc0, + 0xff, 0x5f, 0xa6, 0x3c, 0x70, 0xbe, 0xd5, 0xa8, 0x4c, 0xaf, 0xc7, 0xab, 0x30, 0xf2, 0xde, 0x77, + 0x3e, 0xf9, 0xd8, 0xf9, 0xe4, 0x6b, 0xe7, 0x93, 0xef, 0x00, 0x00, 0x00, 0xff, 0xff, 0xb1, 0xee, + 0x6b, 0xdd, 0x2c, 0x02, 0x00, 0x00, +} diff --git a/benchmarks/proto/test.proto b/benchmarks/proto/test.proto index d1a1c94d7..6d770d98a 100644 --- a/benchmarks/proto/test.proto +++ b/benchmarks/proto/test.proto @@ -7,7 +7,7 @@ message ResultStatus { } message NodeInfo { - required PubKey pubKey = 1; + required ID id = 1; required string moniker = 2; required string network = 3; required string remoteAddr = 4; @@ -16,6 +16,10 @@ message NodeInfo { repeated string other = 7; } +message ID { + required string id = 1; +} + message PubKey { optional PubKeyEd25519 ed25519 = 1; } diff --git a/consensus/common_test.go b/consensus/common_test.go index 5e837ea47..4ddd6b8aa 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -395,7 +395,7 @@ func randConsensusNetWithPeers(nValidators, nPeers int, testName string, tickerF func getSwitchIndex(switches []*p2p.Switch, peer p2p.Peer) int { for i, s := range switches { - if bytes.Equal(peer.NodeInfo().PubKey.Address(), s.NodeInfo().PubKey.Address()) { + if peer.NodeInfo().ID == s.NodeInfo().ID { return i } } diff --git a/docs/how-to-read-logs.rst b/docs/how-to-read-logs.rst index c5a352d72..e0c3f1b4f 100644 --- a/docs/how-to-read-logs.rst +++ b/docs/how-to-read-logs.rst @@ -59,7 +59,7 @@ Next we replay all the messages from the WAL. :: I[10-04|13:54:30.391] Starting RPC HTTP server on tcp socket 0.0.0.0:46657 module=rpc-server - I[10-04|13:54:30.392] Started node module=main nodeInfo="NodeInfo{pk: PubKeyEd25519{DF22D7C92C91082324A1312F092AA1DA197FA598DBBFB6526E177003C4D6FD66}, moniker: anonymous, network: test-chain-3MNw2N [remote , listen 10.0.2.15:46656], version: 0.11.0-10f361fc ([wire_version=0.6.2 p2p_version=0.5.0 consensus_version=v1/0.2.2 rpc_version=0.7.0/3 tx_index=on rpc_addr=tcp://0.0.0.0:46657])}" + I[10-04|13:54:30.392] Started node module=main nodeInfo="NodeInfo{id: DF22D7C92C91082324A1312F092AA1DA197FA598DBBFB6526E, moniker: anonymous, network: test-chain-3MNw2N [remote , listen 10.0.2.15:46656], version: 0.11.0-10f361fc ([wire_version=0.6.2 p2p_version=0.5.0 consensus_version=v1/0.2.2 rpc_version=0.7.0/3 tx_index=on rpc_addr=tcp://0.0.0.0:46657])}" Next follows a standard block creation cycle, where we enter a new round, propose a block, receive more than 2/3 of prevotes, then precommits and finally diff --git a/docs/specification/new-spec/p2p/peer.md b/docs/specification/new-spec/p2p/peer.md index 68615c498..6aa36af79 100644 --- a/docs/specification/new-spec/p2p/peer.md +++ b/docs/specification/new-spec/p2p/peer.md @@ -83,7 +83,7 @@ The Tendermint Version Handshake allows the peers to exchange their NodeInfo: ```golang type NodeInfo struct { - PubKey crypto.PubKey + ID p2p.ID Moniker string Network string RemoteAddr string @@ -95,7 +95,7 @@ type NodeInfo struct { ``` The connection is disconnected if: -- `peer.NodeInfo.PubKey != peer.PubKey` +- `peer.NodeInfo.ID` is not equal `peerConn.ID` - `peer.NodeInfo.Version` is not formatted as `X.X.X` where X are integers known as Major, Minor, and Revision - `peer.NodeInfo.Version` Major is not the same as ours - `peer.NodeInfo.Version` Minor is not the same as ours diff --git a/node/node.go b/node/node.go index 83d27cb48..976066964 100644 --- a/node/node.go +++ b/node/node.go @@ -405,7 +405,7 @@ func (n *Node) OnStart() error { } n.Logger.Info("P2P Node ID", "ID", nodeKey.ID(), "file", n.config.NodeKeyFile()) - nodeInfo := n.makeNodeInfo(nodeKey.PubKey()) + nodeInfo := n.makeNodeInfo(nodeKey.ID()) n.sw.SetNodeInfo(nodeInfo) n.sw.SetNodeKey(nodeKey) @@ -579,13 +579,13 @@ func (n *Node) ProxyApp() proxy.AppConns { return n.proxyApp } -func (n *Node) makeNodeInfo(pubKey crypto.PubKey) p2p.NodeInfo { +func (n *Node) makeNodeInfo(nodeID p2p.ID) p2p.NodeInfo { txIndexerStatus := "on" if _, ok := n.txIndexer.(*null.TxIndex); ok { txIndexerStatus = "off" } nodeInfo := p2p.NodeInfo{ - PubKey: pubKey, + ID: nodeID, Network: n.genesisDoc.ChainID, Version: version.Version, Channels: []byte{ diff --git a/p2p/node_info.go b/p2p/node_info.go index 346de37d3..930f9bfea 100644 --- a/p2p/node_info.go +++ b/p2p/node_info.go @@ -3,8 +3,6 @@ package p2p import ( "fmt" "strings" - - crypto "github.com/tendermint/go-crypto" ) const ( @@ -20,8 +18,8 @@ func MaxNodeInfoSize() int { // between two peers during the Tendermint P2P handshake. type NodeInfo struct { // Authenticate - PubKey crypto.PubKey `json:"pub_key"` // authenticated pubkey - ListenAddr string `json:"listen_addr"` // accepting incoming + ID ID `json:"id"` // authenticated identifier + ListenAddr string `json:"listen_addr"` // accepting incoming // Check compatibility Network string `json:"network"` // network/chain ID @@ -107,19 +105,12 @@ OUTER_LOOP: return nil } -// ID returns node's ID. -func (info NodeInfo) ID() ID { - return PubKeyToID(info.PubKey) -} - // NetAddress returns a NetAddress derived from the NodeInfo - // it includes the authenticated peer ID and the self-reported // ListenAddr. Note that the ListenAddr is not authenticated and // may not match that address actually dialed if its an outbound peer. func (info NodeInfo) NetAddress() *NetAddress { - id := PubKeyToID(info.PubKey) - addr := info.ListenAddr - netAddr, err := NewNetAddressString(IDAddressString(id, addr)) + netAddr, err := NewNetAddressString(IDAddressString(info.ID, info.ListenAddr)) if err != nil { panic(err) // everything should be well formed by now } @@ -127,7 +118,8 @@ func (info NodeInfo) NetAddress() *NetAddress { } func (info NodeInfo) String() string { - return fmt.Sprintf("NodeInfo{pk: %v, moniker: %v, network: %v [listen %v], version: %v (%v)}", info.PubKey, info.Moniker, info.Network, info.ListenAddr, info.Version, info.Other) + return fmt.Sprintf("NodeInfo{id: %v, moniker: %v, network: %v [listen %v], version: %v (%v)}", + info.ID, info.Moniker, info.Network, info.ListenAddr, info.Version, info.Other) } func splitVersion(version string) (string, string, string, error) { diff --git a/p2p/peer.go b/p2p/peer.go index f31a77615..9f3ccbabc 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -202,7 +202,7 @@ func (p *peer) OnStop() { // ID returns the peer's ID - the hex encoded hash of its pubkey. func (p *peer) ID() ID { - return p.nodeInfo.ID() + return p.nodeInfo.ID } // IsOutbound returns true if the connection is outbound, false otherwise. diff --git a/p2p/peer_set_test.go b/p2p/peer_set_test.go index 3041fbe6e..872758355 100644 --- a/p2p/peer_set_test.go +++ b/p2p/peer_set_test.go @@ -13,11 +13,11 @@ import ( // Returns an empty kvstore peer func randPeer() *peer { - pubKey := crypto.GenPrivKeyEd25519().PubKey() + nodeKey := NodeKey{PrivKey: crypto.GenPrivKeyEd25519()} return &peer{ nodeInfo: NodeInfo{ + ID: nodeKey.ID(), ListenAddr: cmn.Fmt("%v.%v.%v.%v:46656", rand.Int()%256, rand.Int()%256, rand.Int()%256, rand.Int()%256), - PubKey: pubKey, }, } } diff --git a/p2p/peer_test.go b/p2p/peer_test.go index ebec00101..24d750a9f 100644 --- a/p2p/peer_test.go +++ b/p2p/peer_test.go @@ -95,7 +95,7 @@ func createOutboundPeerAndPerformHandshake(addr *NetAddress, config *PeerConfig) return nil, err } nodeInfo, err := pc.HandshakeTimeout(NodeInfo{ - PubKey: pk.PubKey(), + ID: addr.ID, Moniker: "host_peer", Network: "testing", Version: "123.123.123", @@ -152,7 +152,7 @@ func (p *remotePeer) accept(l net.Listener) { golog.Fatalf("Failed to create a peer: %+v", err) } _, err = pc.HandshakeTimeout(NodeInfo{ - PubKey: p.PrivKey.PubKey(), + ID: p.Addr().ID, Moniker: "remote_peer", Network: "testing", Version: "123.123.123", diff --git a/p2p/pex/pex_reactor_test.go b/p2p/pex/pex_reactor_test.go index b3f29c5bd..f7297a343 100644 --- a/p2p/pex/pex_reactor_test.go +++ b/p2p/pex/pex_reactor_test.go @@ -289,7 +289,7 @@ func TestPEXReactorCrawlStatus(t *testing.T) { func TestPEXReactorDoesNotAddPrivatePeersToAddrBook(t *testing.T) { peer := p2p.CreateRandomPeer(false) - pexR, book := createReactor(&PEXReactorConfig{PrivatePeerIDs: []string{string(peer.NodeInfo().ID())}}) + pexR, book := createReactor(&PEXReactorConfig{PrivatePeerIDs: []string{string(peer.NodeInfo().ID)}}) defer teardownReactor(book) // we have to send a request to receive responses @@ -356,12 +356,12 @@ func newMockPeer() mockPeer { return mp } -func (mp mockPeer) ID() p2p.ID { return p2p.PubKeyToID(mp.pubKey) } +func (mp mockPeer) ID() p2p.ID { return mp.addr.ID } func (mp mockPeer) IsOutbound() bool { return mp.outbound } func (mp mockPeer) IsPersistent() bool { return mp.persistent } func (mp mockPeer) NodeInfo() p2p.NodeInfo { return p2p.NodeInfo{ - PubKey: mp.pubKey, + ID: mp.addr.ID, ListenAddr: mp.addr.DialString(), } } diff --git a/p2p/switch.go b/p2p/switch.go index 207a4204b..7d7be2fe8 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -515,7 +515,7 @@ func (sw *Switch) addPeer(pc peerConn) error { return err } - peerID := peerNodeInfo.ID() + peerID := peerNodeInfo.ID // ensure connection key matches self reported key if pc.config.AuthEnc { diff --git a/p2p/switch_test.go b/p2p/switch_test.go index c44060c8d..b170ccdc8 100644 --- a/p2p/switch_test.go +++ b/p2p/switch_test.go @@ -221,14 +221,14 @@ func TestConnIDFilter(t *testing.T) { c1, c2 := conn.NetPipe() s1.SetIDFilter(func(id ID) error { - if id == PubKeyToID(s2.nodeInfo.PubKey) { + if id == s2.nodeInfo.ID { return fmt.Errorf("Error: pipe is blacklisted") } return nil }) s2.SetIDFilter(func(id ID) error { - if id == PubKeyToID(s1.nodeInfo.PubKey) { + if id == s1.nodeInfo.ID { return fmt.Errorf("Error: pipe is blacklisted") } return nil diff --git a/p2p/test_util.go b/p2p/test_util.go index b5083dced..9b8300a67 100644 --- a/p2p/test_util.go +++ b/p2p/test_util.go @@ -23,8 +23,8 @@ func CreateRandomPeer(outbound bool) *peer { outbound: outbound, }, nodeInfo: NodeInfo{ + ID: netAddr.ID, ListenAddr: netAddr.DialString(), - PubKey: crypto.GenPrivKeyEd25519().PubKey(), }, mconn: &conn.MConnection{}, } @@ -137,7 +137,7 @@ func MakeSwitch(cfg *cfg.P2PConfig, i int, network, version string, initSwitch f sw.SetLogger(log.TestingLogger()) sw = initSwitch(i, sw) ni := NodeInfo{ - PubKey: nodeKey.PubKey(), + ID: nodeKey.ID(), Moniker: cmn.Fmt("switch%d", i), Network: network, Version: version, diff --git a/rpc/core/net.go b/rpc/core/net.go index 1918abf11..9b04926ab 100644 --- a/rpc/core/net.go +++ b/rpc/core/net.go @@ -43,7 +43,6 @@ func NetInfo() (*ctypes.ResultNetInfo, error) { for _, peer := range p2pSwitch.Peers().List() { peers = append(peers, ctypes.Peer{ NodeInfo: peer.NodeInfo(), - ID: peer.ID(), IsOutbound: peer.IsOutbound(), ConnectionStatus: peer.Status(), }) diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index 7a31ec5d8..729f64c06 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -98,7 +98,6 @@ type ResultDialPeers struct { type Peer struct { p2p.NodeInfo `json:"node_info"` - p2p.ID `json:"node_id"` IsOutbound bool `json:"is_outbound"` ConnectionStatus p2p.ConnectionStatus `json:"connection_status"` } From 64879c1e6af87c447d79f963f0f9891677ac9019 Mon Sep 17 00:00:00 2001 From: Bric3d Date: Wed, 11 Apr 2018 10:38:34 +0200 Subject: [PATCH 095/143] 1417 status response format (#1424) * Reformated the ResultStatus * fix misuse of ResultStatus. * updated changelog * Fixed tests * fixed rpc helper tests * fixed rpc_tests * fixed mock/status_test * fixed typo * fixed ommitempty on validatorstatus and the changelog * fixed extra line in changelog * Updated usage of the /status json response in tests after breaking changes * Updated remaining tests with changes after searching the codebase for usage * Reformated the ResultStatus * fix misuse of ResultStatus. * updated changelog * Fixed tests * fixed rpc helper tests * fixed rpc_tests * fixed mock/status_test * fixed typo * fixed ommitempty on validatorstatus and the changelog * Updated usage of the /status json response in tests after breaking changes * Updated remaining tests with changes after searching the codebase for usage * rebased against develop --- CHANGELOG.md | 2 ++ benchmarks/codec_test.go | 12 +++++--- docs/using-tendermint.rst | 4 +-- lite/client/provider.go | 2 +- rpc/client/helpers.go | 2 +- rpc/client/helpers_test.go | 8 ++--- rpc/client/mock/status_test.go | 16 +++++----- rpc/client/rpc_test.go | 4 +-- rpc/core/status.go | 46 +++++++++++++++------------- rpc/core/types/responses.go | 24 +++++++++------ test/p2p/atomic_broadcast/test.sh | 14 ++++----- test/p2p/basic/test.sh | 4 +-- test/p2p/fast_sync/check_peer.sh | 8 ++--- test/p2p/kill_all/check_peers.sh | 4 +-- test/persist/test_failure_indices.sh | 12 ++++---- test/persist/test_simple.sh | 6 ++-- 16 files changed, 91 insertions(+), 77 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f54c9805d..9a462a993 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,8 @@ BUG FIXES: BREAKING: +- [rpc]: Changed the output format for the `/status` endpoint + Upgrade from go-wire to go-amino. This is a sweeping change that breaks everything that is serialized to disk or over the network. diff --git a/benchmarks/codec_test.go b/benchmarks/codec_test.go index 2f24b776d..d8e4bc82c 100644 --- a/benchmarks/codec_test.go +++ b/benchmarks/codec_test.go @@ -26,10 +26,14 @@ func BenchmarkEncodeStatusWire(b *testing.B) { Version: "SOMEVER", Other: []string{"SOMESTRING", "OTHERSTRING"}, }, - PubKey: nodeKey.PubKey(), - LatestBlockHash: []byte("SOMEBYTES"), - LatestBlockHeight: 123, - LatestBlockTime: time.Unix(0, 1234), + SyncInfo: ctypes.SyncInfo{ + LatestBlockHash: []byte("SOMEBYTES"), + LatestBlockHeight: 123, + LatestBlockTime: time.Unix(0, 1234), + }, + ValidatorInfo: ctypes.ValidatorInfo{ + PubKey: nodeKey.PubKey(), + }, } b.StartTimer() diff --git a/docs/using-tendermint.rst b/docs/using-tendermint.rst index 13c2d882f..1d8e1aaaa 100644 --- a/docs/using-tendermint.rst +++ b/docs/using-tendermint.rst @@ -83,11 +83,11 @@ We can see the chain's status at the ``/status`` end-point: curl http://localhost:46657/status | jsonpp -and the ``latest_app_hash`` in particular: +and the ``sync_info.latest_app_hash`` in particular: :: - curl http://localhost:46657/status | jsonpp | grep app_hash + curl http://localhost:46657/status | jsonpp | grep sync_info.latest_app_hash Visit http://localhost:46657 in your browser to see the list of other endpoints. Some take no arguments (like ``/status``), while others diff --git a/lite/client/provider.go b/lite/client/provider.go index c98297dec..5f3d72450 100644 --- a/lite/client/provider.go +++ b/lite/client/provider.go @@ -93,7 +93,7 @@ func (p *provider) GetLatestCommit() (*ctypes.ResultCommit, error) { if err != nil { return nil, err } - return p.node.Commit(&status.LatestBlockHeight) + return p.node.Commit(&status.SyncInfo.LatestBlockHeight) } // CommitFromResult ... diff --git a/rpc/client/helpers.go b/rpc/client/helpers.go index 86c65919d..7e64d1164 100644 --- a/rpc/client/helpers.go +++ b/rpc/client/helpers.go @@ -41,7 +41,7 @@ func WaitForHeight(c StatusClient, h int64, waiter Waiter) error { if err != nil { return err } - delta = h - s.LatestBlockHeight + delta = h - s.SyncInfo.LatestBlockHeight // wait for the time, or abort early if err := waiter(delta); err != nil { return err diff --git a/rpc/client/helpers_test.go b/rpc/client/helpers_test.go index cef462475..d5c31ddaf 100644 --- a/rpc/client/helpers_test.go +++ b/rpc/client/helpers_test.go @@ -32,7 +32,7 @@ func TestWaitForHeight(t *testing.T) { // now set current block height to 10 m.Call = mock.Call{ - Response: &ctypes.ResultStatus{LatestBlockHeight: 10}, + Response: &ctypes.ResultStatus{SyncInfo: ctypes.SyncInfo{LatestBlockHeight: 10} }, } // we will not wait for more than 10 blocks @@ -52,7 +52,7 @@ func TestWaitForHeight(t *testing.T) { // we use the callback to update the status height myWaiter := func(delta int64) error { // update the height for the next call - m.Call.Response = &ctypes.ResultStatus{LatestBlockHeight: 15} + m.Call.Response = &ctypes.ResultStatus{SyncInfo: ctypes.SyncInfo{LatestBlockHeight: 15}} return client.DefaultWaitStrategy(delta) } @@ -66,11 +66,11 @@ func TestWaitForHeight(t *testing.T) { require.Nil(pre.Error) prer, ok := pre.Response.(*ctypes.ResultStatus) require.True(ok) - assert.Equal(int64(10), prer.LatestBlockHeight) + assert.Equal(int64(10), prer.SyncInfo.LatestBlockHeight) post := r.Calls[4] require.Nil(post.Error) postr, ok := post.Response.(*ctypes.ResultStatus) require.True(ok) - assert.Equal(int64(15), postr.LatestBlockHeight) + assert.Equal(int64(15), postr.SyncInfo.LatestBlockHeight) } diff --git a/rpc/client/mock/status_test.go b/rpc/client/mock/status_test.go index b3827fb13..dafd35080 100644 --- a/rpc/client/mock/status_test.go +++ b/rpc/client/mock/status_test.go @@ -17,9 +17,11 @@ func TestStatus(t *testing.T) { m := &mock.StatusMock{ Call: mock.Call{ Response: &ctypes.ResultStatus{ - LatestBlockHash: cmn.HexBytes("block"), - LatestAppHash: cmn.HexBytes("app"), - LatestBlockHeight: 10, + SyncInfo: ctypes.SyncInfo{ + LatestBlockHash: cmn.HexBytes("block"), + LatestAppHash: cmn.HexBytes("app"), + LatestBlockHeight: 10, + }, }}, } @@ -29,8 +31,8 @@ func TestStatus(t *testing.T) { // make sure response works proper status, err := r.Status() require.Nil(err, "%+v", err) - assert.EqualValues("block", status.LatestBlockHash) - assert.EqualValues(10, status.LatestBlockHeight) + assert.EqualValues("block", status.SyncInfo.LatestBlockHash) + assert.EqualValues(10, status.SyncInfo.LatestBlockHeight) // make sure recorder works properly require.Equal(1, len(r.Calls)) @@ -41,6 +43,6 @@ func TestStatus(t *testing.T) { require.NotNil(rs.Response) st, ok := rs.Response.(*ctypes.ResultStatus) require.True(ok) - assert.EqualValues("block", st.LatestBlockHash) - assert.EqualValues(10, st.LatestBlockHeight) + assert.EqualValues("block", st.SyncInfo.LatestBlockHash) + assert.EqualValues(10, st.SyncInfo.LatestBlockHeight) } diff --git a/rpc/client/rpc_test.go b/rpc/client/rpc_test.go index c5c5822f9..bfdc707f4 100644 --- a/rpc/client/rpc_test.go +++ b/rpc/client/rpc_test.go @@ -50,7 +50,7 @@ func TestInfo(t *testing.T) { info, err := c.ABCIInfo() require.Nil(t, err, "%d: %+v", i, err) // TODO: this is not correct - fix merkleeyes! - // assert.EqualValues(t, status.LatestBlockHeight, info.Response.LastBlockHeight) + // assert.EqualValues(t, status.SyncInfo.LatestBlockHeight, info.Response.LastBlockHeight) assert.True(t, strings.Contains(info.Response.Data, "size")) } } @@ -136,7 +136,7 @@ func TestAppCalls(t *testing.T) { s, err := c.Status() require.Nil(err, "%d: %+v", i, err) // sh is start height or status height - sh := s.LatestBlockHeight + sh := s.SyncInfo.LatestBlockHeight // look for the future h := sh + 2 diff --git a/rpc/core/status.go b/rpc/core/status.go index b4543a61c..9bdf9dd64 100644 --- a/rpc/core/status.go +++ b/rpc/core/status.go @@ -27,15 +27,20 @@ import ( // ```json // { // "result": { -// "syncing": false, -// "latest_block_time": "2017-12-07T18:19:47.617Z", -// "latest_block_height": 6, -// "latest_app_hash": "", -// "latest_block_hash": "A63D0C3307DEDCCFCC82ED411AE9108B70B29E02", -// "pub_key": { -// "data": "8C9A68070CBE33F9C445862BA1E9D96A75CEB68C0CF6ADD3652D07DCAC5D0380", -// "type": "ed25519" -// }, +// "sync_info": { +// "syncing": false, +// "latest_block_time": "2017-12-07T18:19:47.617Z", +// "latest_block_height": 6, +// "latest_app_hash": "", +// "latest_block_hash": "A63D0C3307DEDCCFCC82ED411AE9108B70B29E02", +// } +// "validator_info": { +// "pub_key": { +// "data": "8C9A68070CBE33F9C445862BA1E9D96A75CEB68C0CF6ADD3652D07DCAC5D0380", +// "type": "ed25519" +// }, +// "voting_power": 10 +// } // "node_info": { // "other": [ // "wire_version=0.7.2", @@ -51,9 +56,6 @@ import ( // "network": "test-chain-qhVCa2", // "moniker": "vagrant-ubuntu-trusty-64", // "pub_key": "844981FE99ABB19F7816F2D5E94E8A74276AB1153760A7799E925C75401856C6", -// "validator_status": { -// "voting_power": 10 -// } // } // }, // "id": "", @@ -78,20 +80,20 @@ func Status() (*ctypes.ResultStatus, error) { latestBlockTime := time.Unix(0, latestBlockTimeNano) result := &ctypes.ResultStatus{ - NodeInfo: p2pSwitch.NodeInfo(), - PubKey: pubKey, - LatestBlockHash: latestBlockHash, - LatestAppHash: latestAppHash, - LatestBlockHeight: latestHeight, - LatestBlockTime: latestBlockTime, - Syncing: consensusReactor.FastSync(), + NodeInfo: p2pSwitch.NodeInfo(), + SyncInfo: ctypes.SyncInfo{ + LatestBlockHash: latestBlockHash, + LatestAppHash: latestAppHash, + LatestBlockHeight: latestHeight, + LatestBlockTime: latestBlockTime, + Syncing: consensusReactor.FastSync(), + }, + ValidatorInfo: ctypes.ValidatorInfo{PubKey: pubKey}, } // add ValidatorStatus if node is a validator if val := validatorAtHeight(latestHeight); val != nil { - result.ValidatorStatus = ctypes.ValidatorStatus{ - VotingPower: val.VotingPower, - } + result.ValidatorInfo.VotingPower = val.VotingPower } return result, nil diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index 729f64c06..82bfceaa7 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -54,19 +54,23 @@ func NewResultCommit(header *types.Header, commit *types.Commit, } } -type ValidatorStatus struct { - VotingPower int64 `json:"voting_power"` +type SyncInfo struct { + LatestBlockHash cmn.HexBytes `json:"latest_block_hash"` + LatestAppHash cmn.HexBytes `json:"latest_app_hash"` + LatestBlockHeight int64 `json:"latest_block_height"` + LatestBlockTime time.Time `json:"latest_block_time"` + Syncing bool `json:"syncing"` +} + +type ValidatorInfo struct { + PubKey crypto.PubKey `json:"pub_key"` + VotingPower int64 `json:"voting_power"` } type ResultStatus struct { - NodeInfo p2p.NodeInfo `json:"node_info"` - PubKey crypto.PubKey `json:"pub_key"` - LatestBlockHash cmn.HexBytes `json:"latest_block_hash"` - LatestAppHash cmn.HexBytes `json:"latest_app_hash"` - LatestBlockHeight int64 `json:"latest_block_height"` - LatestBlockTime time.Time `json:"latest_block_time"` - Syncing bool `json:"syncing"` - ValidatorStatus ValidatorStatus `json:"validator_status,omitempty"` + NodeInfo p2p.NodeInfo `json:"node_info"` + SyncInfo SyncInfo `json:"sync_info"` + ValidatorInfo ValidatorInfo `json:"validator_info"` } func (s *ResultStatus) TxIndexEnabled() bool { diff --git a/test/p2p/atomic_broadcast/test.sh b/test/p2p/atomic_broadcast/test.sh index 3224c0427..267330729 100644 --- a/test/p2p/atomic_broadcast/test.sh +++ b/test/p2p/atomic_broadcast/test.sh @@ -17,7 +17,7 @@ for i in $(seq 1 "$N"); do addr=$(test/p2p/ip.sh "$i"):46657 # current state - HASH1=$(curl -s "$addr/status" | jq .result.latest_app_hash) + HASH1=$(curl -s "$addr/status" | jq .result.sync_info.latest_app_hash) # - send a tx TX=aadeadbeefbeefbeef0$i @@ -26,11 +26,11 @@ for i in $(seq 1 "$N"); do echo "" # we need to wait another block to get the new app_hash - h1=$(curl -s "$addr/status" | jq .result.latest_block_height) + h1=$(curl -s "$addr/status" | jq .result.sync_info.latest_block_height) h2=$h1 while [ "$h2" == "$h1" ]; do sleep 1 - h2=$(curl -s "$addr/status" | jq .result.latest_block_height) + h2=$(curl -s "$addr/status" | jq .result.sync_info.latest_block_height) done # wait for all other peers to get to this height @@ -39,16 +39,16 @@ for i in $(seq 1 "$N"); do if [[ "$i" != "$j" ]]; then addrJ=$(test/p2p/ip.sh "$j"):46657 - h=$(curl -s "$addrJ/status" | jq .result.latest_block_height) + h=$(curl -s "$addrJ/status" | jq .result.sync_info.latest_block_height) while [ "$h" -lt "$minHeight" ]; do sleep 1 - h=$(curl -s "$addrJ/status" | jq .result.latest_block_height) + h=$(curl -s "$addrJ/status" | jq .result.sync_info.latest_block_height) done fi done # check that hash was updated - HASH2=$(curl -s "$addr/status" | jq .result.latest_app_hash) + HASH2=$(curl -s "$addr/status" | jq .result.sync_info.latest_app_hash) if [[ "$HASH1" == "$HASH2" ]]; then echo "Expected state hash to update from $HASH1. Got $HASH2" exit 1 @@ -58,7 +58,7 @@ for i in $(seq 1 "$N"); do for j in $(seq 1 "$N"); do if [[ "$i" != "$j" ]]; then addrJ=$(test/p2p/ip.sh "$j"):46657 - HASH3=$(curl -s "$addrJ/status" | jq .result.latest_app_hash) + HASH3=$(curl -s "$addrJ/status" | jq .result.sync_info.latest_app_hash) if [[ "$HASH2" != "$HASH3" ]]; then echo "App hash for node $j doesn't match. Got $HASH3, expected $HASH2" diff --git a/test/p2p/basic/test.sh b/test/p2p/basic/test.sh index 63df62be2..4876c8c54 100755 --- a/test/p2p/basic/test.sh +++ b/test/p2p/basic/test.sh @@ -54,12 +54,12 @@ for i in `seq 1 $N`; do done # - assert block height is greater than 1 - BLOCK_HEIGHT=`curl -s $addr/status | jq .result.latest_block_height` + BLOCK_HEIGHT=`curl -s $addr/status | jq .result.sync_info.latest_block_height` COUNT=0 while [ "$BLOCK_HEIGHT" -le 1 ]; do echo "Waiting for node $i to commit a block ..." sleep 1 - BLOCK_HEIGHT=`curl -s $addr/status | jq .result.latest_block_height` + BLOCK_HEIGHT=`curl -s $addr/status | jq .result.sync_info.latest_block_height` COUNT=$((COUNT+1)) if [ "$COUNT" -gt "$MAX_SLEEP" ]; then echo "Waited too long for node $i to commit a block" diff --git a/test/p2p/fast_sync/check_peer.sh b/test/p2p/fast_sync/check_peer.sh index b10c3efc5..537a5cfe9 100644 --- a/test/p2p/fast_sync/check_peer.sh +++ b/test/p2p/fast_sync/check_peer.sh @@ -15,10 +15,10 @@ peerID=$(( $(($ID % 4)) + 1 )) # 1->2 ... 3->4 ... 4->1 peer_addr=$(test/p2p/ip.sh $peerID):46657 # get another peer's height -h1=`curl -s $peer_addr/status | jq .result.latest_block_height` +h1=`curl -s $peer_addr/status | jq .result.sync_info.latest_block_height` # get another peer's state -root1=`curl -s $peer_addr/status | jq .result.latest_app_hash` +root1=`curl -s $peer_addr/status | jq .result.sync_info.latest_app_hash` echo "Other peer is on height $h1 with state $root1" echo "Waiting for peer $ID to catch up" @@ -29,12 +29,12 @@ set +o pipefail h2="0" while [[ "$h2" -lt "$(($h1+3))" ]]; do sleep 1 - h2=`curl -s $addr/status | jq .result.latest_block_height` + h2=`curl -s $addr/status | jq .result.sync_info.latest_block_height` echo "... $h2" done # check the app hash -root2=`curl -s $addr/status | jq .result.latest_app_hash` +root2=`curl -s $addr/status | jq .result.sync_info.latest_app_hash` if [[ "$root1" != "$root2" ]]; then echo "App hash after fast sync does not match. Got $root2; expected $root1" diff --git a/test/p2p/kill_all/check_peers.sh b/test/p2p/kill_all/check_peers.sh index 37cd13a42..40d9813bf 100644 --- a/test/p2p/kill_all/check_peers.sh +++ b/test/p2p/kill_all/check_peers.sh @@ -23,7 +23,7 @@ set -e # get the first peer's height addr=$(test/p2p/ip.sh 1):46657 -h1=$(curl -s "$addr/status" | jq .result.latest_block_height) +h1=$(curl -s "$addr/status" | jq .result.sync_info.latest_block_height) echo "1st peer is on height $h1" echo "Waiting until other peers reporting a height higher than the 1st one" @@ -33,7 +33,7 @@ for i in $(seq 2 "$NUM_OF_PEERS"); do while [[ $hi -le $h1 ]] ; do addr=$(test/p2p/ip.sh "$i"):46657 - hi=$(curl -s "$addr/status" | jq .result.latest_block_height) + hi=$(curl -s "$addr/status" | jq .result.sync_info.latest_block_height) echo "... peer $i is on height $hi" diff --git a/test/persist/test_failure_indices.sh b/test/persist/test_failure_indices.sh index 42739107f..dd630e1b7 100644 --- a/test/persist/test_failure_indices.sh +++ b/test/persist/test_failure_indices.sh @@ -31,7 +31,7 @@ function start_procs(){ if [[ "$CIRCLECI" == true ]]; then $TM_CMD & else - $TM_CMD &> "tendermint_${name}.log" & + $TM_CMD &> "tendermint_${name}.log" & fi PID_TENDERMINT=$! else @@ -60,8 +60,8 @@ function wait_for_port() { i=0 while [ "$ERR" == 0 ]; do echo "... port $port is still bound. waiting ..." - sleep 1 - nc -z 127.0.0.1 $port + sleep 1 + nc -z 127.0.0.1 $port ERR=$? i=$((i + 1)) if [[ $i == 10 ]]; then @@ -97,7 +97,7 @@ for failIndex in $(seq $failsStart $failsEnd); do ERR=$? i=0 while [ "$ERR" != 0 ]; do - sleep 1 + sleep 1 curl -s --unix-socket "$RPC_ADDR" http://localhost/status > /dev/null ERR=$? i=$((i + 1)) @@ -108,11 +108,11 @@ for failIndex in $(seq $failsStart $failsEnd); do done # wait for a new block - h1=$(curl -s --unix-socket "$RPC_ADDR" http://localhost/status | jq .result.latest_block_height) + h1=$(curl -s --unix-socket "$RPC_ADDR" http://localhost/status | jq .result.sync_info.latest_block_height) h2=$h1 while [ "$h2" == "$h1" ]; do sleep 1 - h2=$(curl -s --unix-socket "$RPC_ADDR" http://localhost/status | jq .result.latest_block_height) + h2=$(curl -s --unix-socket "$RPC_ADDR" http://localhost/status | jq .result.sync_info.latest_block_height) done kill_procs diff --git a/test/persist/test_simple.sh b/test/persist/test_simple.sh index be7852598..171ab747d 100644 --- a/test/persist/test_simple.sh +++ b/test/persist/test_simple.sh @@ -46,7 +46,7 @@ curl -s $addr/status > /dev/null ERR=$? i=0 while [ "$ERR" != 0 ]; do - sleep 1 + sleep 1 curl -s $addr/status > /dev/null ERR=$? i=$(($i + 1)) @@ -57,11 +57,11 @@ while [ "$ERR" != 0 ]; do done # wait for a new block -h1=`curl -s $addr/status | jq .result.latest_block_height` +h1=`curl -s $addr/status | jq .result.sync_info.latest_block_height` h2=$h1 while [ "$h2" == "$h1" ]; do sleep 1 - h2=`curl -s $addr/status | jq .result.latest_block_height` + h2=`curl -s $addr/status | jq .result.sync_info.latest_block_height` done kill_procs From ab00bf7c8b5321e5119b8ae05ca0a6137f4bfb47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Corbi=C3=A8re?= Date: Wed, 11 Apr 2018 11:38:30 +0200 Subject: [PATCH 096/143] standardize PRNG access (#1411) * replace math/rand with tmlibs equivalent. * update tmlibs dependency --- consensus/wal_generator.go | 3 +-- p2p/fuzz.go | 7 ++++--- p2p/pex/addrbook.go | 7 +++---- p2p/pex/pex_reactor.go | 9 ++++----- p2p/switch.go | 8 +++----- p2p/test_util.go | 5 ++--- rpc/lib/client/ws_client.go | 3 +-- 7 files changed, 18 insertions(+), 24 deletions(-) diff --git a/consensus/wal_generator.go b/consensus/wal_generator.go index 7c6fc9ad3..18ff614c5 100644 --- a/consensus/wal_generator.go +++ b/consensus/wal_generator.go @@ -4,7 +4,6 @@ import ( "bufio" "bytes" "fmt" - "math/rand" "os" "path/filepath" "strings" @@ -117,7 +116,7 @@ func makePathname() string { func randPort() int { // returns between base and base + spread base, spread := 20000, 20000 - return base + rand.Intn(spread) + return base + cmn.RandIntn(spread) } func makeAddrs() (string, string, string) { diff --git a/p2p/fuzz.go b/p2p/fuzz.go index fa16e4a26..6bfadc29d 100644 --- a/p2p/fuzz.go +++ b/p2p/fuzz.go @@ -1,10 +1,11 @@ package p2p import ( - "math/rand" "net" "sync" "time" + + cmn "github.com/tendermint/tmlibs/common" ) const ( @@ -124,7 +125,7 @@ func (fc *FuzzedConnection) SetWriteDeadline(t time.Time) error { func (fc *FuzzedConnection) randomDuration() time.Duration { maxDelayMillis := int(fc.config.MaxDelay.Nanoseconds() / 1000) - return time.Millisecond * time.Duration(rand.Int()%maxDelayMillis) // nolint: gas + return time.Millisecond * time.Duration(cmn.RandInt()%maxDelayMillis) // nolint: gas } // implements the fuzz (delay, kill conn) @@ -137,7 +138,7 @@ func (fc *FuzzedConnection) fuzz() bool { switch fc.config.Mode { case FuzzModeDrop: // randomly drop the r/w, drop the conn, or sleep - r := rand.Float64() + r := cmn.RandFloat64() if r <= fc.config.ProbDropRW { return true } else if r < fc.config.ProbDropRW+fc.config.ProbDropConn { diff --git a/p2p/pex/addrbook.go b/p2p/pex/addrbook.go index a8462f375..56a4c5163 100644 --- a/p2p/pex/addrbook.go +++ b/p2p/pex/addrbook.go @@ -9,7 +9,6 @@ import ( "encoding/binary" "fmt" "math" - "math/rand" "net" "sync" "time" @@ -82,7 +81,7 @@ type addrBook struct { // accessed concurrently mtx sync.Mutex - rand *rand.Rand + rand *cmn.Rand ourAddrs map[string]struct{} addrLookup map[p2p.ID]*knownAddress // new & old bucketsOld []map[string]*knownAddress @@ -97,7 +96,7 @@ type addrBook struct { // Use Start to begin processing asynchronous address updates. func NewAddrBook(filePath string, routabilityStrict bool) *addrBook { am := &addrBook{ - rand: rand.New(rand.NewSource(time.Now().UnixNano())), // TODO: seed from outside + rand: cmn.NewRand(), ourAddrs: make(map[string]struct{}), addrLookup: make(map[p2p.ID]*knownAddress), filePath: filePath, @@ -320,7 +319,7 @@ func (a *addrBook) GetSelection() []*p2p.NetAddress { // XXX: What's the point of this if we already loop randomly through addrLookup ? for i := 0; i < numAddresses; i++ { // pick a number between current index and the end - j := rand.Intn(len(allAddr)-i) + i + j := cmn.RandIntn(len(allAddr)-i) + i allAddr[i], allAddr[j] = allAddr[j], allAddr[i] } diff --git a/p2p/pex/pex_reactor.go b/p2p/pex/pex_reactor.go index 00144c35f..70ae7ed11 100644 --- a/p2p/pex/pex_reactor.go +++ b/p2p/pex/pex_reactor.go @@ -2,7 +2,6 @@ package pex import ( "fmt" - "math/rand" "reflect" "sort" "sync" @@ -288,7 +287,7 @@ func (r *PEXReactor) SetEnsurePeersPeriod(d time.Duration) { // Ensures that sufficient peers are connected. (continuous) func (r *PEXReactor) ensurePeersRoutine() { var ( - seed = rand.New(rand.NewSource(time.Now().UnixNano())) + seed = cmn.NewRand() jitter = seed.Int63n(r.ensurePeersPeriod.Nanoseconds()) ) @@ -375,7 +374,7 @@ func (r *PEXReactor) ensurePeers() { peers := r.Switch.Peers().List() peersCount := len(peers) if peersCount > 0 { - peer := peers[rand.Int()%peersCount] // nolint: gas + peer := peers[cmn.RandInt()%peersCount] // nolint: gas r.Logger.Info("We need more addresses. Sending pexRequest to random peer", "peer", peer) r.RequestAddrs(peer) } @@ -404,7 +403,7 @@ func (r *PEXReactor) dialPeer(addr *p2p.NetAddress) { // exponential backoff if it's not our first attempt to dial given address if attempts > 0 { - jitterSeconds := time.Duration(rand.Float64() * float64(time.Second)) // 1s == (1e9 ns) + jitterSeconds := time.Duration(cmn.RandFloat64() * float64(time.Second)) // 1s == (1e9 ns) backoffDuration := jitterSeconds + ((1 << uint(attempts)) * time.Second) sinceLastDialed := time.Since(lastDialed) if sinceLastDialed < backoffDuration { @@ -457,7 +456,7 @@ func (r *PEXReactor) dialSeeds() { } seedAddrs, _ := p2p.NewNetAddressStrings(r.config.Seeds) - perm := rand.Perm(lSeeds) + perm := cmn.RandPerm(lSeeds) // perm := r.Switch.rng.Perm(lSeeds) for _, i := range perm { // dial a random seed diff --git a/p2p/switch.go b/p2p/switch.go index 7d7be2fe8..be7c0ec7a 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -3,7 +3,6 @@ package p2p import ( "fmt" "math" - "math/rand" "net" "sync" "time" @@ -67,7 +66,7 @@ type Switch struct { filterConnByAddr func(net.Addr) error filterConnByID func(ID) error - rng *rand.Rand // seed for randomizing dial times and orders + rng *cmn.Rand // seed for randomizing dial times and orders } // NewSwitch creates a new Switch with the given config. @@ -82,9 +81,8 @@ func NewSwitch(config *cfg.P2PConfig) *Switch { dialing: cmn.NewCMap(), } - // Ensure we have a completely undeterministic PRNG. cmd.RandInt64() draws - // from a seed that's initialized with OS entropy on process start. - sw.rng = rand.New(rand.NewSource(cmn.RandInt64())) + // Ensure we have a completely undeterministic PRNG. + sw.rng = cmn.NewRand() // TODO: collapse the peerConfig into the config ? sw.peerConfig.MConfig.FlushThrottle = time.Duration(config.FlushThrottleTimeout) * time.Millisecond diff --git a/p2p/test_util.go b/p2p/test_util.go index 9b8300a67..fe5282a90 100644 --- a/p2p/test_util.go +++ b/p2p/test_util.go @@ -1,7 +1,6 @@ package p2p import ( - "math/rand" "net" crypto "github.com/tendermint/go-crypto" @@ -35,7 +34,7 @@ func CreateRandomPeer(outbound bool) *peer { func CreateRoutableAddr() (addr string, netAddr *NetAddress) { for { var err error - addr = cmn.Fmt("%X@%v.%v.%v.%v:46656", cmn.RandBytes(20), rand.Int()%256, rand.Int()%256, rand.Int()%256, rand.Int()%256) + addr = cmn.Fmt("%X@%v.%v.%v.%v:46656", cmn.RandBytes(20), cmn.RandInt()%256, cmn.RandInt()%256, cmn.RandInt()%256, cmn.RandInt()%256) netAddr, err = NewNetAddressString(addr) if err != nil { panic(err) @@ -141,7 +140,7 @@ func MakeSwitch(cfg *cfg.P2PConfig, i int, network, version string, initSwitch f Moniker: cmn.Fmt("switch%d", i), Network: network, Version: version, - ListenAddr: cmn.Fmt("%v:%v", network, rand.Intn(64512)+1023), + ListenAddr: cmn.Fmt("%v:%v", network, cmn.RandIntn(64512)+1023), } for ch := range sw.reactorsByCh { ni.Channels = append(ni.Channels, ch) diff --git a/rpc/lib/client/ws_client.go b/rpc/lib/client/ws_client.go index 9b41442a1..a95ce17d2 100644 --- a/rpc/lib/client/ws_client.go +++ b/rpc/lib/client/ws_client.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "math/rand" "net" "net/http" "sync" @@ -266,7 +265,7 @@ func (c *WSClient) reconnect() error { }() for { - jitterSeconds := time.Duration(rand.Float64() * float64(time.Second)) // 1s == (1e9 ns) + jitterSeconds := time.Duration(cmn.RandFloat64() * float64(time.Second)) // 1s == (1e9 ns) backoffDuration := jitterSeconds + ((1 << uint(attempt)) * time.Second) c.Logger.Info("reconnecting", "attempt", attempt+1, "backoff_duration", backoffDuration) From 379f9f875b78340d107752b9b81bc43f6f452d6c Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 11 Apr 2018 17:36:14 +0200 Subject: [PATCH 097/143] update docs for latest develop (#1448) * update docs for latest develop * latest_app_hash, not app_hash --- docs/getting-started.rst | 68 ++++++++++---------- docs/install.rst | 5 +- docs/using-tendermint.rst | 129 ++++++++++++++++++-------------------- 3 files changed, 96 insertions(+), 106 deletions(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index fe49e27b5..9c722f0f1 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -81,9 +81,8 @@ Tendermint node as follows: curl -s localhost:46657/status -The ``-s`` just silences ``curl``. For nicer output, pipe the result -into a tool like `jq `__ or -`jsonpp `__. +The ``-s`` just silences ``curl``. For nicer output, pipe the result into a +tool like `jq `__ or ``json_pp``. Now let's send some transactions to the kvstore. @@ -104,17 +103,23 @@ like: "id": "", "result": { "check_tx": { - "code": 0, - "data": "", - "log": "" + "fee": {} }, "deliver_tx": { - "code": 0, - "data": "", - "log": "" + "tags": [ + { + "key": "YXBwLmNyZWF0b3I=", + "value": "amFl" + }, + { + "key": "YXBwLmtleQ==", + "value": "YWJjZA==" + } + ], + "fee": {} }, - "hash": "2B8EC32BA2579B3B8606E42C06DE2F7AFA2556EF", - "height": 154 + "hash": "9DF66553F98DE3C26E3C3317A3E4CED54F714E39", + "height": 14 } } @@ -134,20 +139,17 @@ The result should look like: "id": "", "result": { "response": { - "code": 0, - "index": 0, - "key": "", - "value": "61626364", - "proof": "", - "height": 0, - "log": "exists" + "log": "exists", + "index": "-1", + "key": "YWJjZA==", + "value": "YWJjZA==" } } } -Note the ``value`` in the result (``61626364``); this is the -hex-encoding of the ASCII of ``abcd``. You can verify this in -a python 2 shell by running ``"61626364".decode('hex')`` or in python 3 shell by running ``import codecs; codecs.decode("61626364", 'hex').decode('ascii')``. Stay +Note the ``value`` in the result (``YWJjZA==``); this is the +base64-encoding of the ASCII of ``abcd``. You can verify this in +a python 2 shell by running ``"61626364".decode('base64')`` or in python 3 shell by running ``import codecs; codecs.decode("61626364", 'base64').decode('ascii')``. Stay tuned for a future release that `makes this output more human-readable `__. Now let's try setting a different key and value: @@ -157,7 +159,7 @@ Now let's try setting a different key and value: curl -s 'localhost:46657/broadcast_tx_commit?tx="name=satoshi"' Now if we query for ``name``, we should get ``satoshi``, or -``7361746F736869`` in hex: +``c2F0b3NoaQ==`` in base64: :: @@ -226,17 +228,15 @@ the number ``1``. If instead, we try to send a ``5``, we get an error: "id": "", "result": { "check_tx": { - "code": 0, - "data": "", - "log": "" + "fee": {} }, "deliver_tx": { - "code": 3, - "data": "", - "log": "Invalid nonce. Expected 1, got 5" + "code": 2, + "log": "Invalid nonce. Expected 1, got 5", + "fee": {} }, "hash": "33B93DFF98749B0D6996A70F64071347060DC19C", - "height": 38 + "height": 34 } } @@ -250,17 +250,13 @@ But if we send a ``1``, it works again: "id": "", "result": { "check_tx": { - "code": 0, - "data": "", - "log": "" + "fee": {} }, "deliver_tx": { - "code": 0, - "data": "", - "log": "" + "fee": {} }, "hash": "F17854A977F6FA7EEA1BD758E296710B86F72F3D", - "height": 87 + "height": 60 } } diff --git a/docs/install.rst b/docs/install.rst index 4ee572604..b2aae6766 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -4,7 +4,7 @@ Install Tendermint From Binary ----------- -To download pre-built binaries, see the `Download page `__. +To download pre-built binaries, see the `Download page `__. From Source ----------- @@ -37,13 +37,13 @@ First, install ``dep``: :: + cd $GOPATH/src/github.com/tendermint/tendermint make get_tools Now we can fetch the correct versions of each dependency by running: :: - cd $GOPATH/src/github.com/tendermint/tendermint make get_vendor_deps make install @@ -96,6 +96,7 @@ If ``go get`` failing bothers you, fetch the code using ``git``: mkdir -p $GOPATH/src/github.com/tendermint git clone https://github.com/tendermint/tendermint $GOPATH/src/github.com/tendermint/tendermint cd $GOPATH/src/github.com/tendermint/tendermint + make get_tools make get_vendor_deps make install diff --git a/docs/using-tendermint.rst b/docs/using-tendermint.rst index 1d8e1aaaa..bc5d73859 100644 --- a/docs/using-tendermint.rst +++ b/docs/using-tendermint.rst @@ -74,20 +74,17 @@ RPC server, for example: curl http://localhost:46657/broadcast_tx_commit?tx=\"abcd\" -For handling responses, we recommend you `install the jsonpp -tool `__ to pretty print the JSON. - We can see the chain's status at the ``/status`` end-point: :: - curl http://localhost:46657/status | jsonpp + curl http://localhost:46657/status | json_pp -and the ``sync_info.latest_app_hash`` in particular: +and the ``latest_app_hash`` in particular: :: - curl http://localhost:46657/status | jsonpp | grep sync_info.latest_app_hash + curl http://localhost:46657/status | json_pp | grep latest_app_hash Visit http://localhost:46657 in your browser to see the list of other endpoints. Some take no arguments (like ``/status``), while others @@ -260,19 +257,19 @@ When ``tendermint init`` is run, both a ``genesis.json`` and :: { - "app_hash": "", - "chain_id": "test-chain-HZw6TB", - "genesis_time": "0001-01-01T00:00:00.000Z", - "validators": [ - { - "power": 10, - "name": "", - "pub_key": [ - 1, - "5770B4DD55B3E08B7F5711C48B516347D8C33F47C30C226315D21AA64E0DFF2E" - ] - } - ] + "validators" : [ + { + "pub_key" : { + "value" : "h3hk+QE8c6QLTySp8TcfzclJw/BG79ziGB/pIA+DfPE=", + "type" : "AC26791624DE60" + }, + "power" : 10, + "name" : "" + } + ], + "app_hash" : "", + "chain_id" : "test-chain-rDlYSN", + "genesis_time" : "0001-01-01T00:00:00Z" } And the ``priv_validator.json``: @@ -280,20 +277,18 @@ And the ``priv_validator.json``: :: { - "address": "4F4D895F882A18E1D1FC608D102601DA8D3570E5", - "last_height": 0, - "last_round": 0, - "last_signature": null, - "last_signbytes": "", - "last_step": 0, - "priv_key": [ - 1, - "F9FA3CD435BDAE54D0BCA8F1BC289D718C23D855C6DB21E8543F5E4F457E62805770B4DD55B3E08B7F5711C48B516347D8C33F47C30C226315D21AA64E0DFF2E" - ], - "pub_key": [ - 1, - "5770B4DD55B3E08B7F5711C48B516347D8C33F47C30C226315D21AA64E0DFF2E" - ] + "last_step" : 0, + "last_round" : 0, + "address" : "B788DEDE4F50AD8BC9462DE76741CCAFF87D51E2", + "pub_key" : { + "value" : "h3hk+QE8c6QLTySp8TcfzclJw/BG79ziGB/pIA+DfPE=", + "type" : "AC26791624DE60" + }, + "last_height" : 0, + "priv_key" : { + "value" : "JPivl82x+LfVkp8i3ztoTjY6c6GJ4pBxQexErOCyhwqHeGT5ATxzpAtPJKnxNx/NyUnD8Ebv3OIYH+kgD4N88Q==", + "type" : "954568A3288910" + } } The ``priv_validator.json`` actually contains a private key, and should @@ -387,20 +382,18 @@ Now we can update our genesis file. For instance, if the new :: { - "address": "AC379688105901436A34A65F185C115B8BB277A1", - "last_height": 0, - "last_round": 0, - "last_signature": null, - "last_signbytes": "", - "last_step": 0, - "priv_key": [ - 1, - "0D2ED337D748ADF79BE28559B9E59EBE1ABBA0BAFE6D65FCB9797985329B950C8F2B5AACAACC9FCE41881349743B0CFDE190DF0177744568D4E82A18F0B7DF94" - ], - "pub_key": [ - 1, - "8F2B5AACAACC9FCE41881349743B0CFDE190DF0177744568D4E82A18F0B7DF94" - ] + "address" : "5AF49D2A2D4F5AD4C7C8C4CC2FB020131E9C4902", + "pub_key" : { + "value" : "l9X9+fjkeBzDfPGbUM7AMIRE6uJN78zN5+lk5OYotek=", + "type" : "AC26791624DE60" + }, + "priv_key" : { + "value" : "EDJY9W6zlAw+su6ITgTKg2nTZcHAH1NMTW5iwlgmNDuX1f35+OR4HMN88ZtQzsAwhETq4k3vzM3n6WTk5ii16Q==", + "type" : "954568A3288910" + }, + "last_step" : 0, + "last_round" : 0, + "last_height" : 0 } then the new ``genesis.json`` will be: @@ -408,27 +401,27 @@ then the new ``genesis.json`` will be: :: { - "app_hash": "", - "chain_id": "test-chain-HZw6TB", - "genesis_time": "0001-01-01T00:00:00.000Z", - "validators": [ - { - "power": 10, - "name": "", - "pub_key": [ - 1, - "5770B4DD55B3E08B7F5711C48B516347D8C33F47C30C226315D21AA64E0DFF2E" - ] - }, - { - "power": 10, - "name": "", - "pub_key": [ - 1, - "8F2B5AACAACC9FCE41881349743B0CFDE190DF0177744568D4E82A18F0B7DF94" - ] - } - ] + "validators" : [ + { + "pub_key" : { + "value" : "h3hk+QE8c6QLTySp8TcfzclJw/BG79ziGB/pIA+DfPE=", + "type" : "AC26791624DE60" + }, + "power" : 10, + "name" : "" + }, + { + "pub_key" : { + "value" : "l9X9+fjkeBzDfPGbUM7AMIRE6uJN78zN5+lk5OYotek=", + "type" : "AC26791624DE60" + }, + "power" : 10, + "name" : "" + } + ], + "app_hash" : "", + "chain_id" : "test-chain-rDlYSN", + "genesis_time" : "0001-01-01T00:00:00Z" } Update the ``genesis.json`` in ``~/.tendermint/config``. Copy the genesis file From 0323b03dafd2f0698a0507f7e4f1aa3d53bdb0bc Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 11 Apr 2018 19:40:53 +0200 Subject: [PATCH 098/143] improve testnet cmd (#1449) * improve testnet cmd * allow non-validators * configurable prefix * populating of persistent peers * relax permissions * cleanup output dir every time * do not remove dir * remove panic comments --- cmd/tendermint/commands/init.go | 15 ++- cmd/tendermint/commands/testnet.go | 155 +++++++++++++++++++++-------- config/toml.go | 17 ++-- 3 files changed, 135 insertions(+), 52 deletions(-) diff --git a/cmd/tendermint/commands/init.go b/cmd/tendermint/commands/init.go index 2faff20fe..32559fa5c 100644 --- a/cmd/tendermint/commands/init.go +++ b/cmd/tendermint/commands/init.go @@ -3,6 +3,7 @@ package commands import ( "github.com/spf13/cobra" + cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/types" pvm "github.com/tendermint/tendermint/types/priv_validator" @@ -13,10 +14,14 @@ import ( var InitFilesCmd = &cobra.Command{ Use: "init", Short: "Initialize Tendermint", - Run: initFiles, + RunE: initFiles, } -func initFiles(cmd *cobra.Command, args []string) { +func initFiles(cmd *cobra.Command, args []string) error { + return initFilesWithConfig(config) +} + +func initFilesWithConfig(config *cfg.Config) error { // private validator privValFile := config.PrivValidatorFile() var pv *pvm.FilePV @@ -34,7 +39,7 @@ func initFiles(cmd *cobra.Command, args []string) { logger.Info("Found node key", "path", nodeKeyFile) } else { if _, err := p2p.LoadOrGenNodeKey(nodeKeyFile); err != nil { - panic(err) + return err } logger.Info("Generated node key", "path", nodeKeyFile) } @@ -53,8 +58,10 @@ func initFiles(cmd *cobra.Command, args []string) { }} if err := genDoc.SaveAs(genFile); err != nil { - panic(err) + return err } logger.Info("Generated genesis file", "path", genFile) } + + return nil } diff --git a/cmd/tendermint/commands/testnet.go b/cmd/tendermint/commands/testnet.go index eb86e4f8d..150ebe681 100644 --- a/cmd/tendermint/commands/testnet.go +++ b/cmd/tendermint/commands/testnet.go @@ -2,60 +2,103 @@ package commands import ( "fmt" + "net" + "os" "path/filepath" + "strings" "time" "github.com/spf13/cobra" cfg "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/types" pvm "github.com/tendermint/tendermint/types/priv_validator" cmn "github.com/tendermint/tmlibs/common" ) -//flags var ( - nValidators int - dataDir string + nValidators int + nNonValidators int + outputDir string + nodeDirPrefix string + + populatePersistentPeers bool + hostnamePrefix string + startingIPAddress string + p2pPort int +) + +const ( + nodeDirPerm = 0755 ) func init() { - TestnetFilesCmd.Flags().IntVar(&nValidators, "n", 4, + TestnetFilesCmd.Flags().IntVar(&nValidators, "v", 4, "Number of validators to initialize the testnet with") - TestnetFilesCmd.Flags().StringVar(&dataDir, "dir", "mytestnet", + TestnetFilesCmd.Flags().IntVar(&nNonValidators, "n", 0, + "Number of non-validators to initialize the testnet with") + TestnetFilesCmd.Flags().StringVar(&outputDir, "o", "./mytestnet", "Directory to store initialization data for the testnet") + TestnetFilesCmd.Flags().StringVar(&nodeDirPrefix, "node-dir-prefix", "node", + "Prefix the directory name for each node with (node results in node0, node1, ...)") + + TestnetFilesCmd.Flags().BoolVar(&populatePersistentPeers, "populate-persistent-peers", true, + "Update config of each node with the list of persistent peers build using either hostname-prefix or starting-ip-address") + TestnetFilesCmd.Flags().StringVar(&hostnamePrefix, "hostname-prefix", "node", + "Hostname prefix (node results in persistent peers list ID0@node0:46656, ID1@node1:46656, ...)") + TestnetFilesCmd.Flags().StringVar(&startingIPAddress, "starting-ip-address", "", + "Starting IP address (192.168.0.1 results in persistent peers list ID0@192.168.0.1:46656, ID1@192.168.0.2:46656, ...)") + TestnetFilesCmd.Flags().IntVar(&p2pPort, "p2p-port", 46656, + "P2P Port") } -// TestnetFilesCmd allows initialisation of files for a -// Tendermint testnet. +// TestnetFilesCmd allows initialisation of files for a Tendermint testnet. var TestnetFilesCmd = &cobra.Command{ Use: "testnet", Short: "Initialize files for a Tendermint testnet", - Run: testnetFiles, + RunE: testnetFiles, } -func testnetFiles(cmd *cobra.Command, args []string) { - +func testnetFiles(cmd *cobra.Command, args []string) error { + config := cfg.DefaultConfig() genVals := make([]types.GenesisValidator, nValidators) - defaultConfig := cfg.DefaultBaseConfig() - // Initialize core dir and priv_validator.json's for i := 0; i < nValidators; i++ { - mach := cmn.Fmt("mach%d", i) - err := initMachCoreDirectory(dataDir, mach) + nodeDirName := cmn.Fmt("%s%d", nodeDirPrefix, i) + nodeDir := filepath.Join(outputDir, nodeDirName) + config.SetRoot(nodeDir) + + err := os.MkdirAll(filepath.Join(nodeDir, "config"), nodeDirPerm) if err != nil { - cmn.Exit(err.Error()) + _ = os.RemoveAll(outputDir) + return err } - // Read priv_validator.json to populate vals - pvFile := filepath.Join(dataDir, mach, defaultConfig.PrivValidator) + + initFilesWithConfig(config) + + pvFile := filepath.Join(nodeDir, config.BaseConfig.PrivValidator) pv := pvm.LoadFilePV(pvFile) genVals[i] = types.GenesisValidator{ PubKey: pv.GetPubKey(), Power: 1, - Name: mach, + Name: nodeDirName, } } + for i := 0; i < nNonValidators; i++ { + nodeDir := filepath.Join(outputDir, cmn.Fmt("%s%d", nodeDirPrefix, i+nValidators)) + config.SetRoot(nodeDir) + + err := os.MkdirAll(filepath.Join(nodeDir, "config"), nodeDirPerm) + if err != nil { + _ = os.RemoveAll(outputDir) + return err + } + + initFilesWithConfig(config) + } + // Generate genesis doc from generated validators genDoc := &types.GenesisDoc{ GenesisTime: time.Now(), @@ -64,36 +107,64 @@ func testnetFiles(cmd *cobra.Command, args []string) { } // Write genesis file. - for i := 0; i < nValidators; i++ { - mach := cmn.Fmt("mach%d", i) - if err := genDoc.SaveAs(filepath.Join(dataDir, mach, defaultConfig.Genesis)); err != nil { - panic(err) + for i := 0; i < nValidators+nNonValidators; i++ { + nodeDir := filepath.Join(outputDir, cmn.Fmt("%s%d", nodeDirPrefix, i)) + if err := genDoc.SaveAs(filepath.Join(nodeDir, config.BaseConfig.Genesis)); err != nil { + _ = os.RemoveAll(outputDir) + return err } } - fmt.Println(cmn.Fmt("Successfully initialized %v node directories", nValidators)) -} - -// Initialize per-machine core directory -func initMachCoreDirectory(base, mach string) error { - // Create priv_validator.json file if not present - defaultConfig := cfg.DefaultBaseConfig() - dir := filepath.Join(base, mach) - pvPath := filepath.Join(dir, defaultConfig.PrivValidator) - dir = filepath.Dir(pvPath) - err := cmn.EnsureDir(dir, 0700) - if err != nil { - return err + if populatePersistentPeers { + err := populatePersistentPeersInConfigAndWriteIt(config) + if err != nil { + _ = os.RemoveAll(outputDir) + return err + } } - ensurePrivValidator(pvPath) + + fmt.Printf("Successfully initialized %v node directories\n", nValidators+nNonValidators) return nil - } -func ensurePrivValidator(file string) { - if cmn.FileExists(file) { - return +func hostnameOrIP(i int) string { + if startingIPAddress != "" { + ip := net.ParseIP(startingIPAddress) + ip = ip.To4() + if ip == nil { + fmt.Printf("%v: non ipv4 address\n", startingIPAddress) + os.Exit(1) + } + + ip = ip.Mask(ip.DefaultMask()) + for j := 0; j <= i; j++ { + ip[3]++ + } + return ip.String() } - pv := pvm.GenFilePV(file) - pv.Save() + + return fmt.Sprintf("%s%d", hostnamePrefix, i) +} + +func populatePersistentPeersInConfigAndWriteIt(config *cfg.Config) error { + persistentPeers := make([]string, nValidators+nNonValidators) + for i := 0; i < nValidators+nNonValidators; i++ { + nodeKey, err := p2p.LoadNodeKey(config.NodeKeyFile()) + if err != nil { + return err + } + persistentPeers[i] = p2p.IDAddressString(nodeKey.ID(), fmt.Sprintf("%s:%d", hostnameOrIP(i), p2pPort)) + } + persistentPeersList := strings.Join(persistentPeers, ",") + + for i := 0; i < nValidators+nNonValidators; i++ { + nodeDir := filepath.Join(outputDir, cmn.Fmt("%s%d", nodeDirPrefix, i)) + config.SetRoot(nodeDir) + config.P2P.PersistentPeers = persistentPeersList + + // overwrite default config + cfg.WriteConfigFile(filepath.Join(nodeDir, "config", "config.toml"), config) + } + + return nil } diff --git a/config/toml.go b/config/toml.go index f10c08ab3..71b8b2e81 100644 --- a/config/toml.go +++ b/config/toml.go @@ -37,16 +37,21 @@ func EnsureRoot(rootDir string) { // Write default config file if missing. if !cmn.FileExists(configFilePath) { - writeConfigFile(configFilePath) + writeDefaultCondigFile(configFilePath) } } // XXX: this func should probably be called by cmd/tendermint/commands/init.go // alongside the writing of the genesis.json and priv_validator.json -func writeConfigFile(configFilePath string) { +func writeDefaultCondigFile(configFilePath string) { + WriteConfigFile(configFilePath, DefaultConfig()) +} + +// WriteConfigFile renders config using the template and writes it to configFilePath. +func WriteConfigFile(configFilePath string, config *Config) { var buffer bytes.Buffer - if err := configTemplate.Execute(&buffer, DefaultConfig()); err != nil { + if err := configTemplate.Execute(&buffer, config); err != nil { panic(err) } @@ -124,11 +129,11 @@ unsafe = {{ .RPC.Unsafe }} laddr = "{{ .P2P.ListenAddress }}" # Comma separated list of seed nodes to connect to -seeds = "" +seeds = "{{ .P2P.Seeds }}" # Comma separated list of nodes to keep persistent connections to # Do not add private peers to this list if you don't want them advertised -persistent_peers = "" +persistent_peers = "{{ .P2P.PersistentPeers }}" # Path to address book addr_book_file = "{{ .P2P.AddrBook }}" @@ -262,7 +267,7 @@ func ResetTestRoot(testName string) *Config { // Write default config file if missing. if !cmn.FileExists(configFilePath) { - writeConfigFile(configFilePath) + writeDefaultCondigFile(configFilePath) } if !cmn.FileExists(genesisFilePath) { cmn.MustWriteFile(genesisFilePath, []byte(testGenesis), 0644) From 1db222424192814719030ae03d76e4ddd7884c48 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 11 Apr 2018 20:53:33 +0200 Subject: [PATCH 099/143] do not use mask in testnet cmd (#1451) fix node ids --- cmd/tendermint/commands/testnet.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/tendermint/commands/testnet.go b/cmd/tendermint/commands/testnet.go index 150ebe681..ef503db3f 100644 --- a/cmd/tendermint/commands/testnet.go +++ b/cmd/tendermint/commands/testnet.go @@ -136,8 +136,7 @@ func hostnameOrIP(i int) string { os.Exit(1) } - ip = ip.Mask(ip.DefaultMask()) - for j := 0; j <= i; j++ { + for j := 0; j < i; j++ { ip[3]++ } return ip.String() @@ -149,6 +148,8 @@ func hostnameOrIP(i int) string { func populatePersistentPeersInConfigAndWriteIt(config *cfg.Config) error { persistentPeers := make([]string, nValidators+nNonValidators) for i := 0; i < nValidators+nNonValidators; i++ { + nodeDir := filepath.Join(outputDir, cmn.Fmt("%s%d", nodeDirPrefix, i)) + config.SetRoot(nodeDir) nodeKey, err := p2p.LoadNodeKey(config.NodeKeyFile()) if err != nil { return err From c0610b2c3273144be09db263fe073477e7de56f3 Mon Sep 17 00:00:00 2001 From: Greg Szabo <16846635+greg-szabo@users.noreply.github.com> Date: Thu, 12 Apr 2018 13:15:16 +0200 Subject: [PATCH 100/143] Greg/localnet (#1450) * Added new Makefile targets for local testnet running using docker * Added localnode docker image description, some documentation and refactored to use the tendermint testnet command * Fixes for the new tendermint testnet command * More fixes on tendermint testnet and docker-compose * Changed logging * Added missing targets to phony --- Makefile | 20 ++++++++- docker-compose.yml | 68 +++++++++++++++++++++++++++++ docker-compose/Makefile | 7 +++ docker-compose/README.rst | 40 +++++++++++++++++ docker-compose/localnode/Dockerfile | 19 ++++++++ docker-compose/localnode/wrapper.sh | 33 ++++++++++++++ 6 files changed, 186 insertions(+), 1 deletion(-) create mode 100644 docker-compose.yml create mode 100644 docker-compose/Makefile create mode 100644 docker-compose/README.rst create mode 100644 docker-compose/localnode/Dockerfile create mode 100755 docker-compose/localnode/wrapper.sh diff --git a/Makefile b/Makefile index 3e51c834e..333553e53 100644 --- a/Makefile +++ b/Makefile @@ -178,7 +178,25 @@ metalinter_all: @echo "--> Running linter (all)" gometalinter.v2 --vendor --deadline=600s --enable-all --disable=lll ./... +########################################################### +### Local testnet using docker + +# Build linux binary on other platforms +build-linux: + GOOS=linux GOARCH=amd64 $(MAKE) build + +# Run a 4-node testnet locally +docker-start: + @echo "Wait until 'Attaching to node0, node1, node2, node3' message appears" + @if ! [ -f build/node0/config/genesis.json ]; then docker run --rm -v `pwd`/build:/tendermint tendermint/localnode testnet --v 4 --o build --populate-persistent-peers --starting-ip-address 10.100.0.2 ; fi + docker-compose up + +# Stop testnet +docker-stop: + docker-compose down + # To avoid unintended conflicts with file names, always add to .PHONY # unless there is a reason not to. # https://www.gnu.org/software/make/manual/html_node/Phony-Targets.html -.PHONY: check build build_race dist install check_tools get_tools update_tools get_vendor_deps draw_deps test_cover test_apps test_persistence test_p2p test test_race test_integrations test_release test100 vagrant_test fmt +.PHONY: check build build_race dist install check_tools get_tools update_tools get_vendor_deps draw_deps test_cover test_apps test_persistence test_p2p test test_race test_integrations test_release test100 vagrant_test fmt build-linux docker-start docker-stop + diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..cc76d8dbf --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,68 @@ +version: '3' + +services: + node0: + container_name: node0 + image: "tendermint/localnode" + ports: + - "46656-46657:46656-46657" + environment: + - ID=0 + - LOG=${LOG:-tendermint.log} + volumes: + - ${FOLDER:-./build}:/tendermint + networks: + localnet: + ipv4_address: 10.100.0.2 + + node1: + container_name: node1 + image: "tendermint/localnode" + ports: + - "46659-46660:46656-46657" + environment: + - ID=1 + - LOG=${LOG:-tendermint.log} + volumes: + - ${FOLDER:-./build}:/tendermint + networks: + localnet: + ipv4_address: 10.100.0.3 + + node2: + container_name: node2 + image: "tendermint/localnode" + environment: + - ID=2 + - LOG=${LOG:-tendermint.log} + ports: + - "46661-46662:46656-46657" + volumes: + - ${FOLDER:-./build}:/tendermint + networks: + localnet: + ipv4_address: 10.100.0.4 + + node3: + container_name: node3 + image: "tendermint/localnode" + environment: + - ID=3 + - LOG=${LOG:-tendermint.log} + ports: + - "46663-46664:46656-46657" + volumes: + - ${FOLDER:-./build}:/tendermint + networks: + localnet: + ipv4_address: 10.100.0.5 + +networks: + localnet: + driver: bridge + ipam: + driver: default + config: + - + subnet: 10.100.0.0/16 + diff --git a/docker-compose/Makefile b/docker-compose/Makefile new file mode 100644 index 000000000..98517851d --- /dev/null +++ b/docker-compose/Makefile @@ -0,0 +1,7 @@ +# Makefile for the "localnode" docker image. + +all: + docker build --tag tendermint/localnode localnode + +.PHONY: all + diff --git a/docker-compose/README.rst b/docker-compose/README.rst new file mode 100644 index 000000000..d22a24d9b --- /dev/null +++ b/docker-compose/README.rst @@ -0,0 +1,40 @@ +localnode +========= + +It is assumed that you have already `setup docker `__. + +Description +----------- +Image for local testnets. + +Add the tendermint binary to the image by attaching it in a folder to the `/tendermint` mount point. + +It assumes that the configuration was created by the `tendermint testnet` command and it is also attached to the `/tendermint` mount point. + +Example: +This example builds a linux tendermint binary under the `build/` folder, creates tendermint configuration for a single-node validator and runs the node: +``` +cd $GOPATH/src/github.com/tendermint/tendermint + +#Build binary +make build-linux + +#Create configuration +docker run -e LOG="stdout" -v `pwd`/build:/tendermint tendermint/localnode testnet --o . --v 1 + +#Run the node +docker run -v `pwd`/build:/tendermint tendermint/localnode +``` + +Logging +------- +Log is saved under the attached volume, in the `tendermint.log` file. If the `LOG` environment variable is set to `stdout` at start, the log is not saved, but printed on the screen. + +Special binaries +---------------- +If you have multiple binaries with different names, you can specify which one to run with the BINARY environment variable. The path of the binary is relative to the attached volume. + +docker-compose.yml +================== +This file creates a 4-node network using the localnode image. The nodes of the network are exposed to the host machine on ports 46656-46657, 46659-46660, 46661-46662, 46663-46664 respectively. + diff --git a/docker-compose/localnode/Dockerfile b/docker-compose/localnode/Dockerfile new file mode 100644 index 000000000..9e5c4f8e2 --- /dev/null +++ b/docker-compose/localnode/Dockerfile @@ -0,0 +1,19 @@ +FROM alpine:3.7 +MAINTAINER Greg Szabo + +RUN apk update && \ + apk upgrade && \ + apk --no-cache add curl jq file && \ + addgroup tmuser && \ + adduser -S -G tmuser tmuser -h /tendermint + +USER tmuser +VOLUME [ /tendermint ] +WORKDIR /tendermint +EXPOSE 46656 46657 +ENTRYPOINT ["/usr/bin/wrapper.sh"] +CMD ["node", "--proxy_app dummy"] +STOPSIGNAL SIGTERM + +COPY --chown=tmuser:tmuser wrapper.sh /usr/bin/wrapper.sh + diff --git a/docker-compose/localnode/wrapper.sh b/docker-compose/localnode/wrapper.sh new file mode 100755 index 000000000..8b55cbc10 --- /dev/null +++ b/docker-compose/localnode/wrapper.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env sh + +## +## Input parameters +## +BINARY=/tendermint/${BINARY:-tendermint} +ID=${ID:-0} +LOG=${LOG:-tendermint.log} + +## +## Assert linux binary +## +if ! [ -f "${BINARY}" ]; then + echo "The binary `basename ${BINARY}` cannot be found. Please add the binary to the shared folder. Please use the BINARY environment variable if the name of the binary is not 'tendermint' E.g.: -e BINARY=tendermint_my_test_version" + exit 1 +fi +BINARY_CHECK="`file $BINARY | grep 'ELF 64-bit LSB executable, x86-64'`" +if [ -z "${BINARY_CHECK}" ]; then + echo "Binary needs to be OS linux, ARCH amd64" + exit 1 +fi + +## +## Run binary with all parameters +## +export TMHOME="/tendermint/node${ID}" + +if [ -d "${TMHOME}/${LOG}" ]; then + "$BINARY" $@ | tee "${TMHOME}/${LOG}" +else + "$BINARY" $@ +fi + From 5babaf9a882a1040d71de4582d66c69257a96c7d Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 12 Apr 2018 13:42:34 +0200 Subject: [PATCH 101/143] [localnet] fix folder permissions errors --- Makefile | 2 +- docker-compose.yml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 333553e53..c2b6ac7b7 100644 --- a/Makefile +++ b/Makefile @@ -188,7 +188,7 @@ build-linux: # Run a 4-node testnet locally docker-start: @echo "Wait until 'Attaching to node0, node1, node2, node3' message appears" - @if ! [ -f build/node0/config/genesis.json ]; then docker run --rm -v `pwd`/build:/tendermint tendermint/localnode testnet --v 4 --o build --populate-persistent-peers --starting-ip-address 10.100.0.2 ; fi + @if ! [ -f build/node0/config/genesis.json ]; then docker run --rm -v `pwd`/build:/tendermint:Z tendermint/localnode testnet --v 4 --o . --populate-persistent-peers --starting-ip-address 10.100.0.2 ; fi docker-compose up # Stop testnet diff --git a/docker-compose.yml b/docker-compose.yml index cc76d8dbf..f474be961 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,7 @@ services: - ID=0 - LOG=${LOG:-tendermint.log} volumes: - - ${FOLDER:-./build}:/tendermint + - ${FOLDER:-./build}:/tendermint:Z networks: localnet: ipv4_address: 10.100.0.2 @@ -24,7 +24,7 @@ services: - ID=1 - LOG=${LOG:-tendermint.log} volumes: - - ${FOLDER:-./build}:/tendermint + - ${FOLDER:-./build}:/tendermint:Z networks: localnet: ipv4_address: 10.100.0.3 @@ -38,7 +38,7 @@ services: ports: - "46661-46662:46656-46657" volumes: - - ${FOLDER:-./build}:/tendermint + - ${FOLDER:-./build}:/tendermint:Z networks: localnet: ipv4_address: 10.100.0.4 @@ -52,7 +52,7 @@ services: ports: - "46663-46664:46656-46657" volumes: - - ${FOLDER:-./build}:/tendermint + - ${FOLDER:-./build}:/tendermint:Z networks: localnet: ipv4_address: 10.100.0.5 From f8ed5783254ef98b2622e45495ce2784c288964a Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 12 Apr 2018 13:44:13 +0200 Subject: [PATCH 102/143] [localnet] execute cmd from root not secure, but we don't care because it's local tooling --- docker-compose/localnode/Dockerfile | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/docker-compose/localnode/Dockerfile b/docker-compose/localnode/Dockerfile index 9e5c4f8e2..7eeba963d 100644 --- a/docker-compose/localnode/Dockerfile +++ b/docker-compose/localnode/Dockerfile @@ -3,11 +3,8 @@ MAINTAINER Greg Szabo RUN apk update && \ apk upgrade && \ - apk --no-cache add curl jq file && \ - addgroup tmuser && \ - adduser -S -G tmuser tmuser -h /tendermint + apk --no-cache add curl jq file -USER tmuser VOLUME [ /tendermint ] WORKDIR /tendermint EXPOSE 46656 46657 @@ -15,5 +12,5 @@ ENTRYPOINT ["/usr/bin/wrapper.sh"] CMD ["node", "--proxy_app dummy"] STOPSIGNAL SIGTERM -COPY --chown=tmuser:tmuser wrapper.sh /usr/bin/wrapper.sh +COPY wrapper.sh /usr/bin/wrapper.sh From 3a0edc561d8a2144a10e3da37af48b831933d128 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 12 Apr 2018 15:20:28 +0200 Subject: [PATCH 103/143] log error from AddrBook#AddAddress in DialPeersAsync Refs #1434 --- p2p/switch.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/p2p/switch.go b/p2p/switch.go index be7c0ec7a..6afc30fe4 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -359,7 +359,9 @@ func (sw *Switch) DialPeersAsync(addrBook AddrBook, peers []string, persistent b for _, netAddr := range netAddrs { // do not add our address or ID if !netAddr.Same(ourAddr) { - addrBook.AddAddress(netAddr, ourAddr) + if err := addrBook.AddAddress(netAddr, ourAddr); err != nil { + sw.Logger.Error("Can't add peer's address to addrbook", "err", err) + } } } // Persist some peers to disk right away. From d06390638db7efc870dbf8c7d3bd57bae12dabb5 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 12 Apr 2018 15:33:32 +0200 Subject: [PATCH 104/143] [localnet] use routable IPs --- Makefile | 2 +- docker-compose.yml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index c2b6ac7b7..da94627d3 100644 --- a/Makefile +++ b/Makefile @@ -188,7 +188,7 @@ build-linux: # Run a 4-node testnet locally docker-start: @echo "Wait until 'Attaching to node0, node1, node2, node3' message appears" - @if ! [ -f build/node0/config/genesis.json ]; then docker run --rm -v `pwd`/build:/tendermint:Z tendermint/localnode testnet --v 4 --o . --populate-persistent-peers --starting-ip-address 10.100.0.2 ; fi + @if ! [ -f build/node0/config/genesis.json ]; then docker run --rm -v `pwd`/build:/tendermint:Z tendermint/localnode testnet --v 4 --o . --populate-persistent-peers --starting-ip-address 192.167.10.2 ; fi docker-compose up # Stop testnet diff --git a/docker-compose.yml b/docker-compose.yml index f474be961..b9a3fc684 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,7 +13,7 @@ services: - ${FOLDER:-./build}:/tendermint:Z networks: localnet: - ipv4_address: 10.100.0.2 + ipv4_address: 192.167.10.2 node1: container_name: node1 @@ -27,7 +27,7 @@ services: - ${FOLDER:-./build}:/tendermint:Z networks: localnet: - ipv4_address: 10.100.0.3 + ipv4_address: 192.167.10.3 node2: container_name: node2 @@ -41,7 +41,7 @@ services: - ${FOLDER:-./build}:/tendermint:Z networks: localnet: - ipv4_address: 10.100.0.4 + ipv4_address: 192.167.10.4 node3: container_name: node3 @@ -55,7 +55,7 @@ services: - ${FOLDER:-./build}:/tendermint:Z networks: localnet: - ipv4_address: 10.100.0.5 + ipv4_address: 192.167.10.5 networks: localnet: @@ -64,5 +64,5 @@ networks: driver: default config: - - subnet: 10.100.0.0/16 + subnet: 192.167.10.0/16 From d0beaba7e8a5652506a34b5fab299cc2dc274c02 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Fri, 13 Apr 2018 01:19:22 -0700 Subject: [PATCH 105/143] Bump version to 0.19.0 --- Gopkg.lock | 14 +++++++------- Gopkg.toml | 6 ++---- version/version.go | 2 +- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index c4d798709..0701a1faa 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -191,8 +191,8 @@ [[projects]] name = "github.com/spf13/pflag" packages = ["."] - revision = "e57e3eeb33f795204c1ca35f56c44f83227c6e66" - version = "v1.0.0" + revision = "583c0c0531f06d5278b7d917446061adc344b5cd" + version = "v1.0.1" [[projects]] name = "github.com/spf13/viper" @@ -270,7 +270,6 @@ version = "v0.7.3" [[projects]] - branch = "develop" name = "github.com/tendermint/tmlibs" packages = [ "autofile", @@ -286,7 +285,8 @@ "pubsub/query", "test" ] - revision = "357648b8d63732df77fb1dd0f5ad813800912854" + revision = "97e1f1ad3f510048929a51475811a18686c894df" + version = "0.8.2-rc0" [[projects]] branch = "master" @@ -301,7 +301,7 @@ "ripemd160", "salsa20/salsa" ] - revision = "b2aa35443fbc700ab74c586ae79b81c171851023" + revision = "d6449816ce06963d9d136eee5a56fca5b0616e7e" [[projects]] branch = "master" @@ -348,7 +348,7 @@ branch = "master" name = "google.golang.org/genproto" packages = ["googleapis/rpc/status"] - revision = "ce84044298496ef4b54b4a0a0909ba593cc60e30" + revision = "51d0944304c3cbce4afe9e5247e21100037bff78" [[projects]] name = "google.golang.org/grpc" @@ -383,6 +383,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "794125e6cc5926371340043eed0ce8731a2212367cc6d4b845e4618ef2dd1996" + inputs-digest = "e70f8692c825e80ae8510546e297840b9560d00e11b2272749a55cc2ffd147f0" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 4bbcefa20..8e65a41a3 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -81,11 +81,9 @@ name = "github.com/tendermint/go-amino" version = "~0.9.6" -[[override]] -# [[constraint]] +[[constraint]] name = "github.com/tendermint/tmlibs" - # version = "~0.8.1" - branch = "develop" + version = "~0.8.2-rc0" [[constraint]] name = "google.golang.org/grpc" diff --git a/version/version.go b/version/version.go index 8859b903c..b0123eb1f 100644 --- a/version/version.go +++ b/version/version.go @@ -7,7 +7,7 @@ const Fix = "0" var ( // Version is the current version of Tendermint // Must be a string because scripts like dist.sh read this file. - Version = "0.19.0-dev" + Version = "0.19.0" // GitCommit is the current HEAD set using ldflags. GitCommit string From 1706ce6f7fd7da9780c2ec1b7e68ac99894cd627 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Fri, 13 Apr 2018 10:49:05 +0200 Subject: [PATCH 106/143] update changelog for 0.19.0 release --- CHANGELOG.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a462a993..967fdef65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,11 +24,12 @@ BUG FIXES: - Graceful handling/recovery for apps that have non-determinism or fail to halt - Graceful handling/recovery for violations of safety, or liveness -## 0.19.0 (TBD) +## 0.19.0 (April 13th, 2018) BREAKING: - -- [rpc]: Changed the output format for the `/status` endpoint +- [cmd] improved `testnet` command; now it can fill in `persistent_peers` for you in the config file and much more (see `tendermint testnet --help` for details) +- [cmd] `show_node_id` now returns an error if there is no node key +- [rpc]: changed the output format for the `/status` endpoint (see https://godoc.org/github.com/tendermint/tendermint/rpc/core#Status) Upgrade from go-wire to go-amino. This is a sweeping change that breaks everything that is serialized to disk or over the network. @@ -38,6 +39,9 @@ See github.com/tendermint/go-amino for details on the new format. See `scripts/wire2amino.go` for a tool to upgrade genesis/priv_validator/node_key JSON files. +FEATURES: +- [cmd] added `gen_node_key` command + ## 0.18.0 (April 6th, 2018) BREAKING: From f2dae2a2d877620fc958b9b4976872301d1e1fd9 Mon Sep 17 00:00:00 2001 From: Greg Szabo Date: Fri, 13 Apr 2018 21:03:25 -0400 Subject: [PATCH 107/143] Moved to networks folder, introduced cloud server scripts using terraform and ansible (sentry nodes) --- Makefile | 22 +- {docker-compose => networks/local}/Makefile | 0 {docker-compose => networks/local}/README.rst | 0 .../local}/localnode/Dockerfile | 0 .../local}/localnode/wrapper.sh | 0 networks/remote/ansible/.gitignore | 1 + networks/remote/ansible/README.rst | 291 ++++++++ networks/remote/ansible/ansible.cfg | 4 + networks/remote/ansible/assets/a_plus_t.png | Bin 0 -> 13830 bytes networks/remote/ansible/install.yml | 11 + networks/remote/ansible/inventory/COPYING | 675 ++++++++++++++++++ .../ansible/inventory/digital_ocean.ini | 34 + .../remote/ansible/inventory/digital_ocean.py | 471 ++++++++++++ networks/remote/ansible/reconfig.yml | 16 + networks/remote/ansible/reset.yml | 14 + networks/remote/ansible/restart.yml | 12 + .../ansible/roles/config/tasks/main.yml | 15 + .../ansible/roles/install/handlers/main.yml | 5 + .../ansible/roles/install/tasks/main.yml | 15 + .../install/templates/systemd.service.j2 | 17 + .../remote/ansible/roles/start/tasks/main.yml | 5 + .../ansible/roles/status/tasks/main.yml | 10 + .../remote/ansible/roles/stop/tasks/main.yml | 5 + .../ansible/roles/unsafe_reset/tasks/main.yml | 3 + networks/remote/ansible/start.yml | 11 + networks/remote/ansible/status.yml | 11 + networks/remote/ansible/stop.yml | 11 + networks/remote/terraform/.gitignore | 4 + networks/remote/terraform/README.rst | 33 + networks/remote/terraform/cluster/main.tf | 28 + networks/remote/terraform/cluster/outputs.tf | 15 + .../remote/terraform/cluster/variables.tf | 25 + networks/remote/terraform/main.tf | 37 + 33 files changed, 1799 insertions(+), 2 deletions(-) rename {docker-compose => networks/local}/Makefile (100%) rename {docker-compose => networks/local}/README.rst (100%) rename {docker-compose => networks/local}/localnode/Dockerfile (100%) rename {docker-compose => networks/local}/localnode/wrapper.sh (100%) create mode 100644 networks/remote/ansible/.gitignore create mode 100644 networks/remote/ansible/README.rst create mode 100644 networks/remote/ansible/ansible.cfg create mode 100644 networks/remote/ansible/assets/a_plus_t.png create mode 100644 networks/remote/ansible/install.yml create mode 100644 networks/remote/ansible/inventory/COPYING create mode 100644 networks/remote/ansible/inventory/digital_ocean.ini create mode 100755 networks/remote/ansible/inventory/digital_ocean.py create mode 100644 networks/remote/ansible/reconfig.yml create mode 100644 networks/remote/ansible/reset.yml create mode 100644 networks/remote/ansible/restart.yml create mode 100644 networks/remote/ansible/roles/config/tasks/main.yml create mode 100644 networks/remote/ansible/roles/install/handlers/main.yml create mode 100644 networks/remote/ansible/roles/install/tasks/main.yml create mode 100644 networks/remote/ansible/roles/install/templates/systemd.service.j2 create mode 100644 networks/remote/ansible/roles/start/tasks/main.yml create mode 100644 networks/remote/ansible/roles/status/tasks/main.yml create mode 100644 networks/remote/ansible/roles/stop/tasks/main.yml create mode 100644 networks/remote/ansible/roles/unsafe_reset/tasks/main.yml create mode 100644 networks/remote/ansible/start.yml create mode 100644 networks/remote/ansible/status.yml create mode 100644 networks/remote/ansible/stop.yml create mode 100644 networks/remote/terraform/.gitignore create mode 100644 networks/remote/terraform/README.rst create mode 100644 networks/remote/terraform/cluster/main.tf create mode 100644 networks/remote/terraform/cluster/outputs.tf create mode 100644 networks/remote/terraform/cluster/variables.tf create mode 100644 networks/remote/terraform/main.tf diff --git a/Makefile b/Makefile index da94627d3..5e5b96754 100644 --- a/Makefile +++ b/Makefile @@ -187,14 +187,32 @@ build-linux: # Run a 4-node testnet locally docker-start: - @echo "Wait until 'Attaching to node0, node1, node2, node3' message appears" - @if ! [ -f build/node0/config/genesis.json ]; then docker run --rm -v `pwd`/build:/tendermint:Z tendermint/localnode testnet --v 4 --o . --populate-persistent-peers --starting-ip-address 192.167.10.2 ; fi + @if ! [ -f build/node0/config/genesis.json ]; then docker run --rm -v $(CURDIR)/build:/tendermint:Z tendermint/localnode testnet --v 4 --o . --populate-persistent-peers --starting-ip-address 192.167.10.2 ; fi docker-compose up # Stop testnet docker-stop: docker-compose down +########################################################### +### Remote full-nodes (sentry) using terraform and ansible + +# Server management +server-setup: + @if ! [ -f $(HOME)/.ssh/id_rsa.pub ]; then ssh-keygen ; fi + cd networks/remote/terraform && terraform init && terraform apply -var DO_API_TOKEN="$(DO_API_TOKEN)" -var SSH_KEY_FILE="$(HOME)/.ssh/id_rsa.pub" +# @if ! [ -f $(CURDIR)/build/node0/config/genesis.json ]; then docker run --rm -v $(CURDIR)/build:/tendermint:Z tendermint/localnode testnet --n 4 --o . ; fi + build/tendermint testnet --n 4 --o build/ + cd networks/remote/ansible && ansible-playbook -i inventory/digital_ocean.py -l remotenet --ssh-common-args '-o StrictHostKeyChecking=False' install.yml + $(MAKE) server-config + +server-destroy: + cd networks/remote/terraform && terraform destroy + +# Configuration management +server-config: + cd networks/remote/ansible && ansible-playbook -i inventory/digital_ocean.py -l remotenet reconfig.yml -e BINARY=$(CURDIR)/build/tendermint -e CONFIGDIR=$(CURDIR)/build + # To avoid unintended conflicts with file names, always add to .PHONY # unless there is a reason not to. # https://www.gnu.org/software/make/manual/html_node/Phony-Targets.html diff --git a/docker-compose/Makefile b/networks/local/Makefile similarity index 100% rename from docker-compose/Makefile rename to networks/local/Makefile diff --git a/docker-compose/README.rst b/networks/local/README.rst similarity index 100% rename from docker-compose/README.rst rename to networks/local/README.rst diff --git a/docker-compose/localnode/Dockerfile b/networks/local/localnode/Dockerfile similarity index 100% rename from docker-compose/localnode/Dockerfile rename to networks/local/localnode/Dockerfile diff --git a/docker-compose/localnode/wrapper.sh b/networks/local/localnode/wrapper.sh similarity index 100% rename from docker-compose/localnode/wrapper.sh rename to networks/local/localnode/wrapper.sh diff --git a/networks/remote/ansible/.gitignore b/networks/remote/ansible/.gitignore new file mode 100644 index 000000000..a8b42eb6e --- /dev/null +++ b/networks/remote/ansible/.gitignore @@ -0,0 +1 @@ +*.retry diff --git a/networks/remote/ansible/README.rst b/networks/remote/ansible/README.rst new file mode 100644 index 000000000..5c416c018 --- /dev/null +++ b/networks/remote/ansible/README.rst @@ -0,0 +1,291 @@ +Using Ansible +============= + +.. figure:: assets/a_plus_t.png + :alt: Ansible plus Tendermint + + Ansible plus Tendermint + +The playbooks in `our ansible directory `__ +run ansible `roles `__ which: + +- install and configure basecoind or ethermint +- start/stop basecoind or ethermint and reset their configuration + +Prerequisites +------------- + +- Ansible 2.0 or higher +- SSH key to the servers + +Optional for DigitalOcean droplets: + +- DigitalOcean API Token +- python dopy package + +For a description on how to get a DigitalOcean API Token, see the explanation +in the `using terraform tutorial <./terraform-digitalocean.html>`__. + +Optional for Amazon AWS instances: + +- Amazon AWS API access key ID and secret access key. + +The cloud inventory scripts come from the ansible team at their +`GitHub `__ page. You can get the +latest version from the ``contrib/inventory`` folder. + +Setup +----- + +Ansible requires a "command machine" or "local machine" or "orchestrator +machine" to run on. This can be your laptop or any machine that can run +ansible. (It does not have to be part of the cloud network that hosts +your servers.) + +Use the official `Ansible installation +guide `__ to +install Ansible. Here are a few examples on basic installation commands: + +Ubuntu/Debian: + +:: + + sudo apt-get install ansible + +CentOS/RedHat: + +:: + + sudo yum install epel-release + sudo yum install ansible + +Mac OSX: If you have `Homebrew `__ installed, then it's: + +:: + + brew install ansible + +If not, you can install it using ``pip``: + +:: + + sudo easy_install pip + sudo pip install ansible + +To make life easier, you can start an SSH Agent and load your SSH +key(s). This way ansible will have an uninterrupted way of connecting to +your servers. + +:: + + ssh-agent > ~/.ssh/ssh.env + source ~/.ssh/ssh.env + + ssh-add private.key + +Subsequently, as long as the agent is running, you can use +``source ~/.ssh/ssh.env`` to load the keys to the current session. Note: +On Mac OSX, you can add the ``-K`` option to ssh-add to store the +passphrase in your keychain. The security of this feature is debated but +it is convenient. + +Optional cloud dependencies +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you are using a cloud provider to host your servers, you need the +below dependencies installed on your local machine. + +DigitalOcean inventory dependencies: +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Ubuntu/Debian: + +:: + + sudo apt-get install python-pip + sudo pip install dopy + +CentOS/RedHat: + +:: + + sudo yum install python-pip + sudo pip install dopy + +Mac OSX: + +:: + + sudo pip install dopy + +Amazon AWS inventory dependencies: +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Ubuntu/Debian: + +:: + + sudo apt-get install python-boto + +CentOS/RedHat: + +:: + + sudo yum install python-boto + +Mac OSX: + +:: + + sudo pip install boto + +Refreshing the DigitalOcean inventory +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you just finished creating droplets, the local DigitalOcean inventory +cache is not up-to-date. To refresh it, run: + +:: + + DO_API_TOKEN="" + python -u inventory/digital_ocean.py --refresh-cache 1> /dev/null + +Refreshing the Amazon AWS inventory +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you just finished creating Amazon AWS EC2 instances, the local AWS +inventory cache is not up-to-date. To refresh it, run: + +:: + + AWS_ACCESS_KEY_ID='' + AWS_SECRET_ACCESS_KEY='' + python -u inventory/ec2.py --refresh-cache 1> /dev/null + +Note: you don't need the access key and secret key set, if you are +running ansible on an Amazon AMI instance with the proper IAM +permissions set. + +Running the playbooks +--------------------- + +The playbooks are locked down to only run if the environment variable +``TF_VAR_TESTNET_NAME`` is populated. This is a precaution so you don't +accidentally run the playbook on all your servers. + +The variable ``TF_VAR_TESTNET_NAME`` contains the testnet name which +ansible translates into an ansible group. If you used Terraform to +create the servers, it was the testnet name used there. + +If the playbook cannot connect to the servers because of public key +denial, your SSH Agent is not set up properly. Alternatively you can add +the SSH key to ansible using the ``--private-key`` option. + +If you need to connect to the nodes as root but your local username is +different, use the ansible option ``-u root`` to tell ansible to connect +to the servers and authenticate as the root user. + +If you secured your server and you need to ``sudo`` for root access, use +the the ``-b`` or ``--become`` option to tell ansible to sudo to root +after connecting to the server. In the Terraform-DigitalOcean example, +if you created the ec2-user by adding the ``noroot=true`` option (or if +you are simply on Amazon AWS), you need to add the options +``-u ec2-user -b`` to ansible to tell it to connect as the ec2-user and +then sudo to root to run the playbook. + +DigitalOcean +~~~~~~~~~~~~ + +:: + + DO_API_TOKEN="" + TF_VAR_TESTNET_NAME="testnet-servers" + ansible-playbook -i inventory/digital_ocean.py install.yml -e service=basecoind + +Amazon AWS +~~~~~~~~~~ + +:: + + AWS_ACCESS_KEY_ID='' + AWS_SECRET_ACCESS_KEY='' + TF_VAR_TESTNET_NAME="testnet-servers" + ansible-playbook -i inventory/ec2.py install.yml -e service=basecoind + +Installing custom versions +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +By default ansible installs the tendermint, basecoind or ethermint binary +versions from the latest release in the repository. If you build your +own version of the binaries, you can tell ansible to install that +instead. + +:: + + GOPATH="" + go get -u github.com/tendermint/basecoin/cmd/basecoind + + DO_API_TOKEN="" + TF_VAR_TESTNET_NAME="testnet-servers" + ansible-playbook -i inventory/digital_ocean.py install.yml -e service=basecoind -e release_install=false + +Alternatively you can change the variable settings in +``group_vars/all``. + +Other commands and roles +------------------------ + +There are few extra playbooks to make life easier managing your servers. + +- install.yml - Install basecoind or ethermint applications. (Tendermint + gets installed automatically.) Use the ``service`` parameter to + define which application to install. Defaults to ``basecoind``. +- reset.yml - Stop the application, reset the configuration and data, + then start the application again. You need to pass + ``-e service=``, like ``-e service=basecoind``. It will + restart the underlying tendermint application too. +- restart.yml - Restart a service on all nodes. You need to pass + ``-e service=``, like ``-e service=basecoind``. It will + restart the underlying tendermint application too. +- stop.yml - Stop the application. You need to pass + ``-e service=``. +- status.yml - Check the service status and print it. You need to pass + ``-e service=``. +- start.yml - Start the application. You need to pass + ``-e service=``. +- ubuntu16-patch.yml - Ubuntu 16.04 does not have the minimum required + python package installed to be able to run ansible. If you are using + ubuntu, run this playbook first on the target machines. This will + install the python pacakge that is required for ansible to work + correctly on the remote nodes. +- upgrade.yml - Upgrade the ``service`` on your testnet. It will stop + the service and restart it at the end. It will only work if the + upgraded version is backward compatible with the installed version. +- upgrade-reset.yml - Upgrade the ``service`` on your testnet and reset + the database. It will stop the service and restart it at the end. It + will work for upgrades where the new version is not + backward-compatible with the installed version - however it will + reset the testnet to its default. + +The roles are self-sufficient under the ``roles/`` folder. + +- install - install the application defined in the ``service`` + parameter. It can install release packages and update them with + custom-compiled binaries. +- unsafe\_reset - delete the database for a service, including the + tendermint database. +- config - configure the application defined in ``service``. It also + configures the underlying tendermint service. Check + ``group_vars/all`` for options. +- stop - stop an application. Requires the ``service`` parameter set. +- status - check the status of an application. Requires the ``service`` + parameter set. +- start - start an application. Requires the ``service`` parameter set. + +Default variables +----------------- + +Default variables are documented under ``group_vars/all``. You can the +parameters there to deploy a previously created genesis.json file +(instead of dynamically creating it) or if you want to deploy custom +built binaries instead of deploying a released version. diff --git a/networks/remote/ansible/ansible.cfg b/networks/remote/ansible/ansible.cfg new file mode 100644 index 000000000..045c1ea60 --- /dev/null +++ b/networks/remote/ansible/ansible.cfg @@ -0,0 +1,4 @@ +[defaults] +retry_files_enabled = False +host_key_checking = False + diff --git a/networks/remote/ansible/assets/a_plus_t.png b/networks/remote/ansible/assets/a_plus_t.png new file mode 100644 index 0000000000000000000000000000000000000000..8f5bc5e950cb4ee31a0983f004f780aeb398c6ea GIT binary patch literal 13830 zcmaL8b95%bw>BEv<{M3Bl8J5Gwr#($J<-J08(R}w6HRQ}&cx2mobUY3cmKHO_Uf*# zs$Or@oU|?X#Qj(&|pZDj_%M=dgv#bfAL;1Yn1I09fDh}pA zcOz#rFd41IJ0lA-WiumFPp1(xUNA7oFIK9WKutMWZW9MPdZT|b^d5GOpJ*^J zUI7nBBNH1lAgQsLg_S)Y`DJ@IIjNN?AGrpb9HX40h?%98q?faqikG~qiI)GjBW0s! zp)+A*Vj^W{r)OeiV`pcgC1qx0VrF3cJlW}(Sh?BQxtTdg|N9{S9L?F(oLgB`{J+Qg zZ1Is>0)dX)3=Hn>?)2^~^bXDz3`|^HT>m(jndv?y=v+MQfkqy5_AV6vVGuQQF>$tX z1X?-Rlm25gGInqU@{xaP`hQHYbNnA#dzb&7rq2mu@Gx>@V4`RIXVQNJ<>dbVMeXeV zN7@CbZ1#Wk{r@C(QT22*V^B78ad34u`E;B)#lN8(xka4KjDQZ#styje|CvQaO9!BX zi=~4jsfY?YDV3a&iIx4o4(k6x$jNa_*}DLZ>`lz1MES@+Iq0pdOu3na*tpmkIanCQ z*o2sv#5jZ)IoQQmImKB;ghUxRSw;SX6?HIiwKKB^{s(LNf3d9pEB0S7*g1ZVENbR# zzKTmA34F#WH5|BW^M-*sXBU$G3I$}s$M?f=Ww|LOXSpMTZ=3Ea=d z|AfAo{b#&8e+IU_EQK@}7_*0zsF141%9$^0fUes7N9#k^gSlH%aXdLZ8CshtDR?{% zbiO1yw2+`El<4D>0u(luz$-0WegJeRsx2IXdgRBK0ANw5K1JRasF~HKbkF^b7lFer zj?8op7M}WL_V)C8uJrqZ#@6oHi`GNNDmGQ}LA*ID%d(8kB6-*H6puHYxiWcI(BfxN z4%zux%u15lVCjlLc#>Zi9@Rm&qA&mCe~537=ZVu0>wsCV`gr{(=r-xDu=f^cL3a6* z7t&K0P)idZ-huLjyq6kpAn3>Im2^j@R5C4T3njWfL`o9#ioS;`rtQu6gZ1rg7%~1M zu)~Ryp$*_#GrItB8%v&x&Bra`W6OMcNU<5v0YbUuo)6@&os0HCx%Q_}D2w;LrN3qy zUL%-00PE!H%;*1;XecUbUh^45c-3Ed_?i1FN-% zVnR)Y1IlO8F(612CxqJyAVchg46V=re(8u)j~3MC07?~3(9<^3(ogL&gB?WB_PR^^ z-%yMe%AA7Xou-z}lqRPhu3&3Fj3`K3M~;AICdB2YFPhOJe=N|bpyd}JW4?Xo%7zHG zRTfDAE)mMb{pHNI*7CYqeS7h>raZ|Jen+{s-lMUG+Y`MM>SAU-|M}o1U5OJQ&ykhCwJB_0y6r7?a^g>61au>2pqL5ZLk)k;eB9&Qyqs58@sh19KGcvlcA zuU^b6()9$ze54~yBH~Bf_a${&{4$gg{P2kT zw{?%IM2QZRBQE3nd_r!;_%J>AeP?KxMo5US1m;pW!VxaTCQ1QMbIwsqK_W7#G$OqeWPe__UXIM-fUB!ku&n15EML1@8Qpm5^s(pwo@C*cN zvS8^K6Jo+w&MagMF31vF6RDY5_$E93*3?Vey7d*)5RcAlsul;lOY3@igs3RCAMw@= zJlV}9khLu;Diy^rm`5D;MWmC{_``e1eqAUV{8Y}byGJGtmX{DR8U?tGBii+(f}|%} zlrgGRhb$AZw9PD7V22csxHd{eu8WVDe{8R6Py4I2uLtrYZ#gG0486Zd%HnL4!aTE0 z3vRANIN$NaGn8&do=C!jOI-+W!N$w!qaT7=9)p{f0xO|PuXZ!jlsE~#Tkf4sCmDo# zx9F*~(S!qRf?N;@AU(#gy~ti}1Bj3NVfgqGT7jLq%sDXh8g% z`uO$IdF-MbJj-BiM@Q<)A81rhuGITd1Jb&I%Fh}tQ7qK4IckjU@X^d=7gvd|6n(RZ zp`)amNkbZT+`WJ^`+AGa!UqJ)k~eBzdu9ru6$k|&5d1BT3Qw&>_!&JU9e8uvoKM7z zXG{i$U0Tk{_0&i7NOZf$5L+N`1@Fe}X~Ue%ts;fIT#W+7hk%Y|HV)A)(|6J&uF1yd z`A}%KY*K7{)HW*;lZDA1I6xXd{USR7C4kmLFMe+DM3l@f)Ox|rf7bgF2*r&Z@F-Fh zow@G5NzDH@LZRoE+2~f&sy*Z)@%_|r6&lXCrnZEu?#s*xb4Vp!d8B$Gk)oxJOwr|z z7!KFD4DTP`!!OBsz6Brjm~cSUFZM)>X9pg#0R#@czwHCy8QLt%Puus>e*0S)=#O5r zHZUD0w^6MczR6Mk&M)7?MWCPW>9tQSfH~?8BN)*Vcvy4UUa_e;_w=3MI_&9%B|+-C zL_M^c##l2TBhe0b%fpO$XOl!_Yk>cykvBd9#BmJ9ur(%Xc$D&%UJ1WOf*{nz%&4Ds z*Z7H?y~YyQpdEpBm7g|IRqGdq=A^_QOXs;;PzNdVP^+h7A*!@j z+{n>uV&uWWvITzq{UWKGF_(T2bx_u}L?lRZRwIID(BJJ5A*{lQLSp&(pC7yKpLk)u ze^tawhsXW%C9^mIZcEfGg`C%fT^3pl2I5GdJUK8?_Ura+?mKW~vT4)LoAbCZ(RyFC zrm~+XdQz5)DFO*{)nuy$n^Okc@)rL`$~hvj3&L2h%-HUx3O?&>edU)^R%@r!;NY&7 z6e5UGh~qDU+DHAV#)JZx_sm7Vp)0A}0@#C$yAL91k@TTi5CGygCnBi#?->^s^N}O+ zw<8LE7@u+3PG#7S(7@gKer6$n$g%0;LYMVD{VF$Vhd`&wfqrUk;)~wyH;t_76Q$xa z>pZP^_GZc#PK)J@V9$7M7atV-%)rZG#$omCCFOp#M@RNQMcD{H|19NMCSHz@C7fFK z$e1WY9K}NepY7%7wm8olX7j7;iJ&32MD7drnrOvx}IoR|d>Abf}7b`x)20z$8q z9578#erMAz3$DoEpG8Ns5e3gqFm|g<4y?cIdLbZvSes3o8w-4&Q&FhG2B~aE-o*Jg zvE^q)5M6nw88FJoU=MJxON{qZGhDT2ET;yh!g|V&mb6X%e(1Aeel9>#gJ{D9!IcqF zdL!t?cwaf`Z@28gGWEKfXm={dXc=QzQ8l{%SnRQEaXS$4>M8nZxX@VsDp(7#{< zf8*JFLaEg6Y{QYyR5mFq1iIPG9>7DT)}}?|Yp{Eit^edE17mA4w2n@| z+ta&t!3mX3>}7(TO2`$ek^@=n*El)<{>GHCzFNCkJ3Tls=iEk*SW<`CcWq?2Kb&SzxCfEOBN>C5dFvH5UmAbf&#Ktdo*weGX2?YLqZO0h}ss-^{_;?tlCf_QTJxwRiyK8zIGuf^6I zu?_v)i+0anAyv<}$x#yBvg**?Ic$;3W`jfII;Yg={QN>eQahh^&K=F`!deh2ynvnN zJlFVc8m~o!lkwOp9#drofF)BY&B7VEm4XYD3KK#k2cwB1z$Nhnu$b}=eu(te6?1A2 zDJZWAa|o)MnrWed9GR(G+uQ>EjOmcG`cVal4#v_|3deIjcHlLCnZncRw$vWh`_!sH z$RSTh=)jNlCM8Mk2s#MM-%TU{2%#;(hX&TQUa4_y268mS9ru`!!E!2)zG>rhkLM1u2nkKy@XD}ztAiykVS_&sg@(_}(+v>AfWF1YmKWtE z){NiuzK08hoZ7rUP5SrI)O09+fwSx#rEfn|rb!ign8QD3OvMTjzGfKs`MT zsuEO}_>Vt^{+11pdT}D-lt3m_<~HV%Mw!?f;r$VWR~P7Oh6HG|U##)fjQMfl4HL)Qe3uX2tA#U!3bb{It>pJVnV*%n)49Cu9z=AUZIjGb@#NTBDLGD;kD0GU`H#adz8R&+}Bv3i9 z1o+=;5qxUf_-b`S`FSe1KqRU%f|F)_1K>moF21K#%?PYMU1DgtDTLzO9!o@r!hphH z%X*MVzW8k-+?c3qGzGA`FO;+eHNd+lqR<73l>aNtA6*8Qi=&!VOLGq3?5n`@F@_!H z1j(G|ky2Iz_u1OsbwpK8zyZPLv`52LR%7=@p&gc*OnJ83h}k0p%%u@9BPUZ;Z_EyDHtI_CncXSLSA zfUvHo7WN~eV@{9J85icfh@LFTt4)Iwe>h~`0Zz7I{7Fk(TY3fX8^nOT2?d^Gq2%nE za)kYGxQL|z$Pp7gw;T_|nqiA|%93f7hzKf2>y9;u<#jQo#aW?U0=r&Qi>tf-dnD47+zJhWJt2b+?hj8~KcQLL5{N#<4QMGzLry`MJl+DYlD-1o1ag+5A> zmQF(DkqCx95cH$mkvB&Z+)=k zIdC!*GCSb!V<3$(y^7bTo07V@R78XV{vd2+^#kA=OY(&!S8R6gS@ z5A0$`j;Uaos%1CLzUx^8NFEE3`o;xn^k|p2K2p$0hzFdak^ypl8=QL&(D&pgs-fC>pbawwCR2H{jmvZQ+bOkaO92TjJnA>ke_nL z=v#*J=wcUFExmRF9Kd5MFmjUt*P|l-)xB+~?9(?4BU{J)gxez&1lz~`!WTqQ#Ww2c zg-HQYkYEayj`8&n$p91FfVo+;J%ydi*ZNuoOdm{lj^geT%)cp!^wpft?+&fUH5?W%#nw9YWR4>Q%A)ZeaEo0pNTn*Be2OPWhqq zle{-bXPxfbPT(k`ANhHFrB;_!1)|GX_uR8&3}h9^4s3H1>|Zq7|ubJd?`uN z5Ra0S9OsMezIsBt=em^Pjc6-WW;~|^scIE(3a?cq;26XP73f%=*VUE<2Xoqq;BSeL5nj*B2!<* zwLAf_@VyF>z^bt)lE`LoIOLR}VREc)`abBj{2e5$*gzexKMcs}Cdo)8F$bxb z4C3s$8_|$z3ztXBr5jw+Wc}I!{rv6jALO9A&%C3d-*b+;({&+H%U?Z~q-8|xuI(^! zO~7#LlTW_NE}K+hqM}nIzUG5+u@n4w&-=RYax>;4X{LsN?V75wmf0)B(t$qijp~ef z&iw~RIX;OL*+FdUf(hNaA4sBs2pRmn{r%m{?4`7yet; zl!}WzURjj^ziGN{bC+q)-NVZpw||CR!n2ppbL{sW;R%K?~F!iB#r^heO@7NWehz)f-2&^H#?=F0I}Uy<@ej zIMtoe+Wd>Pc&U1;-i>1})x1r~PV8TBtFv_`ldYmu{qKY^v6x~9F6qusT2e_UI*?sn zvx2|7o_lnj*TH`-;76YaA^6t8-5z=))@~Qqke7D%Q1?~Iop+XSyi9;KRpjMS(4L{4 z00GtFiAvk$rT7KWkt3Ey1Vy2n78uf1SVN0Z(9jV0tLVA8+syM9+>C5Fr@~J>Gseq3 zM!AAyLM~S%L{fqz_d!Rji{o4jRM^RrqNZI2?(42)tye7e{avM;4^S!$x>4iwXshBMY?h}q~O>ErDG0rp&*V5S94 zGcLUkbGX6ta#p`xR+1A0kNNesZ899-HC(9`J3GO1X`51uLQz%)v&IdJ;!FKJ>2JQD z8IhRn%Wmw3Yt@^}R&Hl4PaxXO^YkRIEpnOxramUyC-N;OF!hjKguF_&K;&7=sor46 z(I58Vmb^T|Td9_IOI9E-KsS#qGxpnc9U2;v&l=~h%oT)P&qP7~PlyLf+32|PA!+G# z8`|N%#jn3MJXTZqJiXg*RH^KTYtQ)rthdT#Q?+S>*a|jdN_W zzAboBNQ0=0=3VLnjPl1Me^+i!xo4Bv&9s~x{>XQD__YlA*!~5L^kYa!FSMsq11>7w zbqM+wC21(Bdm<8muNoV`V;?`F86nHYinFUFTR&3VcoEI10j9w-R`27(B>a_n0-)slv9hG4 z1sQHR&OKDKpLAJq!YcW7-uvXAFzw}DHgEcg<+8D{;aB;-KON%U8IPjNjp%tD7kxGQ znoHH0l7G#L%z-q=h8E^X(z#3bW{(=Ml2ZNe3hp=ou?IIff(q+*=xk0LhBUPrEj;kmg=V*Gu^Cs6FoHHRL`Toe#Wy;&OeiN4~DR z{fM}5diinNY!%~~SvqVV1??=mBkWM=N>3Sr9jM#yoVNt%D!6nbW`}Kj!e*jkWgK^B zI;L+(S>1tiM8VudG8~hJH&Lg;YgX>;HcR|4Q?pLax^RkJ8ATLK3;X&puS)bY?Vue} zCpets(T9Zw#k;m8apI20Xw{4lH}-DQ*dKaw6$M4ZWWHt&YHbJ#C^@Kckas=3D72xy zo|oU$#11*rq9BhoY4ff7x#MPn=6csfYmS(f@OoC(LDaze~ibsnrCPtcv94 z3ooe?8_zTiB%d=ZM$8FbMQ^(@HBSAJm1~m^Cds|%{DZjQvhi5HIlLcHiR@{ol^S8m z_h0ewXT?(r4uUFRBrIiH68b?x=x~jGVmfD>KEJAFPWW0beVr;Yqx4{TOOp(4eYn`K zrj4(Xq#Dr8Phy@Mm0HcEWDF5~_#@#Tb-VT`iOLISk);9jtvL)gsDhs7D?zsEe$_PR z`bmF(31(IIh(H6aZLaSXQ31@&-{iA23+=49zEgJoKK|T93l%}(`dz6Q1-taaGUv6r z{o?yc;`__yN0+H$^@`^ccwuva;q@?N8_DgTnFsW4YwwUzVsjyPq&@!nlwvJo2cNo# zq>lA*(rM)LM;TrNB0)n~EZL((0RYu(L?|^IzW*TPjyeJ6j=o!5;@nAHQC-}BzQ;Ex>XUJ3_Skj#zb{2Ti=Ko4 zAzH@byg@8MYE1ur5cwu(@z$Rd^)Q^=d94VZZ8@Tdn&XVEZw0O0x?hSa@XC1BvitMu zdXxcI6XL1(l`>Tq!hO@1gHvA9Bl-1~&1M11%JurCEbA6+#)eV2EW*rtA7(W)B)gM@ z%_!EOyugZPRqfPMdu(BS#O)%Vi>7oBs|~_P;qQ>l(n2_k9b_uIH4ssiVg;hga)*3W zoRjyguy3dC$1Yqud5v@^HM13h%t_6*f9OMwyliYyMetH5ue zUIEIjP+OMO0Hkyr^Wv?q_DPIQDu6PzE6uW*iD;f1JELaKqnd|75%}BGaObVDY@X(C zHGI|SyMC@&l5l5rLx4_togy$2SEpsW=5@o}!5A~#0i?gGt27kYN5qL3X00yD7>ecZ zBqw#pPhN>*7f5~$=D(H%_?X)Dj(&@G#x}=*~oYz*A2*79|2@bfMN5<7%}`K zqaay7n_akZ0)2Hr2W?qnBH=++I09*8B1>ZJd!|p3yh5uv3oW%E)pus0(c>hmBGDK! zv;_$)ILwooKer-U+ah5&)gUk@x`5Osss2opdR7)k(B!esrd5J*g`4wKJvXfp$y=u(V%XUd1(Nbn zKchkjJ}{*O%KG7QemK}0t-snw5O14KdS?Z@x^;zO%ZxDRrs2}U^X=tqh)!OOY9zIkeh%oMd+ zuUDINPG4Y|Kl9pSt3qOrK?fz^JR+0tVQIslyvPWR z9A3Gn4?=i?Nyt@GV%KbKIVHy5D+`5kL9~S&%WM1omXeX45%TA$lpckIUyp%ckCuts z;tC$13|aV?W)`e~06%S=YN19P&(!6KPyRNVwi!)vjM8w?*^~b$tZ7PaBefI&YhwYh zWj<5>dAu32*Sx0ypO$lM>E-dWLtJ^>qy3-7Ul-W0g1-vX@R6iph3M2758#|9TDM;| z%*tOicfVB)i+-zBp@94ID+>1Nhvhy|(KZS*9~4Cz)*WtaU5yz+=llnq^cYi=#1hj; z3_oj)Tb@}X?05yQ<$%Ptr)A2^qwl0+0Fv%ovU=_ScJ=-V!)mo?3@!4Uo&&P)i{f^?+iZmW@vHVfSzm*=~XP+xZX)o&;}pzLh0n(X`i!vu~|WN4Vm2>3CV3iEZ6nC@Y-c2ocu1hsgJK z`m3ONWO)_2%1^kKJ(qvWAOi+@(XZZX*dw$Chc@q)B&X=+ABvvXwU}u*{hRZD&t(2k z|9xT`QyZ0qKjP(x5Y}yqGWW`*gVG z5Eln(yTNL5ZJ)W`f6C5U|Momzs}UddT|PU0osZQ^g}29~$OaB1qf;DtJ==@m~pP7_}nxXewn|K9KS zdB5cZ_%oRWl)gKmQM*RvyC;l_^tP#`;P` zE6{06cjYoP)~j;|-mW*hoB>&a>~Gq^j4}zJPc_e(hnJi5wcVgO6l<|nT{QBPVJD^S zDWZ-uF&4T4)oGwz+gXK0<=#~E${a4hgW5#sm0TZbrUjh4WyZ~d;aRDzDcJ=3-j~y4)vW_bow1jkLM4Y{{Hcf z)Qzvn*PGOI?r29%@hHIUn{gsVYSH)2fvTTIjdaV%W%S@>7sB?xA4+2w=WTXN7sgEZ zGx6Eq8uQhqppc6E%~bT;Lx=3G1DIB$Lk|v`PL)#q>T6Pz77|e9j;$Z3jA|PD3h%#2 zHlVGrGg&T!mB1-nR*L0;dQH~)bp2!8_<+NRTIPJW>*!ZMp;i$PmQe(kt{es*XVGjH zsC+yLMjz#nLdAZDLmt%#Yfjn-M&zGC6)Kyb8hku2(xLRJkuSQ+-7Wsx9v0K)^8+OP zd6Af1mo){ums%r-Mf-;# z3=KD0Jk=D8D_&2SdA3&m2AVSn?z>^nHqz4U`aT$s(aH&6A{#2nBjjIQ8_$CeHB@QF z!vSIHaZEvm7S9#SrYp~wUYxEyrA&q5>T4zysvulBEgFLzr7|K=X&;&4!Jf88Vh2Ik z6-2|Hd6gHZQTJ*=Ii%P)(%r_=;q&3t4=v$~U4XqeHjN6z7ID3>5zJJqkY&3DBHnDD zx6C8$xj5dy<~bs;Oc_ZXk?h3sVm7y?wPil@23Ry72UOC@k&f3dU0&S``q%^=?6qeO zFf7P7saI#(3Q76J#+VBPtcUhZhLRVz@H}kmudO>1$6|6ireyY<%1QS3qeq`cppues zDnjnZO8^2PQF$Gwf*q58K-ho&F;-AW{PFn2`+QX3*aHrWG`DTJl}roZoC$Iw>2$J_ zca0UPYNrRJ^^La5TlvX*CJJU|$hD1}u-z-*fKS3a{37L~?Jgxuj!^Gvp@chA$jrc` za;h{9(AGGBCzAglnBL9pD^;WTZqG`&u+)X%YsYdKFoi;kmIJYh)d=^~nebLm8;&Y% zbaAVQP^%tm%*3pU2#QM!w<}D4alj@FLR=TeU&o3M@*LD``SFmo-koCfmJVt+1_j7# zCkX0Gm#Y4;A+hwjU$&52{*JJqWNrKEv-9pFaEyRQSvbv6BAATvtjW9?1>4;rUkggD z1H9j>w^uHS;Td`U+Od^0`bC5QrjyeL)jl-wE3<{rT4_`r?MsUmBkp;sdKDctTe{^6 zd7dE%OE*O{m2_YTe(*y+-c|jJ$3FgeW|GE*t_z}DLVloES*qmDx~=l=-D^|SOX{&E zvDfWp?9fLI(U1K0H*oH!<0L^BM}LK#+I|$4SeX>sz_lbv7Z(=Wm4>o!bgonp)?KsC zixt&vf0FV~oQf}7AiM6W9FPmy4LG#}0AGaJhJ1aoX2nRg|AcsSX;j3&qba5z1SBX6 zJ}7eljOuaj?)>A#4*QY(E0 zic@9ZlUZkKLztuPCU4N5Ut4QSEMldJaqQ<2_zkM($N31;bQ&0Sb-q*m38#VWYdVcR zMu`{qFEkK>fMO5iUQ{IW(=7^Xrutqq;kN}z!sxn7^0`n1uCXnD#HKWxN~9e74PP$K z^c+M~ZUh{-+Pnu~cfB*A>><)zOA|hOjQnj472`uF-8wjUYiDPt zIV?|={b-dPiQu!C11U*F(i!WU!aYos1B@r@aQoLKL^bR{lSu1t9Mp~*)8ZY zI{mD(ZN+K9YQ<&&W`#8XEX?-EH{yJ1u60uvD$z>I$fB6?vn*nky0X&h349=5VTe5+?LGApSh+{NeR#nC zkxYO~rs7*{mnT0zwK~m5s3v+d%9-^*LsT+&daFwhNX~b5qZ?0MEln^hfL>veLvzM9 zlO{)KUipD@A1-t-Z_d&M>&J$=Q7o68x{g?{tlAs|X7Nit9`oeX z3%VZjNM1!bAZ5Rr!T1W62U@X25OsDWfj$IH5gvAXETS8K--ET>=%As?3G=MAd&6R` z1fB|!)qr_?FR0{(AgaSu8>@dQIYA2-$2BsKzJ>rGyp&Bt3lq9*tJJPaInD-W6r9S) zMk*&ppwfduRh6twF|_jM_u1my*T&&L<(}LMRL)fvd1nQPh+%|BqF79sQ9km-L!C2| zaUa|S4ACNtI7an0@1zT4;yGqgdV>+j;}*;%$y-OS&X{o~rV^QlO0p{=vtD|05^E_& zD!=EfgJW62vgR8s$dW{?@@bcfaUgy{Kg#?&PhBKPpVXYzegVrP zb@Pp^rh~=mj1gb_D~5UGq;Rq=jrl@HrYUL0bpIlq4WtS7QIHBv2N}U*lyt3UYwYlP z&&|s6H8m>8UrG(Kb@*8qPG|H&ZOBDRJLI)!+RF)0N~{*l#zM%y(3lI^o>U@X|IrS3{8&Ao>ZtjpTxzMMYB5pi4I2n6{{x9x>~>n;ow`?C zBwl7hxM#L9z}#qB5&;1(>YD>HDnda~bjrm38OpAG?Rk=#j^MZa*QsxLxmErWWJ>eX ziQ+fV9)r9mcUvL~L=*2=bxX%$VP+fV6|L|Z@VD$3(xIwR!t{L=r~g;g|sMr(6&n>G9q zJ_X?*Xq7laZ#g7XuOFIhNh7vcpbL&cYPx~=3P^3U7SrlZG>Dc zlCSCKDS1AOpF_`)ICr4#ZxLC*BF&6?+Qtc458rL0Z#Ae!X492)Dx`#Y6eG)tI4+|B zonX*4&=CT3FJA0toc8VA^Yz1GUqK}|((UfFsX@RDdIURTgsTlGl&qoDGuG!&$>F6p zQ)vKZ0zzz3vN6iiM}*I2N5;*Itrx*EE$o1V=(Jj}G=`2eMxew6I%>q$uz@g@ijoxO z>()i)=H3hn-pHqEiFIDTLWWly(n1x4f?><6)>a7&#S^i$nzfa1R;|aam_u?VvIov5 zMD(4QS1^!8dSx!B4W~U9C^~oT_~5T#hi|z*^j)qILd5g^Ve=-rs16Paq#o|!vBiGAKNeef{_cvi7pE)6~2q#cp6#a2>kqKoEe1}12AUOpu zI&b12e2OK*nz=~%t_O(yMIZ}Ifi9`)0ErWG1=*_seoKj)UN?}$56|mZKI;7v-BPy_ zU3;?>sw1#HD}2dDIL{SK8%BSk&Jw(VRC7VL7ro~W%w3$dS+rMmG&nbKM+1m1)P=03Qu8AFUTXp~l=NflS+$Ip(Pdd#IDRx;yj% zYQK6ly}B@|BJ9qmv;I<>%O0UlsCDI&@WzL&oT!W_*mq*bd%ObcI&FWnx|e|sV>4Q$ zFkd7R9HuE6&R~Nr7wg?jTCl^O0H3Dw<4X{$mlFIt8sLRBfu{4Ud$X6*1}+RNS+A%J zF{!$MpFn}KuFU*)RwvU`B==rx97vhhMR83T-A|dK%CK|ug07DRSKxIoLjztu_QG1x zg&1Upf`IOf$^Sz~elsg{G-dmN9t*)QdgzT<8>e}@NuqWmkO%+41*M<^&D>lTP_Up_ z>Y3Y1th}APA17mYOb7e*3q}BoQ?OQa-P=_uNsmj05iB1|O!AQsvG&m0ZRK-wX%T~{ z))OL(UIr=&U#Jfhk|M)Xd{gPZ$)a5VtqM;lfllsJBAUv=3`0h=7pCQcc>n1U1H%wu zElTLnS2>fPez_`3n=^xifXxUoo_A(cNtD(NM-HsHC2S#PL` zj5dhtcGlc&5Z!GsZ(b)Bn=y;hY~|Ek)!+sy-RJPro^tELqfe9|@3OmG%lEenziJ5* zY0%-(_q`hl{9*r)*LmdGeXPP_ImAJDSMWBy!+V$i6mLddbHdxQf%yt}O`T_Ye7-@0_vDO6bD= ziU{{&y}&PsDv2-r4~r`3{-c5$^)P}q<$Qju7d9ltOLCAG+Q_q(AZ9vM(u))H>VZv2 zZ1Of8nxz8O8Vg0jX{=qh$7fz28>&k+m{|Ar9Y+M~b1Tq1^dM2O?fOpXnoomv?9f9A z+0_xR761C>$P95|W=PPf5Vhi8>a?GS;R~_v61irY$e9;K5Uc_c?^kg61FF~uym))E z&G9yEHfTqtG};Q)yrxENrBk_zUz@$T`H2SmK$KN6X$$QU8{hH;mkS=}y{D>MP?NM7 z&KX~FVE^ou&SI$9xxghsjn0}^Wt-u|huamkm>v))a>~v{lLQI$3+a~mqC?H`mzHsG zP<(XAaN^9_?_;mskn_e%ToD0!&Nq>1fn?F0B2ligxVVO_dQ_I#uRZA(>d4XYi}3;w zMkNWO6!l#=5$4ELt0&8CgBQ4TVo*~Jr!nz6o5%*25tDF|EK)B8IDlAE%P>Zz5Y3Pb zycy1e_P;!HlV+|BH6Mx}E>ORtqU(z)dmJvpVSc~S^uB*-e8)tMK3@%Dw)wX9B%1`e zx{H&D9=!YaP5rkCHN|i zMGErazid_eo5RWaMO+49rJjhJ2B^W{X~MIekg;!AVwZob{#x$KGh#a^fd$MVTQ=c= zbd-0~rio?i?~bl}63N)N&pg)5+EJBgH}DXYQ?)C^o;hg_&n4VA@EPr$`a&%%o& ze@n8gpQ^5^jtWkeP{*Sx5cvD=ox^&3I0$L>$4Fx+Y0M_~11A3EUg(fb?S^z=j75(5 zK)l@!-;O@H!Y_50DvJlkv4-ezs|%)uiop|Lh5CY9*HS7IAN|Y2V4YSqvH?uF{0%2X zJyPA2j|qt?` + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. + diff --git a/networks/remote/ansible/inventory/digital_ocean.ini b/networks/remote/ansible/inventory/digital_ocean.ini new file mode 100644 index 000000000..b809554b2 --- /dev/null +++ b/networks/remote/ansible/inventory/digital_ocean.ini @@ -0,0 +1,34 @@ +# Ansible DigitalOcean external inventory script settings +# + +[digital_ocean] + +# The module needs your DigitalOcean API Token. +# It may also be specified on the command line via --api-token +# or via the environment variables DO_API_TOKEN or DO_API_KEY +# +#api_token = 123456abcdefg + + +# API calls to DigitalOcean may be slow. For this reason, we cache the results +# of an API call. Set this to the path you want cache files to be written to. +# One file will be written to this directory: +# - ansible-digital_ocean.cache +# +cache_path = /tmp + + +# The number of seconds a cache file is considered valid. After this many +# seconds, a new API call will be made, and the cache file will be updated. +# +cache_max_age = 300 + +# Use the private network IP address instead of the public when available. +# +use_private_network = False + +# Pass variables to every group, e.g.: +# +# group_variables = { 'ansible_user': 'root' } +# +group_variables = {} diff --git a/networks/remote/ansible/inventory/digital_ocean.py b/networks/remote/ansible/inventory/digital_ocean.py new file mode 100755 index 000000000..24ba64370 --- /dev/null +++ b/networks/remote/ansible/inventory/digital_ocean.py @@ -0,0 +1,471 @@ +#!/usr/bin/env python + +''' +DigitalOcean external inventory script +====================================== + +Generates Ansible inventory of DigitalOcean Droplets. + +In addition to the --list and --host options used by Ansible, there are options +for generating JSON of other DigitalOcean data. This is useful when creating +droplets. For example, --regions will return all the DigitalOcean Regions. +This information can also be easily found in the cache file, whose default +location is /tmp/ansible-digital_ocean.cache). + +The --pretty (-p) option pretty-prints the output for better human readability. + +---- +Although the cache stores all the information received from DigitalOcean, +the cache is not used for current droplet information (in --list, --host, +--all, and --droplets). This is so that accurate droplet information is always +found. You can force this script to use the cache with --force-cache. + +---- +Configuration is read from `digital_ocean.ini`, then from environment variables, +then and command-line arguments. + +Most notably, the DigitalOcean API Token must be specified. It can be specified +in the INI file or with the following environment variables: + export DO_API_TOKEN='abc123' or + export DO_API_KEY='abc123' + +Alternatively, it can be passed on the command-line with --api-token. + +If you specify DigitalOcean credentials in the INI file, a handy way to +get them into your environment (e.g., to use the digital_ocean module) +is to use the output of the --env option with export: + export $(digital_ocean.py --env) + +---- +The following groups are generated from --list: + - ID (droplet ID) + - NAME (droplet NAME) + - image_ID + - image_NAME + - distro_NAME (distribution NAME from image) + - region_NAME + - size_NAME + - status_STATUS + +For each host, the following variables are registered: + - do_backup_ids + - do_created_at + - do_disk + - do_features - list + - do_id + - do_image - object + - do_ip_address + - do_private_ip_address + - do_kernel - object + - do_locked + - do_memory + - do_name + - do_networks - object + - do_next_backup_window + - do_region - object + - do_size - object + - do_size_slug + - do_snapshot_ids - list + - do_status + - do_tags + - do_vcpus + - do_volume_ids + +----- +``` +usage: digital_ocean.py [-h] [--list] [--host HOST] [--all] + [--droplets] [--regions] [--images] [--sizes] + [--ssh-keys] [--domains] [--pretty] + [--cache-path CACHE_PATH] + [--cache-max_age CACHE_MAX_AGE] + [--force-cache] + [--refresh-cache] + [--api-token API_TOKEN] + +Produce an Ansible Inventory file based on DigitalOcean credentials + +optional arguments: + -h, --help show this help message and exit + --list List all active Droplets as Ansible inventory + (default: True) + --host HOST Get all Ansible inventory variables about a specific + Droplet + --all List all DigitalOcean information as JSON + --droplets List Droplets as JSON + --regions List Regions as JSON + --images List Images as JSON + --sizes List Sizes as JSON + --ssh-keys List SSH keys as JSON + --domains List Domains as JSON + --pretty, -p Pretty-print results + --cache-path CACHE_PATH + Path to the cache files (default: .) + --cache-max_age CACHE_MAX_AGE + Maximum age of the cached items (default: 0) + --force-cache Only use data from the cache + --refresh-cache Force refresh of cache by making API requests to + DigitalOcean (default: False - use cache files) + --api-token API_TOKEN, -a API_TOKEN + DigitalOcean API Token +``` + +''' + +# (c) 2013, Evan Wies +# +# Inspired by the EC2 inventory plugin: +# https://github.com/ansible/ansible/blob/devel/contrib/inventory/ec2.py +# +# This file is part of Ansible, +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +###################################################################### + +import os +import sys +import re +import argparse +from time import time +import ConfigParser +import ast + +try: + import json +except ImportError: + import simplejson as json + +try: + from dopy.manager import DoManager +except ImportError as e: + sys.exit("failed=True msg='`dopy` library required for this script'") + + +class DigitalOceanInventory(object): + + ########################################################################### + # Main execution path + ########################################################################### + + def __init__(self): + ''' Main execution path ''' + + # DigitalOceanInventory data + self.data = {} # All DigitalOcean data + self.inventory = {} # Ansible Inventory + + # Define defaults + self.cache_path = '.' + self.cache_max_age = 0 + self.use_private_network = False + self.group_variables = {} + + # Read settings, environment variables, and CLI arguments + self.read_settings() + self.read_environment() + self.read_cli_args() + + # Verify credentials were set + if not hasattr(self, 'api_token'): + sys.stderr.write('''Could not find values for DigitalOcean api_token. +They must be specified via either ini file, command line argument (--api-token), +or environment variables (DO_API_TOKEN)\n''') + sys.exit(-1) + + # env command, show DigitalOcean credentials + if self.args.env: + print("DO_API_TOKEN=%s" % self.api_token) + sys.exit(0) + + # Manage cache + self.cache_filename = self.cache_path + "/ansible-digital_ocean.cache" + self.cache_refreshed = False + + if self.is_cache_valid(): + self.load_from_cache() + if len(self.data) == 0: + if self.args.force_cache: + sys.stderr.write('''Cache is empty and --force-cache was specified\n''') + sys.exit(-1) + + self.manager = DoManager(None, self.api_token, api_version=2) + + # Pick the json_data to print based on the CLI command + if self.args.droplets: + self.load_from_digital_ocean('droplets') + json_data = {'droplets': self.data['droplets']} + elif self.args.regions: + self.load_from_digital_ocean('regions') + json_data = {'regions': self.data['regions']} + elif self.args.images: + self.load_from_digital_ocean('images') + json_data = {'images': self.data['images']} + elif self.args.sizes: + self.load_from_digital_ocean('sizes') + json_data = {'sizes': self.data['sizes']} + elif self.args.ssh_keys: + self.load_from_digital_ocean('ssh_keys') + json_data = {'ssh_keys': self.data['ssh_keys']} + elif self.args.domains: + self.load_from_digital_ocean('domains') + json_data = {'domains': self.data['domains']} + elif self.args.all: + self.load_from_digital_ocean() + json_data = self.data + elif self.args.host: + json_data = self.load_droplet_variables_for_host() + else: # '--list' this is last to make it default + self.load_from_digital_ocean('droplets') + self.build_inventory() + json_data = self.inventory + + if self.cache_refreshed: + self.write_to_cache() + + if self.args.pretty: + print(json.dumps(json_data, sort_keys=True, indent=2)) + else: + print(json.dumps(json_data)) + # That's all she wrote... + + ########################################################################### + # Script configuration + ########################################################################### + + def read_settings(self): + ''' Reads the settings from the digital_ocean.ini file ''' + config = ConfigParser.SafeConfigParser() + config.read(os.path.dirname(os.path.realpath(__file__)) + '/digital_ocean.ini') + + # Credentials + if config.has_option('digital_ocean', 'api_token'): + self.api_token = config.get('digital_ocean', 'api_token') + + # Cache related + if config.has_option('digital_ocean', 'cache_path'): + self.cache_path = config.get('digital_ocean', 'cache_path') + if config.has_option('digital_ocean', 'cache_max_age'): + self.cache_max_age = config.getint('digital_ocean', 'cache_max_age') + + # Private IP Address + if config.has_option('digital_ocean', 'use_private_network'): + self.use_private_network = config.getboolean('digital_ocean', 'use_private_network') + + # Group variables + if config.has_option('digital_ocean', 'group_variables'): + self.group_variables = ast.literal_eval(config.get('digital_ocean', 'group_variables')) + + def read_environment(self): + ''' Reads the settings from environment variables ''' + # Setup credentials + if os.getenv("DO_API_TOKEN"): + self.api_token = os.getenv("DO_API_TOKEN") + if os.getenv("DO_API_KEY"): + self.api_token = os.getenv("DO_API_KEY") + + def read_cli_args(self): + ''' Command line argument processing ''' + parser = argparse.ArgumentParser(description='Produce an Ansible Inventory file based on DigitalOcean credentials') + + parser.add_argument('--list', action='store_true', help='List all active Droplets as Ansible inventory (default: True)') + parser.add_argument('--host', action='store', help='Get all Ansible inventory variables about a specific Droplet') + + parser.add_argument('--all', action='store_true', help='List all DigitalOcean information as JSON') + parser.add_argument('--droplets', '-d', action='store_true', help='List Droplets as JSON') + parser.add_argument('--regions', action='store_true', help='List Regions as JSON') + parser.add_argument('--images', action='store_true', help='List Images as JSON') + parser.add_argument('--sizes', action='store_true', help='List Sizes as JSON') + parser.add_argument('--ssh-keys', action='store_true', help='List SSH keys as JSON') + parser.add_argument('--domains', action='store_true', help='List Domains as JSON') + + parser.add_argument('--pretty', '-p', action='store_true', help='Pretty-print results') + + parser.add_argument('--cache-path', action='store', help='Path to the cache files (default: .)') + parser.add_argument('--cache-max_age', action='store', help='Maximum age of the cached items (default: 0)') + parser.add_argument('--force-cache', action='store_true', default=False, help='Only use data from the cache') + parser.add_argument('--refresh-cache', '-r', action='store_true', default=False, + help='Force refresh of cache by making API requests to DigitalOcean (default: False - use cache files)') + + parser.add_argument('--env', '-e', action='store_true', help='Display DO_API_TOKEN') + parser.add_argument('--api-token', '-a', action='store', help='DigitalOcean API Token') + + self.args = parser.parse_args() + + if self.args.api_token: + self.api_token = self.args.api_token + + # Make --list default if none of the other commands are specified + if (not self.args.droplets and not self.args.regions and + not self.args.images and not self.args.sizes and + not self.args.ssh_keys and not self.args.domains and + not self.args.all and not self.args.host): + self.args.list = True + + ########################################################################### + # Data Management + ########################################################################### + + def load_from_digital_ocean(self, resource=None): + '''Get JSON from DigitalOcean API''' + if self.args.force_cache and os.path.isfile(self.cache_filename): + return + # We always get fresh droplets + if self.is_cache_valid() and not (resource == 'droplets' or resource is None): + return + if self.args.refresh_cache: + resource = None + + if resource == 'droplets' or resource is None: + self.data['droplets'] = self.manager.all_active_droplets() + self.cache_refreshed = True + if resource == 'regions' or resource is None: + self.data['regions'] = self.manager.all_regions() + self.cache_refreshed = True + if resource == 'images' or resource is None: + self.data['images'] = self.manager.all_images(filter=None) + self.cache_refreshed = True + if resource == 'sizes' or resource is None: + self.data['sizes'] = self.manager.sizes() + self.cache_refreshed = True + if resource == 'ssh_keys' or resource is None: + self.data['ssh_keys'] = self.manager.all_ssh_keys() + self.cache_refreshed = True + if resource == 'domains' or resource is None: + self.data['domains'] = self.manager.all_domains() + self.cache_refreshed = True + + def build_inventory(self): + '''Build Ansible inventory of droplets''' + self.inventory = { + 'all': { + 'hosts': [], + 'vars': self.group_variables + }, + '_meta': {'hostvars': {}} + } + + # add all droplets by id and name + for droplet in self.data['droplets']: + # when using private_networking, the API reports the private one in "ip_address". + if 'private_networking' in droplet['features'] and not self.use_private_network: + for net in droplet['networks']['v4']: + if net['type'] == 'public': + dest = net['ip_address'] + else: + continue + else: + dest = droplet['ip_address'] + + self.inventory['all']['hosts'].append(dest) + + self.inventory[droplet['id']] = [dest] + self.inventory[droplet['name']] = [dest] + + # groups that are always present + for group in ('region_' + droplet['region']['slug'], + 'image_' + str(droplet['image']['id']), + 'size_' + droplet['size']['slug'], + 'distro_' + self.to_safe(droplet['image']['distribution']), + 'status_' + droplet['status']): + if group not in self.inventory: + self.inventory[group] = {'hosts': [], 'vars': {}} + self.inventory[group]['hosts'].append(dest) + + # groups that are not always present + for group in (droplet['image']['slug'], + droplet['image']['name']): + if group: + image = 'image_' + self.to_safe(group) + if image not in self.inventory: + self.inventory[image] = {'hosts': [], 'vars': {}} + self.inventory[image]['hosts'].append(dest) + + if droplet['tags']: + for tag in droplet['tags']: + if tag not in self.inventory: + self.inventory[tag] = {'hosts': [], 'vars': {}} + self.inventory[tag]['hosts'].append(dest) + + # hostvars + info = self.do_namespace(droplet) + self.inventory['_meta']['hostvars'][dest] = info + + def load_droplet_variables_for_host(self): + '''Generate a JSON response to a --host call''' + host = int(self.args.host) + droplet = self.manager.show_droplet(host) + info = self.do_namespace(droplet) + return {'droplet': info} + + ########################################################################### + # Cache Management + ########################################################################### + + def is_cache_valid(self): + ''' Determines if the cache files have expired, or if it is still valid ''' + if os.path.isfile(self.cache_filename): + mod_time = os.path.getmtime(self.cache_filename) + current_time = time() + if (mod_time + self.cache_max_age) > current_time: + return True + return False + + def load_from_cache(self): + ''' Reads the data from the cache file and assigns it to member variables as Python Objects''' + try: + cache = open(self.cache_filename, 'r') + json_data = cache.read() + cache.close() + data = json.loads(json_data) + except IOError: + data = {'data': {}, 'inventory': {}} + + self.data = data['data'] + self.inventory = data['inventory'] + + def write_to_cache(self): + ''' Writes data in JSON format to a file ''' + data = {'data': self.data, 'inventory': self.inventory} + json_data = json.dumps(data, sort_keys=True, indent=2) + + cache = open(self.cache_filename, 'w') + cache.write(json_data) + cache.close() + + ########################################################################### + # Utilities + ########################################################################### + + def push(self, my_dict, key, element): + ''' Pushed an element onto an array that may not have been defined in the dict ''' + if key in my_dict: + my_dict[key].append(element) + else: + my_dict[key] = [element] + + def to_safe(self, word): + ''' Converts 'bad' characters in a string to underscores so they can be used as Ansible groups ''' + return re.sub("[^A-Za-z0-9\-\.]", "_", word) + + def do_namespace(self, data): + ''' Returns a copy of the dictionary with all the keys put in a 'do_' namespace ''' + info = {} + for k, v in data.items(): + info['do_' + k] = v + return info + + +########################################################################### +# Run the script +DigitalOceanInventory() diff --git a/networks/remote/ansible/reconfig.yml b/networks/remote/ansible/reconfig.yml new file mode 100644 index 000000000..08603ed7c --- /dev/null +++ b/networks/remote/ansible/reconfig.yml @@ -0,0 +1,16 @@ +--- + +#Requires BINARY and CONFIGDIR variables set. + +- hosts: all + user: root + any_errors_fatal: true + gather_facts: no + vars: + - service: tendermint + roles: + - stop + - unsafe_reset + - config + - start + diff --git a/networks/remote/ansible/reset.yml b/networks/remote/ansible/reset.yml new file mode 100644 index 000000000..63b1733c7 --- /dev/null +++ b/networks/remote/ansible/reset.yml @@ -0,0 +1,14 @@ +--- + +- hosts: all + user: root + any_errors_fatal: true + gather_facts: no + vars: + - service: tendermint + roles: + - stop + - unsafe_reset + - start + + diff --git a/networks/remote/ansible/restart.yml b/networks/remote/ansible/restart.yml new file mode 100644 index 000000000..71d4bc66d --- /dev/null +++ b/networks/remote/ansible/restart.yml @@ -0,0 +1,12 @@ +--- + +- hosts: all + user: root + any_errors_fatal: true + gather_facts: no + vars: + - service: tendermint + roles: + - stop + - start + diff --git a/networks/remote/ansible/roles/config/tasks/main.yml b/networks/remote/ansible/roles/config/tasks/main.yml new file mode 100644 index 000000000..1568b5005 --- /dev/null +++ b/networks/remote/ansible/roles/config/tasks/main.yml @@ -0,0 +1,15 @@ +--- + +- name: Copy binary + copy: + src: "{{BINARY}}" + dest: /usr/bin + mode: 0755 + +- name: Copy config + copy: + src: "{{CONFIGDIR}}/node0/" + dest: "/home/{{service}}/.{{service}}/" + owner: "{{service}}" + group: "{{service}}" + diff --git a/networks/remote/ansible/roles/install/handlers/main.yml b/networks/remote/ansible/roles/install/handlers/main.yml new file mode 100644 index 000000000..16afbb618 --- /dev/null +++ b/networks/remote/ansible/roles/install/handlers/main.yml @@ -0,0 +1,5 @@ +--- + +- name: reload services + systemd: "name={{service}} daemon_reload=yes enabled=yes" + diff --git a/networks/remote/ansible/roles/install/tasks/main.yml b/networks/remote/ansible/roles/install/tasks/main.yml new file mode 100644 index 000000000..9e5a7524a --- /dev/null +++ b/networks/remote/ansible/roles/install/tasks/main.yml @@ -0,0 +1,15 @@ +--- + +- name: Create service group + group: "name={{service}}" + +- name: Create service user + user: "name={{service}} group={{service}} home=/home/{{service}}" + +- name: Change user folder to more permissive + file: "path=/home/{{service}} mode=0755" + +- name: Create service + template: "src=systemd.service.j2 dest=/etc/systemd/system/{{service}}.service" + notify: reload services + diff --git a/networks/remote/ansible/roles/install/templates/systemd.service.j2 b/networks/remote/ansible/roles/install/templates/systemd.service.j2 new file mode 100644 index 000000000..34ba3ecf5 --- /dev/null +++ b/networks/remote/ansible/roles/install/templates/systemd.service.j2 @@ -0,0 +1,17 @@ +[Unit] +Description={{service}} +Requires=network-online.target +After=network-online.target + +[Service] +Restart=on-failure +User={{service}} +Group={{service}} +PermissionsStartOnly=true +ExecStart=/usr/bin/tendermint node --proxy_app=dummy +ExecReload=/bin/kill -HUP $MAINPID +KillSignal=SIGTERM + +[Install] +WantedBy=multi-user.target + diff --git a/networks/remote/ansible/roles/start/tasks/main.yml b/networks/remote/ansible/roles/start/tasks/main.yml new file mode 100644 index 000000000..6bc611c91 --- /dev/null +++ b/networks/remote/ansible/roles/start/tasks/main.yml @@ -0,0 +1,5 @@ +--- + +- name: start service + service: "name={{service}} state=started" + diff --git a/networks/remote/ansible/roles/status/tasks/main.yml b/networks/remote/ansible/roles/status/tasks/main.yml new file mode 100644 index 000000000..50170c746 --- /dev/null +++ b/networks/remote/ansible/roles/status/tasks/main.yml @@ -0,0 +1,10 @@ +--- + +- name: application service status + command: "service {{service}} status" + changed_when: false + register: status + +- name: Result + debug: var=status.stdout_lines + diff --git a/networks/remote/ansible/roles/stop/tasks/main.yml b/networks/remote/ansible/roles/stop/tasks/main.yml new file mode 100644 index 000000000..7db356f22 --- /dev/null +++ b/networks/remote/ansible/roles/stop/tasks/main.yml @@ -0,0 +1,5 @@ +--- + +- name: stop service + service: "name={{service}} state=stopped" + diff --git a/networks/remote/ansible/roles/unsafe_reset/tasks/main.yml b/networks/remote/ansible/roles/unsafe_reset/tasks/main.yml new file mode 100644 index 000000000..6c5831986 --- /dev/null +++ b/networks/remote/ansible/roles/unsafe_reset/tasks/main.yml @@ -0,0 +1,3 @@ +- command: "{{service}} node unsafe_reset_all" + become_user: "{{service}}" + diff --git a/networks/remote/ansible/start.yml b/networks/remote/ansible/start.yml new file mode 100644 index 000000000..2be07dc73 --- /dev/null +++ b/networks/remote/ansible/start.yml @@ -0,0 +1,11 @@ +--- + +- hosts: all + user: root + any_errors_fatal: true + gather_facts: no + vars: + - service: tendermint + roles: + - start + diff --git a/networks/remote/ansible/status.yml b/networks/remote/ansible/status.yml new file mode 100644 index 000000000..a1721b87b --- /dev/null +++ b/networks/remote/ansible/status.yml @@ -0,0 +1,11 @@ +--- + +- hosts: all + user: root + any_errors_fatal: true + gather_facts: no + vars: + - service: tendermint + roles: + - status + diff --git a/networks/remote/ansible/stop.yml b/networks/remote/ansible/stop.yml new file mode 100644 index 000000000..abc6031d5 --- /dev/null +++ b/networks/remote/ansible/stop.yml @@ -0,0 +1,11 @@ +--- + +- hosts: all + user: root + any_errors_fatal: true + gather_facts: no + vars: + - service: tendermint + roles: + - stop + diff --git a/networks/remote/terraform/.gitignore b/networks/remote/terraform/.gitignore new file mode 100644 index 000000000..0cc2d499a --- /dev/null +++ b/networks/remote/terraform/.gitignore @@ -0,0 +1,4 @@ +.terraform +terraform.tfstate +terraform.tfstate.backup +terraform.tfstate.d diff --git a/networks/remote/terraform/README.rst b/networks/remote/terraform/README.rst new file mode 100644 index 000000000..243d1c148 --- /dev/null +++ b/networks/remote/terraform/README.rst @@ -0,0 +1,33 @@ +Using Terraform +=============== + +This is a `Terraform `__ configuration that sets up DigitalOcean droplets. + +Prerequisites +------------- + +- Install `HashiCorp Terraform `__ on a linux machine. +- Create a `DigitalOcean API token `__ with read and write capability. +- Create SSH keys + +Build +----- + +:: + + export DO_API_TOKEN="abcdef01234567890abcdef01234567890" + export SSH_KEY_FILE="$HOME/.ssh/id_rsa.pub" + + terraform init + terraform apply -var DO_API_TOKEN="$DO_API_TOKEN" -var SSH_KEY_FILE="$SSH_KEY_FILE" + +At the end you will get a list of IP addresses that belongs to your new droplets. + +Destroy +------- + +Run the below: + +:: + + terraform destroy diff --git a/networks/remote/terraform/cluster/main.tf b/networks/remote/terraform/cluster/main.tf new file mode 100644 index 000000000..98ab37cee --- /dev/null +++ b/networks/remote/terraform/cluster/main.tf @@ -0,0 +1,28 @@ +resource "digitalocean_tag" "cluster" { + name = "${var.name}" +} + +resource "digitalocean_ssh_key" "cluster" { + name = "${var.name}" + public_key = "${file(var.ssh_key)}" +} + +resource "digitalocean_droplet" "cluster" { + name = "${var.name}-node${count.index}" + image = "centos-7-x64" + size = "${var.instance_size}" + region = "${element(var.regions, count.index)}" + ssh_keys = ["${digitalocean_ssh_key.cluster.id}"] + count = "${var.servers}" + tags = ["${digitalocean_tag.cluster.id}"] + + lifecycle = { + prevent_destroy = false + } + + connection { + timeout = "30s" + } + +} + diff --git a/networks/remote/terraform/cluster/outputs.tf b/networks/remote/terraform/cluster/outputs.tf new file mode 100644 index 000000000..78291b6a9 --- /dev/null +++ b/networks/remote/terraform/cluster/outputs.tf @@ -0,0 +1,15 @@ +// The cluster name +output "name" { + value = "${var.name}" +} + +// The list of cluster instance IDs +output "instances" { + value = ["${digitalocean_droplet.cluster.*.id}"] +} + +// The list of cluster instance public IPs +output "public_ips" { + value = ["${digitalocean_droplet.cluster.*.ipv4_address}"] +} + diff --git a/networks/remote/terraform/cluster/variables.tf b/networks/remote/terraform/cluster/variables.tf new file mode 100644 index 000000000..3aa837a27 --- /dev/null +++ b/networks/remote/terraform/cluster/variables.tf @@ -0,0 +1,25 @@ +variable "name" { + description = "The cluster name, e.g cdn" +} + +variable "regions" { + description = "Regions to launch in" + type = "list" + default = ["AMS2", "FRA1", "LON1", "NYC3", "SFO2", "SGP1", "TOR1"] +} + +variable "ssh_key" { + description = "SSH key filename to copy to the nodes" + type = "string" +} + +variable "instance_size" { + description = "The instance size to use" + default = "2gb" +} + +variable "servers" { + description = "Desired instance count" + default = 4 +} + diff --git a/networks/remote/terraform/main.tf b/networks/remote/terraform/main.tf new file mode 100644 index 000000000..5618689d2 --- /dev/null +++ b/networks/remote/terraform/main.tf @@ -0,0 +1,37 @@ +#Terraform Configuration + +variable "DO_API_TOKEN" { + description = "DigitalOcean Access Token" +} + +variable "TESTNET_NAME" { + description = "Name of the testnet" + default = "remotenet" +} + +variable "SSH_KEY_FILE" { + description = "SSH public key file to be used on the nodes" + type = "string" +} + +variable "SERVERS" { + description = "Number of nodes in testnet" + default = "4" +} + +provider "digitalocean" { + token = "${var.DO_API_TOKEN}" +} + +module "cluster" { + source = "./cluster" + name = "${var.TESTNET_NAME}" + ssh_key = "${var.SSH_KEY_FILE}" + servers = "${var.SERVERS}" +} + + +output "public_ips" { + value = "${module.cluster.public_ips}" +} + From b3904b8da8e077e8e17cf647e07bfa6f155b37b5 Mon Sep 17 00:00:00 2001 From: Greg Szabo Date: Fri, 13 Apr 2018 21:34:05 -0400 Subject: [PATCH 108/143] ShellChecked wrapper.sh --- networks/local/localnode/wrapper.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/networks/local/localnode/wrapper.sh b/networks/local/localnode/wrapper.sh index 8b55cbc10..e82741e68 100755 --- a/networks/local/localnode/wrapper.sh +++ b/networks/local/localnode/wrapper.sh @@ -11,10 +11,10 @@ LOG=${LOG:-tendermint.log} ## Assert linux binary ## if ! [ -f "${BINARY}" ]; then - echo "The binary `basename ${BINARY}` cannot be found. Please add the binary to the shared folder. Please use the BINARY environment variable if the name of the binary is not 'tendermint' E.g.: -e BINARY=tendermint_my_test_version" + echo "The binary $(basename "${BINARY}") cannot be found. Please add the binary to the shared folder. Please use the BINARY environment variable if the name of the binary is not 'tendermint' E.g.: -e BINARY=tendermint_my_test_version" exit 1 fi -BINARY_CHECK="`file $BINARY | grep 'ELF 64-bit LSB executable, x86-64'`" +BINARY_CHECK="$(file "$BINARY" | grep 'ELF 64-bit LSB executable, x86-64')" if [ -z "${BINARY_CHECK}" ]; then echo "Binary needs to be OS linux, ARCH amd64" exit 1 @@ -26,8 +26,8 @@ fi export TMHOME="/tendermint/node${ID}" if [ -d "${TMHOME}/${LOG}" ]; then - "$BINARY" $@ | tee "${TMHOME}/${LOG}" + "$BINARY" "$@" | tee "${TMHOME}/${LOG}" else - "$BINARY" $@ + "$BINARY" "$@" fi From 8bdfe15de90fbd77862b1cee91bcc5efaad1e23a Mon Sep 17 00:00:00 2001 From: Greg Szabo Date: Mon, 16 Apr 2018 11:34:01 -0400 Subject: [PATCH 109/143] Dockerfile, localnode, sentry node scripts changes - Updated Dockerfile and created build-docker target - Changed localnode docker image to set permissions to more permissive (docker has different user than host system) - Added sentry node terraform and ansible script --- DOCKER/.gitignore | 1 + DOCKER/Dockerfile | 59 ++++++------------- Makefile | 27 ++++++--- networks/local/localnode/wrapper.sh | 2 + .../ansible/{reconfig.yml => config.yml} | 6 +- .../ansible/roles/config/tasks/main.yml | 4 +- .../ansible/roles/unsafe_reset/tasks/main.yml | 3 +- 7 files changed, 48 insertions(+), 54 deletions(-) create mode 100644 DOCKER/.gitignore mode change 100644 => 100755 Makefile rename networks/remote/ansible/{reconfig.yml => config.yml} (78%) diff --git a/DOCKER/.gitignore b/DOCKER/.gitignore new file mode 100644 index 000000000..9059c6848 --- /dev/null +++ b/DOCKER/.gitignore @@ -0,0 +1 @@ +tendermint diff --git a/DOCKER/Dockerfile b/DOCKER/Dockerfile index cf45ac2f1..ecd863d88 100644 --- a/DOCKER/Dockerfile +++ b/DOCKER/Dockerfile @@ -1,45 +1,22 @@ FROM alpine:3.7 +MAINTAINER Greg Szabo -# This is the release of tendermint to pull in. -ENV TM_VERSION 0.17.1 -ENV TM_SHA256SUM d57008c63d2d9176861137e38ed203da486febf20ae7d388fb810a75afff8f24 - -# Tendermint will be looking for genesis file in /tendermint (unless you change -# `genesis_file` in config.toml). You can put your config.toml and private -# validator file into /tendermint. -# -# The /tendermint/data dir is used by tendermint to store state. -ENV DATA_ROOT /tendermint -ENV TMHOME $DATA_ROOT - -# Set user right away for determinism -RUN addgroup tmuser && \ - adduser -S -G tmuser tmuser - -# Create directory for persistence and give our user ownership -RUN mkdir -p $DATA_ROOT && \ - chown -R tmuser:tmuser $DATA_ROOT - -# jq and curl used for extracting `pub_key` from private validator while -# deploying tendermint with Kubernetes. It is nice to have bash so the users -# could execute bash commands. -RUN apk add --no-cache bash curl jq - -RUN apk add --no-cache openssl && \ - wget https://github.com/tendermint/tendermint/releases/download/v${TM_VERSION}/tendermint_${TM_VERSION}_linux_amd64.zip && \ - echo "${TM_SHA256SUM} tendermint_${TM_VERSION}_linux_amd64.zip" | sha256sum -c && \ - unzip -d /bin tendermint_${TM_VERSION}_linux_amd64.zip && \ - apk del openssl && \ - rm -f tendermint_${TM_VERSION}_linux_amd64.zip - -# Expose the data directory as a volume since there's mutable state in there -VOLUME $DATA_ROOT - -# p2p port -EXPOSE 46656 -# rpc port -EXPOSE 46657 - -ENTRYPOINT ["tendermint"] +#Default home for tendermint. The node command will look for $TMHOME/config/genesis.json at initialization. +ENV TMHOME /tendermint +RUN apk update && \ + apk upgrade && \ + apk --no-cache add curl jq && \ + addgroup tmuser && \ + adduser -S -G tmuser tmuser -h "$TMHOME" +USER tmuser +VOLUME [ $TMHOME ] +WORKDIR $TMHOME +EXPOSE 46656 46657 +ENTRYPOINT ["/usr/bin/tendermint"] CMD ["node", "--moniker=`hostname`"] +STOPSIGNAL SIGTERM + +ARG BINARY=tendermint +COPY $BINARY /usr/bin/tendermint + diff --git a/Makefile b/Makefile old mode 100644 new mode 100755 index 5e5b96754..809352509 --- a/Makefile +++ b/Makefile @@ -178,6 +178,14 @@ metalinter_all: @echo "--> Running linter (all)" gometalinter.v2 --vendor --deadline=600s --enable-all --disable=lll ./... +########################################################### +### Docker image + +build-docker: + cp build/tendermint DOCKER/tendermint + docker build --label=tendermint --tag="tendermint/tendermint" DOCKER + rm -rf DOCKER/tendermint + ########################################################### ### Local testnet using docker @@ -199,22 +207,23 @@ docker-stop: # Server management server-setup: + @if [ -z "$(DO_API_TOKEN)" ]; then echo "DO_API_TOKEN environment variable not set." ; false ; fi @if ! [ -f $(HOME)/.ssh/id_rsa.pub ]; then ssh-keygen ; fi cd networks/remote/terraform && terraform init && terraform apply -var DO_API_TOKEN="$(DO_API_TOKEN)" -var SSH_KEY_FILE="$(HOME)/.ssh/id_rsa.pub" -# @if ! [ -f $(CURDIR)/build/node0/config/genesis.json ]; then docker run --rm -v $(CURDIR)/build:/tendermint:Z tendermint/localnode testnet --n 4 --o . ; fi - build/tendermint testnet --n 4 --o build/ - cd networks/remote/ansible && ansible-playbook -i inventory/digital_ocean.py -l remotenet --ssh-common-args '-o StrictHostKeyChecking=False' install.yml - $(MAKE) server-config - -server-destroy: - cd networks/remote/terraform && terraform destroy + @if ! [ -f $(CURDIR)/build/node0/config/genesis.json ]; then docker run --rm -v $(CURDIR)/build:/tendermint:Z tendermint/localnode testnet --v 0 --n 4 --o . ; fi + cd networks/remote/ansible && ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook -i inventory/digital_ocean.py -l remotenet install.yml + @echo "Next step: Add your validator setup in the genesis.json and config.tml files and run \"make server-config\". (Public key of validator, chain ID, peer IP and node ID.)" # Configuration management server-config: - cd networks/remote/ansible && ansible-playbook -i inventory/digital_ocean.py -l remotenet reconfig.yml -e BINARY=$(CURDIR)/build/tendermint -e CONFIGDIR=$(CURDIR)/build + cd networks/remote/ansible && ansible-playbook -i inventory/digital_ocean.py -l remotenet config.yml -e BINARY=$(CURDIR)/build/tendermint -e CONFIGDIR=$(CURDIR)/build + +server-destroy: + @if [ -z "$(DO_API_TOKEN)" ]; then echo "DO_API_TOKEN environment variable not set." ; false ; fi + cd networks/remote/terraform && terraform destroy -var DO_API_TOKEN="$(DO_API_TOKEN)" -var SSH_KEY_FILE="$(HOME)/.ssh/id_rsa.pub" # To avoid unintended conflicts with file names, always add to .PHONY # unless there is a reason not to. # https://www.gnu.org/software/make/manual/html_node/Phony-Targets.html -.PHONY: check build build_race dist install check_tools get_tools update_tools get_vendor_deps draw_deps test_cover test_apps test_persistence test_p2p test test_race test_integrations test_release test100 vagrant_test fmt build-linux docker-start docker-stop +.PHONY: check build build_race dist install check_tools get_tools update_tools get_vendor_deps draw_deps test_cover test_apps test_persistence test_p2p test test_race test_integrations test_release test100 vagrant_test fmt build-linux docker-start docker-stop build-docker server-setup server-config server-destroy diff --git a/networks/local/localnode/wrapper.sh b/networks/local/localnode/wrapper.sh index e82741e68..83332a21c 100755 --- a/networks/local/localnode/wrapper.sh +++ b/networks/local/localnode/wrapper.sh @@ -31,3 +31,5 @@ else "$BINARY" "$@" fi +chmod 777 -R /tendermint + diff --git a/networks/remote/ansible/reconfig.yml b/networks/remote/ansible/config.yml similarity index 78% rename from networks/remote/ansible/reconfig.yml rename to networks/remote/ansible/config.yml index 08603ed7c..7b772fb70 100644 --- a/networks/remote/ansible/reconfig.yml +++ b/networks/remote/ansible/config.yml @@ -1,16 +1,18 @@ --- #Requires BINARY and CONFIGDIR variables set. +#N=4 hosts by default. - hosts: all user: root any_errors_fatal: true - gather_facts: no + gather_facts: yes vars: - service: tendermint + - N: 4 roles: - stop - - unsafe_reset - config + - unsafe_reset - start diff --git a/networks/remote/ansible/roles/config/tasks/main.yml b/networks/remote/ansible/roles/config/tasks/main.yml index 1568b5005..5186b6aec 100644 --- a/networks/remote/ansible/roles/config/tasks/main.yml +++ b/networks/remote/ansible/roles/config/tasks/main.yml @@ -7,9 +7,11 @@ mode: 0755 - name: Copy config + when: item <= N and ansible_hostname == 'remotenet-node' ~ item copy: - src: "{{CONFIGDIR}}/node0/" + src: "{{CONFIGDIR}}/node{{item}}/" dest: "/home/{{service}}/.{{service}}/" owner: "{{service}}" group: "{{service}}" + loop: [ 0, 1, 2, 3, 4, 5, 6, 7 ] diff --git a/networks/remote/ansible/roles/unsafe_reset/tasks/main.yml b/networks/remote/ansible/roles/unsafe_reset/tasks/main.yml index 6c5831986..6ac1ec55a 100644 --- a/networks/remote/ansible/roles/unsafe_reset/tasks/main.yml +++ b/networks/remote/ansible/roles/unsafe_reset/tasks/main.yml @@ -1,3 +1,4 @@ -- command: "{{service}} node unsafe_reset_all" +- command: "{{service}} unsafe_reset_all {{ (service != 'tendermint') | ternary('node','') }} --home /home/{{service}}/.{{service}}" become_user: "{{service}}" + become: yes From 5b5acbb343e793608b74fa0c1b48991007ec792f Mon Sep 17 00:00:00 2001 From: Greg Szabo Date: Tue, 17 Apr 2018 12:28:05 -0400 Subject: [PATCH 110/143] Ansible README update and small fixes --- DOCKER/Dockerfile | 21 ++- Makefile | 6 +- networks/remote/ansible/README.rst | 286 ++--------------------------- 3 files changed, 39 insertions(+), 274 deletions(-) diff --git a/DOCKER/Dockerfile b/DOCKER/Dockerfile index ecd863d88..de7f5a1d0 100644 --- a/DOCKER/Dockerfile +++ b/DOCKER/Dockerfile @@ -1,18 +1,35 @@ FROM alpine:3.7 MAINTAINER Greg Szabo -#Default home for tendermint. The node command will look for $TMHOME/config/genesis.json at initialization. +# Tendermint will be looking for the genesis file in /tendermint/config/genesis.json +# (unless you change `genesis_file` in config.toml). You can put your config.toml and +# private validator file into /tendermint/config. +# +# The /tendermint/data dir is used by tendermint to store state. ENV TMHOME /tendermint + +# OS environment setup +# Set user right away for determinism, create directory for persistence and give our user ownership +# jq and curl used for extracting `pub_key` from private validator while +# deploying tendermint with Kubernetes. It is nice to have bash so the users +# could execute bash commands. RUN apk update && \ apk upgrade && \ - apk --no-cache add curl jq && \ + apk --no-cache add curl jq bash && \ addgroup tmuser && \ adduser -S -G tmuser tmuser -h "$TMHOME" +# Run the container with tmuser by default. (UID=100, GID=1000) USER tmuser + +# Expose the data directory as a volume since there's mutable state in there VOLUME [ $TMHOME ] + WORKDIR $TMHOME + +# p2p and rpc port EXPOSE 46656 46657 + ENTRYPOINT ["/usr/bin/tendermint"] CMD ["node", "--moniker=`hostname`"] STOPSIGNAL SIGTERM diff --git a/Makefile b/Makefile index 809352509..3ce4f291b 100755 --- a/Makefile +++ b/Makefile @@ -194,12 +194,12 @@ build-linux: GOOS=linux GOARCH=amd64 $(MAKE) build # Run a 4-node testnet locally -docker-start: +localnet-start: @if ! [ -f build/node0/config/genesis.json ]; then docker run --rm -v $(CURDIR)/build:/tendermint:Z tendermint/localnode testnet --v 4 --o . --populate-persistent-peers --starting-ip-address 192.167.10.2 ; fi docker-compose up # Stop testnet -docker-stop: +localnet-stop: docker-compose down ########################################################### @@ -225,5 +225,5 @@ server-destroy: # To avoid unintended conflicts with file names, always add to .PHONY # unless there is a reason not to. # https://www.gnu.org/software/make/manual/html_node/Phony-Targets.html -.PHONY: check build build_race dist install check_tools get_tools update_tools get_vendor_deps draw_deps test_cover test_apps test_persistence test_p2p test test_race test_integrations test_release test100 vagrant_test fmt build-linux docker-start docker-stop build-docker server-setup server-config server-destroy +.PHONY: check build build_race dist install check_tools get_tools update_tools get_vendor_deps draw_deps test_cover test_apps test_persistence test_p2p test test_race test_integrations test_release test100 vagrant_test fmt build-linux localnet-start localnet-stop build-docker server-setup server-config server-destroy diff --git a/networks/remote/ansible/README.rst b/networks/remote/ansible/README.rst index 5c416c018..0f95de73f 100644 --- a/networks/remote/ansible/README.rst +++ b/networks/remote/ansible/README.rst @@ -6,286 +6,34 @@ Using Ansible Ansible plus Tendermint -The playbooks in `our ansible directory `__ -run ansible `roles `__ which: - -- install and configure basecoind or ethermint -- start/stop basecoind or ethermint and reset their configuration +The playbooks in `the ansible directory `__ +run ansible `roles `__ to configure the sentry node architecture. Prerequisites ------------- -- Ansible 2.0 or higher -- SSH key to the servers +- Install `Ansible 2.0 or higher `__ on a linux machine. +- Create a `DigitalOcean API token `__ with read and write capability. +- Create SSH keys +- Install the python dopy package (for the digital_ocean.py script) -Optional for DigitalOcean droplets: -- DigitalOcean API Token -- python dopy package - -For a description on how to get a DigitalOcean API Token, see the explanation -in the `using terraform tutorial <./terraform-digitalocean.html>`__. - -Optional for Amazon AWS instances: - -- Amazon AWS API access key ID and secret access key. - -The cloud inventory scripts come from the ansible team at their -`GitHub `__ page. You can get the -latest version from the ``contrib/inventory`` folder. - -Setup +Build ----- -Ansible requires a "command machine" or "local machine" or "orchestrator -machine" to run on. This can be your laptop or any machine that can run -ansible. (It does not have to be part of the cloud network that hosts -your servers.) - -Use the official `Ansible installation -guide `__ to -install Ansible. Here are a few examples on basic installation commands: - -Ubuntu/Debian: - :: - sudo apt-get install ansible + export DO_API_TOKEN="abcdef01234567890abcdef01234567890" + export SSH_KEY_FILE="$HOME/.ssh/id_rsa.pub" -CentOS/RedHat: + + ansible-playbook -i inventory/digital_ocean.py -l remotenet install.yml -:: + # The scripts assume that you have your validator set up already. + # You can create the folder structure for the sentry nodes using `tendermint testnet`. + # For example: tendermint testnet --v 0 --n 4 --o build/ + # Then copy your genesis.json and modify the config.toml as you see fit. - sudo yum install epel-release - sudo yum install ansible + # Reconfig the sentry nodes with a new BINARY and the configuration files from the build folder: + ansible-playbook -i inventory/digital_ocean.py -l remotenet config.yml -e BINARY=`pwd`/build/tendermint -e CONFIGDIR=`pwd`/build -Mac OSX: If you have `Homebrew `__ installed, then it's: - -:: - - brew install ansible - -If not, you can install it using ``pip``: - -:: - - sudo easy_install pip - sudo pip install ansible - -To make life easier, you can start an SSH Agent and load your SSH -key(s). This way ansible will have an uninterrupted way of connecting to -your servers. - -:: - - ssh-agent > ~/.ssh/ssh.env - source ~/.ssh/ssh.env - - ssh-add private.key - -Subsequently, as long as the agent is running, you can use -``source ~/.ssh/ssh.env`` to load the keys to the current session. Note: -On Mac OSX, you can add the ``-K`` option to ssh-add to store the -passphrase in your keychain. The security of this feature is debated but -it is convenient. - -Optional cloud dependencies -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -If you are using a cloud provider to host your servers, you need the -below dependencies installed on your local machine. - -DigitalOcean inventory dependencies: -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Ubuntu/Debian: - -:: - - sudo apt-get install python-pip - sudo pip install dopy - -CentOS/RedHat: - -:: - - sudo yum install python-pip - sudo pip install dopy - -Mac OSX: - -:: - - sudo pip install dopy - -Amazon AWS inventory dependencies: -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Ubuntu/Debian: - -:: - - sudo apt-get install python-boto - -CentOS/RedHat: - -:: - - sudo yum install python-boto - -Mac OSX: - -:: - - sudo pip install boto - -Refreshing the DigitalOcean inventory -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If you just finished creating droplets, the local DigitalOcean inventory -cache is not up-to-date. To refresh it, run: - -:: - - DO_API_TOKEN="" - python -u inventory/digital_ocean.py --refresh-cache 1> /dev/null - -Refreshing the Amazon AWS inventory -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If you just finished creating Amazon AWS EC2 instances, the local AWS -inventory cache is not up-to-date. To refresh it, run: - -:: - - AWS_ACCESS_KEY_ID='' - AWS_SECRET_ACCESS_KEY='' - python -u inventory/ec2.py --refresh-cache 1> /dev/null - -Note: you don't need the access key and secret key set, if you are -running ansible on an Amazon AMI instance with the proper IAM -permissions set. - -Running the playbooks ---------------------- - -The playbooks are locked down to only run if the environment variable -``TF_VAR_TESTNET_NAME`` is populated. This is a precaution so you don't -accidentally run the playbook on all your servers. - -The variable ``TF_VAR_TESTNET_NAME`` contains the testnet name which -ansible translates into an ansible group. If you used Terraform to -create the servers, it was the testnet name used there. - -If the playbook cannot connect to the servers because of public key -denial, your SSH Agent is not set up properly. Alternatively you can add -the SSH key to ansible using the ``--private-key`` option. - -If you need to connect to the nodes as root but your local username is -different, use the ansible option ``-u root`` to tell ansible to connect -to the servers and authenticate as the root user. - -If you secured your server and you need to ``sudo`` for root access, use -the the ``-b`` or ``--become`` option to tell ansible to sudo to root -after connecting to the server. In the Terraform-DigitalOcean example, -if you created the ec2-user by adding the ``noroot=true`` option (or if -you are simply on Amazon AWS), you need to add the options -``-u ec2-user -b`` to ansible to tell it to connect as the ec2-user and -then sudo to root to run the playbook. - -DigitalOcean -~~~~~~~~~~~~ - -:: - - DO_API_TOKEN="" - TF_VAR_TESTNET_NAME="testnet-servers" - ansible-playbook -i inventory/digital_ocean.py install.yml -e service=basecoind - -Amazon AWS -~~~~~~~~~~ - -:: - - AWS_ACCESS_KEY_ID='' - AWS_SECRET_ACCESS_KEY='' - TF_VAR_TESTNET_NAME="testnet-servers" - ansible-playbook -i inventory/ec2.py install.yml -e service=basecoind - -Installing custom versions -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -By default ansible installs the tendermint, basecoind or ethermint binary -versions from the latest release in the repository. If you build your -own version of the binaries, you can tell ansible to install that -instead. - -:: - - GOPATH="" - go get -u github.com/tendermint/basecoin/cmd/basecoind - - DO_API_TOKEN="" - TF_VAR_TESTNET_NAME="testnet-servers" - ansible-playbook -i inventory/digital_ocean.py install.yml -e service=basecoind -e release_install=false - -Alternatively you can change the variable settings in -``group_vars/all``. - -Other commands and roles ------------------------- - -There are few extra playbooks to make life easier managing your servers. - -- install.yml - Install basecoind or ethermint applications. (Tendermint - gets installed automatically.) Use the ``service`` parameter to - define which application to install. Defaults to ``basecoind``. -- reset.yml - Stop the application, reset the configuration and data, - then start the application again. You need to pass - ``-e service=``, like ``-e service=basecoind``. It will - restart the underlying tendermint application too. -- restart.yml - Restart a service on all nodes. You need to pass - ``-e service=``, like ``-e service=basecoind``. It will - restart the underlying tendermint application too. -- stop.yml - Stop the application. You need to pass - ``-e service=``. -- status.yml - Check the service status and print it. You need to pass - ``-e service=``. -- start.yml - Start the application. You need to pass - ``-e service=``. -- ubuntu16-patch.yml - Ubuntu 16.04 does not have the minimum required - python package installed to be able to run ansible. If you are using - ubuntu, run this playbook first on the target machines. This will - install the python pacakge that is required for ansible to work - correctly on the remote nodes. -- upgrade.yml - Upgrade the ``service`` on your testnet. It will stop - the service and restart it at the end. It will only work if the - upgraded version is backward compatible with the installed version. -- upgrade-reset.yml - Upgrade the ``service`` on your testnet and reset - the database. It will stop the service and restart it at the end. It - will work for upgrades where the new version is not - backward-compatible with the installed version - however it will - reset the testnet to its default. - -The roles are self-sufficient under the ``roles/`` folder. - -- install - install the application defined in the ``service`` - parameter. It can install release packages and update them with - custom-compiled binaries. -- unsafe\_reset - delete the database for a service, including the - tendermint database. -- config - configure the application defined in ``service``. It also - configures the underlying tendermint service. Check - ``group_vars/all`` for options. -- stop - stop an application. Requires the ``service`` parameter set. -- status - check the status of an application. Requires the ``service`` - parameter set. -- start - start an application. Requires the ``service`` parameter set. - -Default variables ------------------ - -Default variables are documented under ``group_vars/all``. You can the -parameters there to deploy a previously created genesis.json file -(instead of dynamically creating it) or if you want to deploy custom -built binaries instead of deploying a released version. From ece3f678da00de1c171e29f5f10afe88bfa416ec Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Tue, 17 Apr 2018 19:38:10 +0200 Subject: [PATCH 111/143] [docs/spec] update msg type and Tendermint behavior (#1468) Refs #1422 --- docs/specification/genesis.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/specification/genesis.rst b/docs/specification/genesis.rst index 5edd1b7b9..427c88bb2 100644 --- a/docs/specification/genesis.rst +++ b/docs/specification/genesis.rst @@ -18,8 +18,8 @@ Fields - ``power``: The validator's voting power. - ``name``: Name of the validator (optional). - ``app_hash``: The expected application hash (as returned by the - ``Commit`` ABCI message) upon genesis. If the app's hash does not - match, a warning message is printed. + ``ResponseInfo`` ABCI message) upon genesis. If the app's hash does not + match, Tendermint will panic. - ``app_state``: The application state (e.g. initial distribution of tokens). Sample genesis.json From 659762736c2a4c0b38bce80de8cf747464cf6405 Mon Sep 17 00:00:00 2001 From: Greg Szabo Date: Wed, 18 Apr 2018 05:56:13 -0400 Subject: [PATCH 112/143] Makefile targets for remotenet fixed --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 3ce4f291b..8c818bf17 100755 --- a/Makefile +++ b/Makefile @@ -206,7 +206,7 @@ localnet-stop: ### Remote full-nodes (sentry) using terraform and ansible # Server management -server-setup: +remotenet-start: @if [ -z "$(DO_API_TOKEN)" ]; then echo "DO_API_TOKEN environment variable not set." ; false ; fi @if ! [ -f $(HOME)/.ssh/id_rsa.pub ]; then ssh-keygen ; fi cd networks/remote/terraform && terraform init && terraform apply -var DO_API_TOKEN="$(DO_API_TOKEN)" -var SSH_KEY_FILE="$(HOME)/.ssh/id_rsa.pub" @@ -215,15 +215,15 @@ server-setup: @echo "Next step: Add your validator setup in the genesis.json and config.tml files and run \"make server-config\". (Public key of validator, chain ID, peer IP and node ID.)" # Configuration management -server-config: +remotenet-config: cd networks/remote/ansible && ansible-playbook -i inventory/digital_ocean.py -l remotenet config.yml -e BINARY=$(CURDIR)/build/tendermint -e CONFIGDIR=$(CURDIR)/build -server-destroy: +remotenet-stop: @if [ -z "$(DO_API_TOKEN)" ]; then echo "DO_API_TOKEN environment variable not set." ; false ; fi cd networks/remote/terraform && terraform destroy -var DO_API_TOKEN="$(DO_API_TOKEN)" -var SSH_KEY_FILE="$(HOME)/.ssh/id_rsa.pub" # To avoid unintended conflicts with file names, always add to .PHONY # unless there is a reason not to. # https://www.gnu.org/software/make/manual/html_node/Phony-Targets.html -.PHONY: check build build_race dist install check_tools get_tools update_tools get_vendor_deps draw_deps test_cover test_apps test_persistence test_p2p test test_race test_integrations test_release test100 vagrant_test fmt build-linux localnet-start localnet-stop build-docker server-setup server-config server-destroy +.PHONY: check build build_race dist install check_tools get_tools update_tools get_vendor_deps draw_deps test_cover test_apps test_persistence test_p2p test test_race test_integrations test_release test100 vagrant_test fmt build-linux localnet-start localnet-stop build-docker remotenet-start remotenet-config remotenet-stop From b20e777f532b55e07c3320842b4207b6c6a9c34b Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Fri, 20 Apr 2018 23:20:44 -0700 Subject: [PATCH 113/143] lower_case json field names --- consensus/types/state.go | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/consensus/types/state.go b/consensus/types/state.go index 0c3626cc7..255595e54 100644 --- a/consensus/types/state.go +++ b/consensus/types/state.go @@ -55,25 +55,25 @@ func (rs RoundStepType) String() string { // NOTE: Not thread safe. Should only be manipulated by functions downstream // of the cs.receiveRoutine type RoundState struct { - Height int64 // Height we are working on - Round int - Step RoundStepType - StartTime time.Time - CommitTime time.Time // Subjective time when +2/3 precommits for Block at Round were found - Validators *types.ValidatorSet - Proposal *types.Proposal - ProposalBlock *types.Block - ProposalBlockParts *types.PartSet - LockedRound int - LockedBlock *types.Block - LockedBlockParts *types.PartSet - ValidRound int - ValidBlock *types.Block - ValidBlockParts *types.PartSet - Votes *HeightVoteSet - CommitRound int // - LastCommit *types.VoteSet // Last precommits at Height-1 - LastValidators *types.ValidatorSet + Height int64 `json:"height"` // Height we are working on + Round int `json:"round"` + Step RoundStepType `json:"step"` + StartTime time.Time `json:"start_time"` + CommitTime time.Time `json:"commit_time"` // Subjective time when +2/3 precommits for Block at Round were found + Validators *types.ValidatorSet `json:"validators"` + Proposal *types.Proposal `json:"proposal"` + ProposalBlock *types.Block `json:"proposal_block"` + ProposalBlockParts *types.PartSet `json:"proposal_block_parts"` + LockedRound int `json:"locked_round"` + LockedBlock *types.Block `json:"locked_block"` + LockedBlockParts *types.PartSet `json:"locked_block_parts"` + ValidRound int `json:"valid_round"` + ValidBlock *types.Block `json:"valid_block"` + ValidBlockParts *types.PartSet `json:"valid_block_parts"` + Votes *HeightVoteSet `json:"votes"` + CommitRound int `json:"commit_round"` // + LastCommit *types.VoteSet `json:"last_commit"` // Last precommits at Height-1 + LastValidators *types.ValidatorSet `json:"last_validators"` } // RoundStateEvent returns the H/R/S of the RoundState as an event. From b42d5a2211a818f46b54af114e4340e40039a63a Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Sat, 21 Apr 2018 20:24:50 -0700 Subject: [PATCH 114/143] blockID -> block_id for JSON --- types/block.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/types/block.go b/types/block.go index 34d2cb85d..9cfef1f94 100644 --- a/types/block.go +++ b/types/block.go @@ -265,7 +265,7 @@ type Commit struct { // NOTE: The Precommits are in order of address to preserve the bonded ValidatorSet order. // Any peer with a block can gossip precommits by index with a peer without recalculating the // active ValidatorSet. - BlockID BlockID `json:"blockID"` + BlockID BlockID `json:"block_id"` Precommits []*Vote `json:"precommits"` // Volatile From 18c3f8f3f132632da1f91855330e6761e18aa1b4 Mon Sep 17 00:00:00 2001 From: Rigel Date: Mon, 23 Apr 2018 15:01:40 -0400 Subject: [PATCH 115/143] writeDefaultCondigFile -> writeDefaultConfigFile (#1490) --- config/toml.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/toml.go b/config/toml.go index 71b8b2e81..f1c869355 100644 --- a/config/toml.go +++ b/config/toml.go @@ -37,13 +37,13 @@ func EnsureRoot(rootDir string) { // Write default config file if missing. if !cmn.FileExists(configFilePath) { - writeDefaultCondigFile(configFilePath) + writeDefaultConfigFile(configFilePath) } } // XXX: this func should probably be called by cmd/tendermint/commands/init.go // alongside the writing of the genesis.json and priv_validator.json -func writeDefaultCondigFile(configFilePath string) { +func writeDefaultConfigFile(configFilePath string) { WriteConfigFile(configFilePath, DefaultConfig()) } @@ -267,7 +267,7 @@ func ResetTestRoot(testName string) *Config { // Write default config file if missing. if !cmn.FileExists(configFilePath) { - writeDefaultCondigFile(configFilePath) + writeDefaultConfigFile(configFilePath) } if !cmn.FileExists(genesisFilePath) { cmn.MustWriteFile(genesisFilePath, []byte(testGenesis), 0644) From 17a5c6fa1ae41f141ac3923e35ee716bebebe174 Mon Sep 17 00:00:00 2001 From: Max Levy <35595512+maxim-levy@users.noreply.github.com> Date: Mon, 23 Apr 2018 17:39:01 +0900 Subject: [PATCH 116/143] typo fix --- test/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/README.md b/test/README.md index ee53905bf..fc436948a 100644 --- a/test/README.md +++ b/test/README.md @@ -1,7 +1,7 @@ # Tendermint Tests The unit tests (ie. the `go test` s) can be run with `make test`. -The integration tests can be run wtih `make test_integrations`. +The integration tests can be run with `make test_integrations`. Running the integrations test will build a docker container with local version of tendermint and run the following tests in docker containers: From 089ce6744cb526d06f3b3cb22d5dbd05c435c381 Mon Sep 17 00:00:00 2001 From: Greg Szabo Date: Mon, 23 Apr 2018 22:38:49 -0400 Subject: [PATCH 117/143] Added ansible playbook to remote networks to ship logs to logz.io --- networks/remote/ansible/README.rst | 15 +- networks/remote/ansible/logzio.yml | 13 + .../roles/logzio/files/journalbeat.service | 15 + .../ansible/roles/logzio/handlers/main.yml | 8 + .../ansible/roles/logzio/tasks/main.yml | 27 ++ .../roles/logzio/templates/journalbeat.yml.j2 | 342 ++++++++++++++++++ 6 files changed, 419 insertions(+), 1 deletion(-) create mode 100644 networks/remote/ansible/logzio.yml create mode 100644 networks/remote/ansible/roles/logzio/files/journalbeat.service create mode 100644 networks/remote/ansible/roles/logzio/handlers/main.yml create mode 100644 networks/remote/ansible/roles/logzio/tasks/main.yml create mode 100644 networks/remote/ansible/roles/logzio/templates/journalbeat.yml.j2 diff --git a/networks/remote/ansible/README.rst b/networks/remote/ansible/README.rst index 0f95de73f..e9f2825c6 100644 --- a/networks/remote/ansible/README.rst +++ b/networks/remote/ansible/README.rst @@ -17,7 +17,6 @@ Prerequisites - Create SSH keys - Install the python dopy package (for the digital_ocean.py script) - Build ----- @@ -37,3 +36,17 @@ Build # Reconfig the sentry nodes with a new BINARY and the configuration files from the build folder: ansible-playbook -i inventory/digital_ocean.py -l remotenet config.yml -e BINARY=`pwd`/build/tendermint -e CONFIGDIR=`pwd`/build +Shipping logs to logz.io +------------------------ + +Logz.io is an Elastic stack (Elastic search, Logstash and Kibana) service provider. You can set up your nodes to log there automatically. Create an account and get your API key from the notes on `this page `__. + +:: + + yum install systemd-devel || echo "This will only work on RHEL-based systems." + apt-get install libsystemd-dev || echo "This will only work on Debian-based systems." + + go get github.com/mheese/journalbeat + ansible-playbook -i inventory/digital_ocean.py -l remotenet logzio.yml -e LOGZIO_TOKEN=ABCDEFGHIJKLMNOPQRSTUVWXYZ012345 + + diff --git a/networks/remote/ansible/logzio.yml b/networks/remote/ansible/logzio.yml new file mode 100644 index 000000000..7e5148813 --- /dev/null +++ b/networks/remote/ansible/logzio.yml @@ -0,0 +1,13 @@ +--- + +- hosts: all + user: root + any_errors_fatal: true + gather_facts: no + vars: + - service: tendermint + - JOURNALBEAT_BINARY: "{{lookup('env', 'GOPATH')}}/bin/journalbeat" +# - LOGZIO_TOKEN: ABCDEFGHIJKLMNOPQRSTUVWXYZ012345 + roles: + - logzio + diff --git a/networks/remote/ansible/roles/logzio/files/journalbeat.service b/networks/remote/ansible/roles/logzio/files/journalbeat.service new file mode 100644 index 000000000..3cb66a454 --- /dev/null +++ b/networks/remote/ansible/roles/logzio/files/journalbeat.service @@ -0,0 +1,15 @@ +[Unit] +Description=journalbeat +#propagates activation, deactivation and activation fails. +Requires=network-online.target +After=network-online.target + +[Service] +Restart=on-failure +ExecStart=/usr/bin/journalbeat -c /etc/journalbeat/journalbeat.yml -path.home /usr/share/journalbeat -path.config /etc/journalbeat -path.data /var/lib/journalbeat -path.logs /var/log/journalbeat +Restart=always + +[Install] +WantedBy=multi-user.target + + diff --git a/networks/remote/ansible/roles/logzio/handlers/main.yml b/networks/remote/ansible/roles/logzio/handlers/main.yml new file mode 100644 index 000000000..0b371fc51 --- /dev/null +++ b/networks/remote/ansible/roles/logzio/handlers/main.yml @@ -0,0 +1,8 @@ +--- + +- name: reload daemon + command: "systemctl daemon-reload" + +- name: restart journalbeat + service: name=journalbeat state=restarted + diff --git a/networks/remote/ansible/roles/logzio/tasks/main.yml b/networks/remote/ansible/roles/logzio/tasks/main.yml new file mode 100644 index 000000000..ab3976f22 --- /dev/null +++ b/networks/remote/ansible/roles/logzio/tasks/main.yml @@ -0,0 +1,27 @@ +--- + +- name: Copy journalbeat binary + copy: src="{{JOURNALBEAT_BINARY}}" dest=/usr/bin/journalbeat mode=0755 + notify: restart journalbeat + +- name: Create folders + file: "path={{item}} state=directory recurse=yes" + with_items: + - /etc/journalbeat + - /etc/pki/tls/certs + - /usr/share/journalbeat + - /var/log/journalbeat + +- name: Copy journalbeat config + template: src=journalbeat.yml.j2 dest=/etc/journalbeat/journalbeat.yml mode=0600 + notify: restart journalbeat + +- name: Get server certificate for Logz.io + get_url: "url=https://raw.githubusercontent.com/logzio/public-certificates/master/COMODORSADomainValidationSecureServerCA.crt force=yes dest=/etc/pki/tls/certs/COMODORSADomainValidationSecureServerCA.crt" + +- name: Copy journalbeat service config + copy: src=journalbeat.service dest=/etc/systemd/system/journalbeat.service + notify: + - reload daemon + - restart journalbeat + diff --git a/networks/remote/ansible/roles/logzio/templates/journalbeat.yml.j2 b/networks/remote/ansible/roles/logzio/templates/journalbeat.yml.j2 new file mode 100644 index 000000000..a421ec8a5 --- /dev/null +++ b/networks/remote/ansible/roles/logzio/templates/journalbeat.yml.j2 @@ -0,0 +1,342 @@ +#======================== Journalbeat Configuration ============================ + +journalbeat: + # What position in journald to seek to at start up + # options: cursor, tail, head (defaults to tail) + #seek_position: tail + + # If seek_position is set to cursor and seeking to cursor fails + # fall back to this method. If set to none will it will exit + # options: tail, head, none (defaults to tail) + #cursor_seek_fallback: tail + + # Store the cursor of the successfully published events + #write_cursor_state: true + + # Path to the file to store the cursor (defaults to ".journalbeat-cursor-state") + #cursor_state_file: .journalbeat-cursor-state + + # How frequently should we save the cursor to disk (defaults to 5s) + #cursor_flush_period: 5s + + # Path to the file to store the queue of events pending (defaults to ".journalbeat-pending-queue") + #pending_queue.file: .journalbeat-pending-queue + + # How frequently should we save the queue to disk (defaults to 1s). + # Pending queue represents the WAL of events queued to be published + # or being published and waiting for acknowledgement. In case of a + # regular restart of journalbeat all the events not yet acknowledged + # will be flushed to disk during the shutdown. + # In case of disaster most probably journalbeat won't get a chance to shutdown + # itself gracefully and this flush period option will serve you as a + # backup creation frequency option. + #pending_queue.flush_period: 1s + + # Lowercase and remove leading underscores, e.g. "_MESSAGE" -> "message" + # (defaults to false) + #clean_field_names: false + + # All journal entries are strings by default. You can try to convert them to numbers. + # (defaults to false) + #convert_to_numbers: false + + # Store all the fields of the Systemd Journal entry under this field + # Can be almost any string suitable to be a field name of an ElasticSearch document. + # Dots can be used to create nested fields. + # Two exceptions: + # - no repeated dots; + # - no trailing dots, e.g. "journal..field_name." will fail + # (defaults to "" hence stores on the upper level of the event) + #move_metadata_to_field: "" + + # Specific units to monitor. + units: ["{{service}}.service"] + + # Specify Journal paths to open. You can pass an array of paths to Systemd Journal paths. + # If you want to open Journal from directory just pass an array consisting of one element + # representing the path. See: https://www.freedesktop.org/software/systemd/man/sd_journal_open.html + # By default this setting is empty thus journalbeat will attempt to find all journal files automatically + #journal_paths: ["/var/log/journal"] + + #default_type: journal + +#================================ General ====================================== + +# The name of the shipper that publishes the network data. It can be used to group +# all the transactions sent by a single shipper in the web interface. +# If this options is not defined, the hostname is used. +#name: journalbeat + +# The tags of the shipper are included in their own field with each +# transaction published. Tags make it easy to group servers by different +# logical properties. +tags: ["{{service}}"] + +# Optional fields that you can specify to add additional information to the +# output. Fields can be scalar values, arrays, dictionaries, or any nested +# combination of these. +fields: + logzio_codec: plain + token: {{LOGZIO_TOKEN}} + +# If this option is set to true, the custom fields are stored as top-level +# fields in the output document instead of being grouped under a fields +# sub-dictionary. Default is false. +fields_under_root: true + +# Internal queue size for single events in processing pipeline +#queue_size: 1000 + +# The internal queue size for bulk events in the processing pipeline. +# Do not modify this value. +#bulk_queue_size: 0 + +# Sets the maximum number of CPUs that can be executing simultaneously. The +# default is the number of logical CPUs available in the system. +#max_procs: + +#================================ Processors =================================== + +# Processors are used to reduce the number of fields in the exported event or to +# enhance the event with external metadata. This section defines a list of +# processors that are applied one by one and the first one receives the initial +# event: +# +# event -> filter1 -> event1 -> filter2 ->event2 ... +# +# The supported processors are drop_fields, drop_event, include_fields, and +# add_cloud_metadata. +# +# For example, you can use the following processors to keep the fields that +# contain CPU load percentages, but remove the fields that contain CPU ticks +# values: +# +processors: +#- include_fields: +# fields: ["cpu"] +- drop_fields: + fields: ["beat.name", "beat.version", "logzio_codec", "SYSLOG_IDENTIFIER", "SYSLOG_FACILITY", "PRIORITY"] +# +# The following example drops the events that have the HTTP response code 200: +# +#processors: +#- drop_event: +# when: +# equals: +# http.code: 200 +# +# The following example enriches each event with metadata from the cloud +# provider about the host machine. It works on EC2, GCE, and DigitalOcean. +# +#processors: +#- add_cloud_metadata: +# + +#================================ Outputs ====================================== + +# Configure what outputs to use when sending the data collected by the beat. +# Multiple outputs may be used. + +#----------------------------- Logstash output --------------------------------- +output.logstash: + # Boolean flag to enable or disable the output module. + enabled: true + + # The Logstash hosts + hosts: ["listener.logz.io:5015"] + + # Number of workers per Logstash host. + #worker: 1 + + # Set gzip compression level. + #compression_level: 3 + + # Optional load balance the events between the Logstash hosts + #loadbalance: true + + # Number of batches to be send asynchronously to logstash while processing + # new batches. + #pipelining: 0 + + # Optional index name. The default index name is set to name of the beat + # in all lowercase. + #index: 'beatname' + + # SOCKS5 proxy server URL + #proxy_url: socks5://user:password@socks5-server:2233 + + # Resolve names locally when using a proxy server. Defaults to false. + #proxy_use_local_resolver: false + + # Enable SSL support. SSL is automatically enabled, if any SSL setting is set. + ssl.enabled: true + + # Configure SSL verification mode. If `none` is configured, all server hosts + # and certificates will be accepted. In this mode, SSL based connections are + # susceptible to man-in-the-middle attacks. Use only for testing. Default is + # `full`. + ssl.verification_mode: full + + # List of supported/valid TLS versions. By default all TLS versions 1.0 up to + # 1.2 are enabled. + #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2] + + # Optional SSL configuration options. SSL is off by default. + # List of root certificates for HTTPS server verifications + ssl.certificate_authorities: ["/etc/pki/tls/certs/COMODORSADomainValidationSecureServerCA.crt"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #ssl.key: "/etc/pki/client/cert.key" + + # Optional passphrase for decrypting the Certificate Key. + #ssl.key_passphrase: '' + + # Configure cipher suites to be used for SSL connections + #ssl.cipher_suites: [] + + # Configure curve types for ECDHE based cipher suites + #ssl.curve_types: [] + +#------------------------------- File output ----------------------------------- +#output.file: + # Boolean flag to enable or disable the output module. + #enabled: true + + # Path to the directory where to save the generated files. The option is + # mandatory. + #path: "/tmp/beatname" + + # Name of the generated files. The default is `beatname` and it generates + # files: `beatname`, `beatname.1`, `beatname.2`, etc. + #filename: beatname + + # Maximum size in kilobytes of each file. When this size is reached, and on + # every beatname restart, the files are rotated. The default value is 10240 + # kB. + #rotate_every_kb: 10000 + + # Maximum number of files under path. When this number of files is reached, + # the oldest file is deleted and the rest are shifted from last to first. The + # default is 7 files. + #number_of_files: 7 + + +#----------------------------- Console output --------------------------------- +#output.console: + # Boolean flag to enable or disable the output module. + #enabled: true + + # Pretty print json event + #pretty: false + +#================================= Paths ====================================== + +# The home path for the beatname installation. This is the default base path +# for all other path settings and for miscellaneous files that come with the +# distribution (for example, the sample dashboards). +# If not set by a CLI flag or in the configuration file, the default for the +# home path is the location of the binary. +#path.home: + +# The configuration path for the beatname installation. This is the default +# base path for configuration files, including the main YAML configuration file +# and the Elasticsearch template file. If not set by a CLI flag or in the +# configuration file, the default for the configuration path is the home path. +#path.config: ${path.home} + +# The data path for the beatname installation. This is the default base path +# for all the files in which beatname needs to store its data. If not set by a +# CLI flag or in the configuration file, the default for the data path is a data +# subdirectory inside the home path. +#path.data: ${path.home}/data + +# The logs path for a beatname installation. This is the default location for +# the Beat's log files. If not set by a CLI flag or in the configuration file, +# the default for the logs path is a logs subdirectory inside the home path. +#path.logs: ${path.home}/logs + +#============================== Dashboards ===================================== +# These settings control loading the sample dashboards to the Kibana index. Loading +# the dashboards is disabled by default and can be enabled either by setting the +# options here, or by using the `-setup` CLI flag. +#dashboards.enabled: false + +# The URL from where to download the dashboards archive. By default this URL +# has a value which is computed based on the Beat name and version. For released +# versions, this URL points to the dashboard archive on the artifacts.elastic.co +# website. +#dashboards.url: + +# The directory from where to read the dashboards. It is used instead of the URL +# when it has a value. +#dashboards.directory: + +# The file archive (zip file) from where to read the dashboards. It is used instead +# of the URL when it has a value. +#dashboards.file: + +# If this option is enabled, the snapshot URL is used instead of the default URL. +#dashboards.snapshot: false + +# The URL from where to download the snapshot version of the dashboards. By default +# this has a value which is computed based on the Beat name and version. +#dashboards.snapshot_url + +# In case the archive contains the dashboards from multiple Beats, this lets you +# select which one to load. You can load all the dashboards in the archive by +# setting this to the empty string. +#dashboards.beat: beatname + +# The name of the Kibana index to use for setting the configuration. Default is ".kibana" +#dashboards.kibana_index: .kibana + +# The Elasticsearch index name. This overwrites the index name defined in the +# dashboards and index pattern. Example: testbeat-* +#dashboards.index: + +#================================ Logging ====================================== +# There are three options for the log output: syslog, file, stderr. +# Under Windows systems, the log files are per default sent to the file output, +# under all other system per default to syslog. + +# Sets log level. The default log level is info. +# Available log levels are: critical, error, warning, info, debug +#logging.level: info + +# Enable debug output for selected components. To enable all selectors use ["*"] +# Other available selectors are "beat", "publish", "service" +# Multiple selectors can be chained. +#logging.selectors: [ ] + +# Send all logging output to syslog. The default is false. +#logging.to_syslog: true + +# If enabled, beatname periodically logs its internal metrics that have changed +# in the last period. For each metric that changed, the delta from the value at +# the beginning of the period is logged. Also, the total values for +# all non-zero internal metrics are logged on shutdown. The default is true. +#logging.metrics.enabled: true + +# The period after which to log the internal metrics. The default is 30s. +#logging.metrics.period: 30s + +# Logging to rotating files files. Set logging.to_files to false to disable logging to +# files. +logging.to_files: true +logging.files: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + #path: /var/log/beatname + + # The name of the files where the logs are written to. + #name: beatname + + # Configure log file size limit. If limit is reached, log file will be + # automatically rotated + #rotateeverybytes: 10485760 # = 10MB + + # Number of rotated log files to keep. Oldest files will be deleted first. + #keepfiles: 7 From 6c04465d3d11e4192d2b593de4961cb66031bd53 Mon Sep 17 00:00:00 2001 From: suyuhuang Date: Tue, 24 Apr 2018 15:56:25 +0800 Subject: [PATCH 118/143] Fix `Prove` int abci.ABCIQuery (#1485) * fix Prove in abci query * fix Prove in abci query * fix doc * fix doc --- rpc/core/abci.go | 2 +- rpc/core/routes.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rpc/core/abci.go b/rpc/core/abci.go index 874becae3..1607b6cce 100644 --- a/rpc/core/abci.go +++ b/rpc/core/abci.go @@ -10,7 +10,7 @@ import ( // Query the application for some information. // // ```shell -// curl 'localhost:46657/abci_query?path=""&data="abcd"&prove=true' +// curl 'localhost:46657/abci_query?path=""&data="abcd"&trusted=false' // ``` // // ```go diff --git a/rpc/core/routes.go b/rpc/core/routes.go index f1150f03b..be3881c36 100644 --- a/rpc/core/routes.go +++ b/rpc/core/routes.go @@ -34,7 +34,7 @@ var Routes = map[string]*rpc.RPCFunc{ "broadcast_tx_async": rpc.NewRPCFunc(BroadcastTxAsync, "tx"), // abci API - "abci_query": rpc.NewRPCFunc(ABCIQuery, "path,data,height,prove"), + "abci_query": rpc.NewRPCFunc(ABCIQuery, "path,data,height,trusted"), "abci_info": rpc.NewRPCFunc(ABCIInfo, ""), } From 6ce6b209938d9ef48df7a7c837dac92c36752f36 Mon Sep 17 00:00:00 2001 From: Greg Szabo Date: Tue, 24 Apr 2018 14:22:19 -0400 Subject: [PATCH 119/143] Renamed remotenet to sentrynet to match the purpose better. --- Makefile | 12 ++++++------ networks/remote/ansible/README.rst | 6 +++--- networks/remote/ansible/logzio.yml | 3 ++- networks/remote/ansible/roles/config/tasks/main.yml | 2 +- networks/remote/terraform/main.tf | 2 +- 5 files changed, 13 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 8c818bf17..4601c970c 100755 --- a/Makefile +++ b/Makefile @@ -206,24 +206,24 @@ localnet-stop: ### Remote full-nodes (sentry) using terraform and ansible # Server management -remotenet-start: +sentry-start: @if [ -z "$(DO_API_TOKEN)" ]; then echo "DO_API_TOKEN environment variable not set." ; false ; fi @if ! [ -f $(HOME)/.ssh/id_rsa.pub ]; then ssh-keygen ; fi cd networks/remote/terraform && terraform init && terraform apply -var DO_API_TOKEN="$(DO_API_TOKEN)" -var SSH_KEY_FILE="$(HOME)/.ssh/id_rsa.pub" @if ! [ -f $(CURDIR)/build/node0/config/genesis.json ]; then docker run --rm -v $(CURDIR)/build:/tendermint:Z tendermint/localnode testnet --v 0 --n 4 --o . ; fi - cd networks/remote/ansible && ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook -i inventory/digital_ocean.py -l remotenet install.yml + cd networks/remote/ansible && ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook -i inventory/digital_ocean.py -l sentrynet install.yml @echo "Next step: Add your validator setup in the genesis.json and config.tml files and run \"make server-config\". (Public key of validator, chain ID, peer IP and node ID.)" # Configuration management -remotenet-config: - cd networks/remote/ansible && ansible-playbook -i inventory/digital_ocean.py -l remotenet config.yml -e BINARY=$(CURDIR)/build/tendermint -e CONFIGDIR=$(CURDIR)/build +sentry-config: + cd networks/remote/ansible && ansible-playbook -i inventory/digital_ocean.py -l sentrynet config.yml -e BINARY=$(CURDIR)/build/tendermint -e CONFIGDIR=$(CURDIR)/build -remotenet-stop: +sentry-stop: @if [ -z "$(DO_API_TOKEN)" ]; then echo "DO_API_TOKEN environment variable not set." ; false ; fi cd networks/remote/terraform && terraform destroy -var DO_API_TOKEN="$(DO_API_TOKEN)" -var SSH_KEY_FILE="$(HOME)/.ssh/id_rsa.pub" # To avoid unintended conflicts with file names, always add to .PHONY # unless there is a reason not to. # https://www.gnu.org/software/make/manual/html_node/Phony-Targets.html -.PHONY: check build build_race dist install check_tools get_tools update_tools get_vendor_deps draw_deps test_cover test_apps test_persistence test_p2p test test_race test_integrations test_release test100 vagrant_test fmt build-linux localnet-start localnet-stop build-docker remotenet-start remotenet-config remotenet-stop +.PHONY: check build build_race dist install check_tools get_tools update_tools get_vendor_deps draw_deps test_cover test_apps test_persistence test_p2p test test_race test_integrations test_release test100 vagrant_test fmt build-linux localnet-start localnet-stop build-docker sentry-start sentry-config sentry-stop diff --git a/networks/remote/ansible/README.rst b/networks/remote/ansible/README.rst index e9f2825c6..713e124e9 100644 --- a/networks/remote/ansible/README.rst +++ b/networks/remote/ansible/README.rst @@ -26,7 +26,7 @@ Build export SSH_KEY_FILE="$HOME/.ssh/id_rsa.pub" - ansible-playbook -i inventory/digital_ocean.py -l remotenet install.yml + ansible-playbook -i inventory/digital_ocean.py -l sentrynet install.yml # The scripts assume that you have your validator set up already. # You can create the folder structure for the sentry nodes using `tendermint testnet`. @@ -34,7 +34,7 @@ Build # Then copy your genesis.json and modify the config.toml as you see fit. # Reconfig the sentry nodes with a new BINARY and the configuration files from the build folder: - ansible-playbook -i inventory/digital_ocean.py -l remotenet config.yml -e BINARY=`pwd`/build/tendermint -e CONFIGDIR=`pwd`/build + ansible-playbook -i inventory/digital_ocean.py -l sentrynet config.yml -e BINARY=`pwd`/build/tendermint -e CONFIGDIR=`pwd`/build Shipping logs to logz.io ------------------------ @@ -47,6 +47,6 @@ Logz.io is an Elastic stack (Elastic search, Logstash and Kibana) service provid apt-get install libsystemd-dev || echo "This will only work on Debian-based systems." go get github.com/mheese/journalbeat - ansible-playbook -i inventory/digital_ocean.py -l remotenet logzio.yml -e LOGZIO_TOKEN=ABCDEFGHIJKLMNOPQRSTUVWXYZ012345 + ansible-playbook -i inventory/digital_ocean.py -l sentrynet logzio.yml -e LOGZIO_TOKEN=ABCDEFGHIJKLMNOPQRSTUVWXYZ012345 diff --git a/networks/remote/ansible/logzio.yml b/networks/remote/ansible/logzio.yml index 7e5148813..53f637f2f 100644 --- a/networks/remote/ansible/logzio.yml +++ b/networks/remote/ansible/logzio.yml @@ -1,5 +1,7 @@ --- +#Note: You need to add LOGZIO_TOKEN variable with your API key. Like tihs: ansible-playbook -e LOGZIO_TOKEN=ABCXYZ123456 + - hosts: all user: root any_errors_fatal: true @@ -7,7 +9,6 @@ vars: - service: tendermint - JOURNALBEAT_BINARY: "{{lookup('env', 'GOPATH')}}/bin/journalbeat" -# - LOGZIO_TOKEN: ABCDEFGHIJKLMNOPQRSTUVWXYZ012345 roles: - logzio diff --git a/networks/remote/ansible/roles/config/tasks/main.yml b/networks/remote/ansible/roles/config/tasks/main.yml index 5186b6aec..a51098caa 100644 --- a/networks/remote/ansible/roles/config/tasks/main.yml +++ b/networks/remote/ansible/roles/config/tasks/main.yml @@ -7,7 +7,7 @@ mode: 0755 - name: Copy config - when: item <= N and ansible_hostname == 'remotenet-node' ~ item + when: item <= N and ansible_hostname == 'sentrynet-node' ~ item copy: src: "{{CONFIGDIR}}/node{{item}}/" dest: "/home/{{service}}/.{{service}}/" diff --git a/networks/remote/terraform/main.tf b/networks/remote/terraform/main.tf index 5618689d2..a768ee13a 100644 --- a/networks/remote/terraform/main.tf +++ b/networks/remote/terraform/main.tf @@ -6,7 +6,7 @@ variable "DO_API_TOKEN" { variable "TESTNET_NAME" { description = "Name of the testnet" - default = "remotenet" + default = "sentrynet" } variable "SSH_KEY_FILE" { From 9ba208c1f5c156cefb1a7b80daf7045cd77d1d6a Mon Sep 17 00:00:00 2001 From: Greg Szabo Date: Tue, 24 Apr 2018 16:40:51 -0400 Subject: [PATCH 120/143] wrapper.sh logfile check fix --- networks/local/localnode/wrapper.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/networks/local/localnode/wrapper.sh b/networks/local/localnode/wrapper.sh index 83332a21c..fe8031e66 100755 --- a/networks/local/localnode/wrapper.sh +++ b/networks/local/localnode/wrapper.sh @@ -25,7 +25,7 @@ fi ## export TMHOME="/tendermint/node${ID}" -if [ -d "${TMHOME}/${LOG}" ]; then +if [ -d "`dirname ${TMHOME}/${LOG}`" ]; then "$BINARY" "$@" | tee "${TMHOME}/${LOG}" else "$BINARY" "$@" From e2f0778c14ada5fd89c17ab9c40359961ae50554 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 26 Apr 2018 09:09:56 -0400 Subject: [PATCH 121/143] spec: update encoding.md --- docs/specification/new-spec/encoding.md | 212 +++++++++++------------- docs/specification/new-spec/wire.go | 80 --------- 2 files changed, 99 insertions(+), 193 deletions(-) delete mode 100644 docs/specification/new-spec/wire.go diff --git a/docs/specification/new-spec/encoding.md b/docs/specification/new-spec/encoding.md index e0317b7ef..b7dd47e8f 100644 --- a/docs/specification/new-spec/encoding.md +++ b/docs/specification/new-spec/encoding.md @@ -1,101 +1,119 @@ # Tendermint Encoding -## Binary Serialization (TMBIN) +## Amino -Tendermint aims to encode data structures in a manner similar to how the corresponding Go structs -are laid out in memory. -Variable length items are length-prefixed. -While the encoding was inspired by Go, it is easily implemented in other languages as well, given its intuitive design. +Tendermint uses the Protobuf3 derrivative [Amino]() for all data structures. +Thik of Amino as an object-oriented Protobuf3 with native JSON support. +The goal of the Amino encoding protocol is to bring parity between application +logic objects and persistence objects. -XXX: This is changing to use real varints and 4-byte-prefixes. -See https://github.com/tendermint/go-wire/tree/sdk2. +Please see the [Amino +specification](https://github.com/tendermint/go-amino#amino-encoding-for-go) for +more details. -### Fixed Length Integers +Notably, every object that satisfies an interface (eg. a particular kind of p2p message, +or a particular kind of pubkey) is registered with a global name, the hash of +which is included in the object's encoding as the so-called "prefix bytes". -Fixed length integers are encoded in Big-Endian using the specified number of bytes. -So `uint8` and `int8` use one byte, `uint16` and `int16` use two bytes, -`uint32` and `int32` use 3 bytes, and `uint64` and `int64` use 4 bytes. +## Public Key Cryptography -Negative integers are encoded via twos-complement. +Tendermint uses Amino to distinguish between different types of private keys, +public keys, and signatures. Additionally, for each public key, Tendermint +defines an Address function that can be used as a more compact identifier in +place of the public key. Here we list the concrete types, their names, +and prefix bytes for public keys and signatures. Note for brevity we don't +include details of the private keys beyond their type and name, as they can be +derrived the same way as the others using Amino. -Examples: +All registered objects are encoded by Amino using a 4-byte PrefixBytes that +uniquely identifies the object and includes information about its underlying +type. For details on how PrefixBytes are computed, see the [Amino +spec](https://github.com/tendermint/go-amino#computing-the-prefix-and-disambiguation-bytes). -```go -encode(uint8(6)) == [0x06] -encode(uint32(6)) == [0x00, 0x00, 0x00, 0x06] +In what follows, we provide the type names and prefix bytes directly. +Notice that when encoding byte-arrays, the length of the byte-array is appended +to the PrefixBytes. Thus the encoding of a byte array becomes ` + ` -encode(int8(-6)) == [0xFA] -encode(int32(-6)) == [0xFF, 0xFF, 0xFF, 0xFA] +### PubKeyEd25519 + +``` +// Name: tendermint/PubKeyEd25519 +// PrefixBytes: 0x1624DE62 +// Length: 0x20 +// Notes: raw 32-byte Ed25519 pubkey +type PubKeyEd25519 [32]byte ``` -### Variable Length Integers +For example, the 32-byte Ed25519 pubkey +`76852933A4686A721442E931A8415F62F5F1AEDF4910F1F252FB393F74C40C85` would be +encoded as +`1624DE622076852933A4686A721442E931A8415F62F5F1AEDF4910F1F252FB393F74C40C85` -Variable length integers are encoded as length-prefixed Big-Endian integers. -The length-prefix consists of a single byte and corresponds to the length of the encoded integer. +### SignatureEd25519 -Negative integers are encoded by flipping the leading bit of the length-prefix to a `1`. - -Zero is encoded as `0x00`. It is not length-prefixed. - -Examples: - -```go -encode(uint(6)) == [0x01, 0x06] -encode(uint(70000)) == [0x03, 0x01, 0x11, 0x70] - -encode(int(-6)) == [0xF1, 0x06] -encode(int(-70000)) == [0xF3, 0x01, 0x11, 0x70] - -encode(int(0)) == [0x00] +``` +// Name: tendermint/SignatureKeyEd25519 +// PrefixBytes: 0x3DA1DB2A +// Length: 0x40 +// Notes: raw 64-byte Ed25519 signature +type SignatureEd25519 [64]byte ``` -### Strings +For example, the 64-byte Ed25519 signature +`005E76B3B0D790959B03F862A9EF8F6236457032B5F522C4CAB5AAD7C44A00A12669E1A2761798E70A0A923DA0CF981839558123CF6466553BCBFF25DADD630F` +would be encoded as +`3DA1DB2A40005E76B3B0D790959B03F862A9EF8F6236457032B5F522C4CAB5AAD7C44A00A12669E1A2761798E70A0A923DA0CF981839558123CF6466553BCBFF25DADD630F` -An encoded string is length-prefixed followed by the underlying bytes of the string. -The length-prefix is itself encoded as an `int`. +### PrivKeyEd25519 -The empty string is encoded as `0x00`. It is not length-prefixed. +``` +// Name: tendermint/PrivKeyEd25519 +// Notes: raw 32-byte priv key concatenated to raw 32-byte pub key +type PrivKeyEd25519 [64]byte +``` +### PubKeySecp256k1 -Examples: - -```go -encode("") == [0x00] -encode("a") == [0x01, 0x01, 0x61] -encode("hello") == [0x01, 0x05, 0x68, 0x65, 0x6C, 0x6C, 0x6F] -encode("¥") == [0x01, 0x02, 0xC2, 0xA5] +``` +// Name: tendermint/PubKeySecp256k1 +// PrefixBytes: 0xEB5AE982 +// Length: 0x21 +// Notes: OpenSSL compressed pubkey prefixed with 0x02 or 0x03 +type PubKeySecp256k1 [33]byte ``` -### Arrays (fixed length) +For example, the 33-byte Secp256k1 pubkey +`03573E0EC1F989DECC3913AC7D44D0509C1A992ECE700845594A1078DAF19A3380` would be +encoded as +`EB5AE9822103573E0EC1F989DECC3913AC7D44D0509C1A992ECE700845594A1078DAF19A3380` -An encoded fix-lengthed array is the concatenation of the encoding of its elements. -There is no length-prefix. +### SignatureSecp256k1 -Examples: - -```go -encode([4]int8{1, 2, 3, 4}) == [0x01, 0x02, 0x03, 0x04] -encode([4]int16{1, 2, 3, 4}) == [0x00, 0x01, 0x00, 0x02, 0x00, 0x03, 0x00, 0x04] -encode([4]int{1, 2, 3, 4}) == [0x01, 0x01, 0x01, 0x02, 0x01, 0x03, 0x01, 0x04] -encode([2]string{"abc", "efg"}) == [0x01, 0x03, 0x61, 0x62, 0x63, 0x01, 0x03, 0x65, 0x66, 0x67] +``` +// Name: tendermint/SignatureKeySecp256k1 +// PrefixBytes: 0x16E1FEEA +// Length: Variable +// Encoding prefix: Variable +// Notes: raw bytes of the Secp256k1 signature +type SignatureSecp256k1 []byte ``` -### Slices (variable length) +For example, the Secp256k1 signature +`304402207447640A5C12A72BAA052D110B666FB6DF717A7B863361C092E751D016C6C08802205C20F9DEBF8915DED310B98BFA890105F43925FDB2B67B78510FE18EDA2B30DA` would +be encoded as +`16E1FEEA46304402202C10C874E413AF538D97EBEF2B01024719F8B7CC559CEEBDC7C380F9DCC4A6E002200EDE9B62F8531933F88DB2A62E73BA3D43ACEB1CBD23070C2F792AAA18717A4A` -An encoded variable-length array is length-prefixed followed by the concatenation of the encoding of -its elements. -The length-prefix is itself encoded as an `int`. +### PrivKeySecp256k1 -An empty slice is encoded as `0x00`. It is not length-prefixed. - -Examples: - -```go -encode([]int8{}) == [0x00] -encode([]int8{1, 2, 3, 4}) == [0x01, 0x04, 0x01, 0x02, 0x03, 0x04] -encode([]int16{1, 2, 3, 4}) == [0x01, 0x04, 0x00, 0x01, 0x00, 0x02, 0x00, 0x03, 0x00, 0x04] -encode([]int{1, 2, 3, 4}) == [0x01, 0x04, 0x01, 0x01, 0x01, 0x02, 0x01, 0x03, 0x01, 0x4] -encode([]string{"abc", "efg"}) == [0x01, 0x02, 0x01, 0x03, 0x61, 0x62, 0x63, 0x01, 0x03, 0x65, 0x66, 0x67] ``` +// Name: tendermint/PrivKeySecp256k1 +// Notes: raw 32-byte priv key +type PrivKeySecp256k1 [32]byte +``` + + + +## Other Common Types ### BitArray @@ -109,36 +127,22 @@ type BitArray struct { } ``` -### Time - -Time is encoded as an `int64` of the number of nanoseconds since January 1, 1970, -rounded to the nearest millisecond. - -Times before then are invalid. - -Examples: +### Part ```go -encode(time.Time("Jan 1 00:00:00 UTC 1970")) == [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] -encode(time.Time("Jan 1 00:00:01 UTC 1970")) == [0x00, 0x00, 0x00, 0x00, 0x3B, 0x9A, 0xCA, 0x00] // 1,000,000,000 ns -encode(time.Time("Mon Jan 2 15:04:05 -0700 MST 2006")) == [0x0F, 0xC4, 0xBB, 0xC1, 0x53, 0x03, 0x12, 0x00] +type Part struct { + Index int + Bytes byte[] + Proof byte[] +} ``` -### Structs +### MakeParts -An encoded struct is the concatenation of the encoding of its elements. -There is no length-prefix. - -Examples: +Encode an object using Amino and slice it into parts. ```go -type MyStruct struct{ - A int - B string - C time.Time -} -encode(MyStruct{4, "hello", time.Time("Mon Jan 2 15:04:05 -0700 MST 2006")}) == - [0x01, 0x04, 0x01, 0x05, 0x68, 0x65, 0x6C, 0x6C, 0x6F, 0x0F, 0xC4, 0xBB, 0xC1, 0x53, 0x03, 0x12, 0x00] +MakeParts(object, partSize) ``` ## Merkle Trees @@ -169,10 +173,9 @@ For `struct` arguments, we compute a `[][]byte` by sorting elements of the `stru field name and then hashing them. For `[]struct` arguments, we compute a `[][]byte` by hashing the individual `struct` elements. -## JSON (TMJSON) +## AminoJSON -Signed messages (eg. votes, proposals) in the consensus are encoded in TMJSON, rather than TMBIN. -TMJSON is JSON where `[]byte` are encoded as uppercase hex, rather than base64. +Signed messages (eg. votes, proposals) in the consensus are encoded in AminoJSON, rather than binary Amino. When signing, the elements of a message are sorted by key and the sorted message is embedded in an outer JSON that includes a `chain_id` field. @@ -185,22 +188,5 @@ like: Note how the fields within each level are sorted. -## Other -### MakeParts -Encode an object using TMBIN and slice it into parts. - -```go -MakeParts(object, partSize) -``` - -### Part - -```go -type Part struct { - Index int - Bytes byte[] - Proof byte[] -} -``` diff --git a/docs/specification/new-spec/wire.go b/docs/specification/new-spec/wire.go deleted file mode 100644 index af76f3669..000000000 --- a/docs/specification/new-spec/wire.go +++ /dev/null @@ -1,80 +0,0 @@ -package main - -import ( - "fmt" - "time" - - wire "github.com/tendermint/go-wire" -) - -func main() { - - encode(uint8(6)) - encode(uint32(6)) - encode(int8(-6)) - encode(int32(-6)) - Break() - encode(uint(6)) - encode(uint(70000)) - encode(int(0)) - encode(int(-6)) - encode(int(-70000)) - Break() - encode("") - encode("a") - encode("hello") - encode("¥") - Break() - encode([4]int8{1, 2, 3, 4}) - encode([4]int16{1, 2, 3, 4}) - encode([4]int{1, 2, 3, 4}) - encode([2]string{"abc", "efg"}) - Break() - encode([]int8{}) - encode([]int8{1, 2, 3, 4}) - encode([]int16{1, 2, 3, 4}) - encode([]int{1, 2, 3, 4}) - encode([]string{"abc", "efg"}) - Break() - - timeFmt := "Mon Jan 2 15:04:05 -0700 MST 2006" - t1, _ := time.Parse(timeFmt, timeFmt) - n := (t1.UnixNano() / 1000000.) * 1000000 - encode(n) - encode(t1) - - t2, _ := time.Parse(timeFmt, "Thu Jan 1 00:00:00 -0000 UTC 1970") - encode(t2) - - t2, _ = time.Parse(timeFmt, "Thu Jan 1 00:00:01 -0000 UTC 1970") - fmt.Println("N", t2.UnixNano()) - encode(t2) - Break() - encode(struct { - A int - B string - C time.Time - }{ - 4, - "hello", - t1, - }) -} - -func encode(i interface{}) { - Println(wire.BinaryBytes(i)) - -} - -func Println(b []byte) { - s := "[" - for _, x := range b { - s += fmt.Sprintf("0x%.2X, ", x) - } - s = s[:len(s)-2] + "]" - fmt.Println(s) -} - -func Break() { - fmt.Println("------") -} From 91c81ef9a1139eef5affa750e48027bb78207bc3 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 26 Apr 2018 10:42:58 -0400 Subject: [PATCH 122/143] spec: note on byte arrays, clean up bitarrays and more, add merkle proof, add crypto.go script --- docs/specification/new-spec/encoding.md | 91 ++++++++++++++- docs/specification/new-spec/scripts/crypto.go | 108 ++++++++++++++++++ 2 files changed, 193 insertions(+), 6 deletions(-) create mode 100644 docs/specification/new-spec/scripts/crypto.go diff --git a/docs/specification/new-spec/encoding.md b/docs/specification/new-spec/encoding.md index b7dd47e8f..de7478927 100644 --- a/docs/specification/new-spec/encoding.md +++ b/docs/specification/new-spec/encoding.md @@ -15,6 +15,18 @@ Notably, every object that satisfies an interface (eg. a particular kind of p2p or a particular kind of pubkey) is registered with a global name, the hash of which is included in the object's encoding as the so-called "prefix bytes". +## Byte Arrays + +The encoding of a byte array is simply the raw-bytes prefixed with the length of +the array as a `UVarint` (what Protobuf calls a `Varint`). + +For details on varints, see the [protobuf +spec](https://developers.google.com/protocol-buffers/docs/encoding#varints). + +For example, the byte-array `[0xA, 0xB]` would be encoded as `0x020A0B`, +while a byte-array containing 300 entires beginning with `[0xA, 0xB, ...]` would +be encoded as `0xAC020A0B...` where `0xAC02` is the UVarint encoding of 300. + ## Public Key Cryptography Tendermint uses Amino to distinguish between different types of private keys, @@ -117,8 +129,10 @@ type PrivKeySecp256k1 [32]byte ### BitArray -BitArray is encoded as an `int` of the number of bits, and with an array of `uint64` to encode -value of each array element. +The BitArray is used in block headers and some consensus messages to signal +whether or not something was done by each validator. BitArray is represented +with a struct containing the number of bits (`Bits`) and the bit-array itself +encoded in base64 (`Elems`). ```go type BitArray struct { @@ -127,8 +141,21 @@ type BitArray struct { } ``` +This type is easily encoded directly by Amino. + +Note BitArray receives a special JSON encoding in the form of `x` and `_` +representing `1` and `0`. Ie. the BitArray `10110` would be JSON encoded as +`"x_xx_"` + ### Part +Part is used to break up blocks into pieces that can be gossiped in parallel +and securely verified using a Merkle tree of the parts. + +Part contains the index of the part in the larger set (`Index`), the actual +underlying data of the part (`Bytes`), and a simple Merkle proof that the part is contained in +the larger set (`Proof`). + ```go type Part struct { Index int @@ -142,14 +169,16 @@ type Part struct { Encode an object using Amino and slice it into parts. ```go -MakeParts(object, partSize) +func MakeParts(obj interface{}, partSize int) []Part ``` ## Merkle Trees Simple Merkle trees are used in numerous places in Tendermint to compute a cryptographic digest of a data structure. -RIPEMD160 is always used as the hashing function. +SHA256 is always used as the hashing function. + +### Simple Merkle Root The function `SimpleMerkleRoot` is a simple recursive function defined as follows: @@ -163,16 +192,66 @@ func SimpleMerkleRoot(hashes [][]byte) []byte{ default: left := SimpleMerkleRoot(hashes[:(len(hashes)+1)/2]) right := SimpleMerkleRoot(hashes[(len(hashes)+1)/2:]) - return RIPEMD160(append(left, right)) + return SimpleConcatHash(left, right) } } + +func SimpleConcatHash(left, right []byte) []byte{ + left = encodeByteSlice(left) + right = encodeByteSlice(right) + return SHA256(append(left, right)) +} ``` -Note: we abuse notion and call `SimpleMerkleRoot` with arguments of type `struct` or type `[]struct`. +Note that the leaves are Amino encoded as byte-arrays (ie. simple Uvarint length +prefix) before being concatenated together and hashed. + +Note: we will abuse notion and invoke `SimpleMerkleRoot` with arguments of type `struct` or type `[]struct`. For `struct` arguments, we compute a `[][]byte` by sorting elements of the `struct` according to field name and then hashing them. For `[]struct` arguments, we compute a `[][]byte` by hashing the individual `struct` elements. +### Simple Merkle Proof + +Proof that a leaf is in a Merkle tree consists of a simple structure: + + +``` +type SimpleProof struct { + Aunts [][]byte +} +``` + +Which is verified using the following: + +``` +func (proof SimpleProof) Verify(index, total int, leafHash, rootHash []byte) bool { + computedHash := computeHashFromAunts(index, total, leafHash, proof.Aunts) + return computedHash == rootHash +} + +func computeHashFromAunts(index, total int, leafHash []byte, innerHashes [][]byte) []byte{ + assert(index < total && index >= 0 && total > 0) + + if total == 1{ + assert(len(proof.Aunts) == 0) + return leafHash + } + + assert(len(innerHashes) > 0) + + numLeft := (total + 1) / 2 + if index < numLeft { + leftHash := computeHashFromAunts(index, numLeft, leafHash, innerHashes[:len(innerHashes)-1]) + assert(leftHash != nil) + return SimpleHashFromTwoHashes(leftHash, innerHashes[len(innerHashes)-1]) + } + rightHash := computeHashFromAunts(index-numLeft, total-numLeft, leafHash, innerHashes[:len(innerHashes)-1]) + assert(rightHash != nil) + return SimpleHashFromTwoHashes(innerHashes[len(innerHashes)-1], rightHash) +} +``` + ## AminoJSON Signed messages (eg. votes, proposals) in the consensus are encoded in AminoJSON, rather than binary Amino. diff --git a/docs/specification/new-spec/scripts/crypto.go b/docs/specification/new-spec/scripts/crypto.go new file mode 100644 index 000000000..704a21c30 --- /dev/null +++ b/docs/specification/new-spec/scripts/crypto.go @@ -0,0 +1,108 @@ +package main + +import ( + "fmt" + + crypto "github.com/tendermint/go-crypto" +) + +func printEd() { + priv := crypto.GenPrivKeyEd25519() + pub := priv.PubKey().(crypto.PubKeyEd25519) + sig := priv.Sign([]byte("hello")).(crypto.SignatureEd25519) + + name := "tendermint/PubKeyEd25519" + length := len(pub[:]) + + fmt.Println("### PubKeyEd25519") + fmt.Println("") + fmt.Println("```") + fmt.Printf("// Name: %s\n", name) + fmt.Printf("// PrefixBytes: 0x%X \n", pub.Bytes()[:4]) + fmt.Printf("// Length: 0x%X \n", length) + fmt.Println("// Notes: raw 32-byte Ed25519 pubkey") + fmt.Println("type PubKeyEd25519 [32]byte") + fmt.Println("```") + fmt.Println("") + fmt.Printf("For example, the 32-byte Ed25519 pubkey `%X` would be encoded as `%X`\n", pub[:], pub.Bytes()) + fmt.Println("") + + name = "tendermint/SignatureKeyEd25519" + length = len(sig[:]) + + fmt.Println("### SignatureEd25519") + fmt.Println("") + fmt.Println("```") + fmt.Printf("// Name: %s\n", name) + fmt.Printf("// PrefixBytes: 0x%X \n", sig.Bytes()[:4]) + fmt.Printf("// Length: 0x%X \n", length) + fmt.Println("// Notes: raw 64-byte Ed25519 signature") + fmt.Println("type SignatureEd25519 [64]byte") + fmt.Println("```") + fmt.Println("") + fmt.Printf("For example, the 64-byte Ed25519 signature `%X` would be encoded as `%X`\n", sig[:], sig.Bytes()) + fmt.Println("") + + name = "tendermint/PrivKeyEd25519" + + fmt.Println("### PrivKeyEd25519") + fmt.Println("") + fmt.Println("```") + fmt.Println("// Name:", name) + fmt.Println("// Notes: raw 32-byte priv key concatenated to raw 32-byte pub key") + fmt.Println("type PrivKeyEd25519 [64]byte") + fmt.Println("```") +} + +func printSecp() { + priv := crypto.GenPrivKeySecp256k1() + pub := priv.PubKey().(crypto.PubKeySecp256k1) + sig := priv.Sign([]byte("hello")).(crypto.SignatureSecp256k1) + + name := "tendermint/PubKeySecp256k1" + length := len(pub[:]) + + fmt.Println("### PubKeySecp256k1") + fmt.Println("") + fmt.Println("```") + fmt.Printf("// Name: %s\n", name) + fmt.Printf("// PrefixBytes: 0x%X \n", pub.Bytes()[:4]) + fmt.Printf("// Length: 0x%X \n", length) + fmt.Println("// Notes: OpenSSL compressed pubkey prefixed with 0x02 or 0x03") + fmt.Println("type PubKeySecp256k1 [33]byte") + fmt.Println("```") + fmt.Println("") + fmt.Printf("For example, the 33-byte Secp256k1 pubkey `%X` would be encoded as `%X`\n", pub[:], pub.Bytes()) + fmt.Println("") + + name = "tendermint/SignatureKeySecp256k1" + + fmt.Println("### SignatureSecp256k1") + fmt.Println("") + fmt.Println("```") + fmt.Printf("// Name: %s\n", name) + fmt.Printf("// PrefixBytes: 0x%X \n", sig.Bytes()[:4]) + fmt.Printf("// Length: Variable\n") + fmt.Printf("// Encoding prefix: Variable\n") + fmt.Println("// Notes: raw bytes of the Secp256k1 signature") + fmt.Println("type SignatureSecp256k1 []byte") + fmt.Println("```") + fmt.Println("") + fmt.Printf("For example, the Secp256k1 signature `%X` would be encoded as `%X`\n", []byte(sig[:]), sig.Bytes()) + fmt.Println("") + + name = "tendermint/PrivKeySecp256k1" + + fmt.Println("### PrivKeySecp256k1") + fmt.Println("") + fmt.Println("```") + fmt.Println("// Name:", name) + fmt.Println("// Notes: raw 32-byte priv key") + fmt.Println("type PrivKeySecp256k1 [32]byte") + fmt.Println("```") +} + +func main() { + printEd() + printSecp() +} From 0d9004a8545e37a8aa5a1f2bde8acda48746b40b Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 26 Apr 2018 16:52:11 +0200 Subject: [PATCH 123/143] [cmd] Turn off strict routability when using testnet cmd (#1493) * [cmd] Turn off strict routability when using testnet cmd Refs https://github.com/tendermint/tendermint/pull/1454 * use "testnet" instead of "it" --- cmd/tendermint/commands/testnet.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/cmd/tendermint/commands/testnet.go b/cmd/tendermint/commands/testnet.go index ef503db3f..3b29e20a2 100644 --- a/cmd/tendermint/commands/testnet.go +++ b/cmd/tendermint/commands/testnet.go @@ -57,7 +57,18 @@ func init() { var TestnetFilesCmd = &cobra.Command{ Use: "testnet", Short: "Initialize files for a Tendermint testnet", - RunE: testnetFiles, + Long: `testnet will create "v" + "n" number of directories and populate each with +necessary files (private validator, genesis, config, etc.). + +Note, strict routability for addresses is turned off in the config file. + +Optionally, it will fill in persistent_peers list in config file using either hostnames or IPs. + +Example: + + tendermint testnet --v 4 --o ./output --populate-persistent-peers --starting-ip-address 192.168.10.2 + `, + RunE: testnetFiles, } func testnetFiles(cmd *cobra.Command, args []string) error { @@ -162,6 +173,7 @@ func populatePersistentPeersInConfigAndWriteIt(config *cfg.Config) error { nodeDir := filepath.Join(outputDir, cmn.Fmt("%s%d", nodeDirPrefix, i)) config.SetRoot(nodeDir) config.P2P.PersistentPeers = persistentPeersList + config.P2P.AddrBookStrict = false // overwrite default config cfg.WriteConfigFile(filepath.Join(nodeDir, "config", "config.toml"), config) From 97be1eef870786f256fb322c7eee19e89ef2bca4 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 26 Apr 2018 11:08:34 -0400 Subject: [PATCH 124/143] add abci notes --- docs/specification/new-spec/abci.md | 48 +++++++++++++++++++++++++ docs/specification/new-spec/encoding.md | 3 +- 2 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 docs/specification/new-spec/abci.md diff --git a/docs/specification/new-spec/abci.md b/docs/specification/new-spec/abci.md new file mode 100644 index 000000000..d942c800a --- /dev/null +++ b/docs/specification/new-spec/abci.md @@ -0,0 +1,48 @@ +# Application Blockchain Interface (ABCI) + +ABCI is the interface between Tendermint (a state-machine replication engine) +and an application (the actual state machine). + +The ABCI message types are defined in a [protobuf +file](https://github.com/tendermint/abci/blob/master/types/types.proto). +For full details on the ABCI message types and protocol, see the [ABCI +specificaiton](https://github.com/tendermint/abci/blob/master/specification.rst). +For additional details on server implementation, see the [ABCI +readme](https://github.com/tendermint/abci#implementation). + +Here we provide some more details around the use of ABCI by Tendermint and +clarify common "gotchas". + +## Validator Updates + +Updates to the Tendermint validator set can be made by returning `Validator` +objects in the `ResponseBeginBlock`: + +``` +message Validator { + bytes pub_key = 1; + int64 power = 2; +} +``` + +The `pub_key` is the Amino encoded public key for the validator. For details on +Amino encoded public keys, see the [section of the encoding spec](./encoding.md#public-key-cryptography). + +For example, the 32-byte Ed25519 pubkey +`76852933A4686A721442E931A8415F62F5F1AEDF4910F1F252FB393F74C40C85` would be +Amino encoded as +`1624DE622076852933A4686A721442E931A8415F62F5F1AEDF4910F1F252FB393F74C40C85` + +The `power` is the new voting power for the validator, with the +following rules: + +- power must be non-negative +- if power is 0, the validator must already exist, and will be removed from the + validator set +- if power is non-0: + - if the validator does not already exist, it will be added to the validator + set with the given power + - if the validator does already exist, its power will be adjusted to the given power + + +## Query diff --git a/docs/specification/new-spec/encoding.md b/docs/specification/new-spec/encoding.md index de7478927..83154c859 100644 --- a/docs/specification/new-spec/encoding.md +++ b/docs/specification/new-spec/encoding.md @@ -33,7 +33,8 @@ Tendermint uses Amino to distinguish between different types of private keys, public keys, and signatures. Additionally, for each public key, Tendermint defines an Address function that can be used as a more compact identifier in place of the public key. Here we list the concrete types, their names, -and prefix bytes for public keys and signatures. Note for brevity we don't +and prefix bytes for public keys and signatures, as well as the address schemes +for each PubKey. Note for brevity we don't include details of the private keys beyond their type and name, as they can be derrived the same way as the others using Amino. From 0e1414ef9def6ae352aed64a5b3aa3e1110388cc Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 26 Apr 2018 11:46:20 -0400 Subject: [PATCH 125/143] spec: add Address spec. notes about Query --- docs/specification/new-spec/abci.md | 22 ++++++++- docs/specification/new-spec/encoding.md | 46 ++++++++++++++----- docs/specification/new-spec/scripts/crypto.go | 25 ++++++++-- 3 files changed, 76 insertions(+), 17 deletions(-) diff --git a/docs/specification/new-spec/abci.md b/docs/specification/new-spec/abci.md index d942c800a..75eed6b58 100644 --- a/docs/specification/new-spec/abci.md +++ b/docs/specification/new-spec/abci.md @@ -28,7 +28,7 @@ message Validator { The `pub_key` is the Amino encoded public key for the validator. For details on Amino encoded public keys, see the [section of the encoding spec](./encoding.md#public-key-cryptography). -For example, the 32-byte Ed25519 pubkey +For Ed25519 pubkeys, the Amino prefix is always "1624DE6220". For example, the 32-byte Ed25519 pubkey `76852933A4686A721442E931A8415F62F5F1AEDF4910F1F252FB393F74C40C85` would be Amino encoded as `1624DE622076852933A4686A721442E931A8415F62F5F1AEDF4910F1F252FB393F74C40C85` @@ -44,5 +44,23 @@ following rules: set with the given power - if the validator does already exist, its power will be adjusted to the given power - ## Query + +Query is a generic message type with lots of flexibility to enable diverse sets +of queries from applications. Tendermint has no requirements from the Query +message for normal operation - that is, the ABCI app developer need not implement Query functionality if they do not wish too. +That said, Tendermint makes a number of queries to support some optional +features. These are: + +### Peer Filtering + +When Tendermint connects to a peer, it sends two queries to the ABCI application +using the following paths, with no additional data: + + - `/p2p/filter/addr/`, where `` denote the IP address and + the port of the connection + - `p2p/filter/pubkey/`, where `` is the peer node ID (ie. the + pubkey.Address() for the peer's PubKey) + +If either of these queries return a non-zero ABCI code, Tendermint will refuse +to connect to the peer. diff --git a/docs/specification/new-spec/encoding.md b/docs/specification/new-spec/encoding.md index 83154c859..e15c370b3 100644 --- a/docs/specification/new-spec/encoding.md +++ b/docs/specification/new-spec/encoding.md @@ -15,6 +15,9 @@ Notably, every object that satisfies an interface (eg. a particular kind of p2p or a particular kind of pubkey) is registered with a global name, the hash of which is included in the object's encoding as the so-called "prefix bytes". +We define the `func AminoEncode(obj interface{}) []byte` function to take an +arbitrary object and return the Amino encoded bytes. + ## Byte Arrays The encoding of a byte array is simply the raw-bytes prefixed with the length of @@ -48,6 +51,9 @@ Notice that when encoding byte-arrays, the length of the byte-array is appended to the PrefixBytes. Thus the encoding of a byte array becomes ` ` +(NOTE: the remainder of this section on Public Key Cryptography can be generated +from [this script](./scripts/crypto.go)) + ### PubKeyEd25519 ``` @@ -56,12 +62,21 @@ to the PrefixBytes. Thus the encoding of a byte array becomes ` // Length: 0x20 // Notes: raw 32-byte Ed25519 pubkey type PubKeyEd25519 [32]byte + +func (pubkey PubKeyEd25519) Address() []byte { + // NOTE: hash of the Amino encoded bytes! + return RIPEMD160(AminoEncode(pubkey)) +} ``` For example, the 32-byte Ed25519 pubkey -`76852933A4686A721442E931A8415F62F5F1AEDF4910F1F252FB393F74C40C85` would be +`CCACD52F9B29D04393F01CD9AF6535455668115641F3D8BAEFD2295F24BAF60E` would be encoded as -`1624DE622076852933A4686A721442E931A8415F62F5F1AEDF4910F1F252FB393F74C40C85` +`1624DE6220CCACD52F9B29D04393F01CD9AF6535455668115641F3D8BAEFD2295F24BAF60E`. + +The address would then be +`RIPEMD160(0x1624DE6220CCACD52F9B29D04393F01CD9AF6535455668115641F3D8BAEFD2295F24BAF60E)` +or `430FF75BAF1EC4B0D51BB3EEC2955479D0071605` ### SignatureEd25519 @@ -74,9 +89,9 @@ type SignatureEd25519 [64]byte ``` For example, the 64-byte Ed25519 signature -`005E76B3B0D790959B03F862A9EF8F6236457032B5F522C4CAB5AAD7C44A00A12669E1A2761798E70A0A923DA0CF981839558123CF6466553BCBFF25DADD630F` +`1B6034A8ED149D3C94FDA13EC03B26CC0FB264D9B0E47D3FA3DEF9FCDE658E49C80B35F9BE74949356401B15B18FB817D6E54495AD1C4A8401B248466CB0DB0B` would be encoded as -`3DA1DB2A40005E76B3B0D790959B03F862A9EF8F6236457032B5F522C4CAB5AAD7C44A00A12669E1A2761798E70A0A923DA0CF981839558123CF6466553BCBFF25DADD630F` +`3DA1DB2A401B6034A8ED149D3C94FDA13EC03B26CC0FB264D9B0E47D3FA3DEF9FCDE658E49C80B35F9BE74949356401B15B18FB817D6E54495AD1C4A8401B248466CB0DB0B` ### PrivKeyEd25519 @@ -85,6 +100,7 @@ would be encoded as // Notes: raw 32-byte priv key concatenated to raw 32-byte pub key type PrivKeyEd25519 [64]byte ``` + ### PubKeySecp256k1 ``` @@ -93,12 +109,22 @@ type PrivKeyEd25519 [64]byte // Length: 0x21 // Notes: OpenSSL compressed pubkey prefixed with 0x02 or 0x03 type PubKeySecp256k1 [33]byte + +func (pubkey PubKeySecp256k1) Address() []byte { + // NOTE: hash of the raw pubkey bytes (not Amino encoded!). + // Compatible with Bitcoin addresses. + return RIPEMD160(SHA256(pubkey[:])) +} ``` For example, the 33-byte Secp256k1 pubkey -`03573E0EC1F989DECC3913AC7D44D0509C1A992ECE700845594A1078DAF19A3380` would be +`020BD40F225A57ED383B440CF073BC5539D0341F5767D2BF2D78406D00475A2EE9` would be encoded as -`EB5AE9822103573E0EC1F989DECC3913AC7D44D0509C1A992ECE700845594A1078DAF19A3380` +`EB5AE98221020BD40F225A57ED383B440CF073BC5539D0341F5767D2BF2D78406D00475A2EE9` + +The address would then be +`RIPEMD160(SHA256(0x020BD40F225A57ED383B440CF073BC5539D0341F5767D2BF2D78406D00475A2EE9))` +or `0AE5BEE929ABE51BAD345DB925EEA652680783FC` ### SignatureSecp256k1 @@ -112,9 +138,9 @@ type SignatureSecp256k1 []byte ``` For example, the Secp256k1 signature -`304402207447640A5C12A72BAA052D110B666FB6DF717A7B863361C092E751D016C6C08802205C20F9DEBF8915DED310B98BFA890105F43925FDB2B67B78510FE18EDA2B30DA` would -be encoded as -`16E1FEEA46304402202C10C874E413AF538D97EBEF2B01024719F8B7CC559CEEBDC7C380F9DCC4A6E002200EDE9B62F8531933F88DB2A62E73BA3D43ACEB1CBD23070C2F792AAA18717A4A` +`304402201CD4B8C764D2FD8AF23ECFE6666CA8A53886D47754D951295D2D311E1FEA33BF02201E0F906BB1CF2C30EAACFFB032A7129358AFF96B9F79B06ACFFB18AC90C2ADD7` +would be encoded as +`16E1FEEA46304402201CD4B8C764D2FD8AF23ECFE6666CA8A53886D47754D951295D2D311E1FEA33BF02201E0F906BB1CF2C30EAACFFB032A7129358AFF96B9F79B06ACFFB18AC90C2ADD7` ### PrivKeySecp256k1 @@ -124,8 +150,6 @@ be encoded as type PrivKeySecp256k1 [32]byte ``` - - ## Other Common Types ### BitArray diff --git a/docs/specification/new-spec/scripts/crypto.go b/docs/specification/new-spec/scripts/crypto.go index 704a21c30..e4dbd8a24 100644 --- a/docs/specification/new-spec/scripts/crypto.go +++ b/docs/specification/new-spec/scripts/crypto.go @@ -6,8 +6,11 @@ import ( crypto "github.com/tendermint/go-crypto" ) +// SECRET +var SECRET = []byte("some secret") + func printEd() { - priv := crypto.GenPrivKeyEd25519() + priv := crypto.GenPrivKeyEd25519FromSecret(SECRET) pub := priv.PubKey().(crypto.PubKeyEd25519) sig := priv.Sign([]byte("hello")).(crypto.SignatureEd25519) @@ -22,9 +25,15 @@ func printEd() { fmt.Printf("// Length: 0x%X \n", length) fmt.Println("// Notes: raw 32-byte Ed25519 pubkey") fmt.Println("type PubKeyEd25519 [32]byte") + fmt.Println("") + fmt.Println(`func (pubkey PubKeyEd25519) Address() []byte { + // NOTE: hash of the Amino encoded bytes! + return RIPEMD160(AminoEncode(pubkey)) +}`) fmt.Println("```") fmt.Println("") - fmt.Printf("For example, the 32-byte Ed25519 pubkey `%X` would be encoded as `%X`\n", pub[:], pub.Bytes()) + fmt.Printf("For example, the 32-byte Ed25519 pubkey `%X` would be encoded as `%X`.\n\n", pub[:], pub.Bytes()) + fmt.Printf("The address would then be `RIPEMD160(0x%X)` or `%X`\n", pub.Bytes(), pub.Address()) fmt.Println("") name = "tendermint/SignatureKeyEd25519" @@ -55,7 +64,7 @@ func printEd() { } func printSecp() { - priv := crypto.GenPrivKeySecp256k1() + priv := crypto.GenPrivKeySecp256k1FromSecret(SECRET) pub := priv.PubKey().(crypto.PubKeySecp256k1) sig := priv.Sign([]byte("hello")).(crypto.SignatureSecp256k1) @@ -70,9 +79,16 @@ func printSecp() { fmt.Printf("// Length: 0x%X \n", length) fmt.Println("// Notes: OpenSSL compressed pubkey prefixed with 0x02 or 0x03") fmt.Println("type PubKeySecp256k1 [33]byte") + fmt.Println("") + fmt.Println(`func (pubkey PubKeySecp256k1) Address() []byte { + // NOTE: hash of the raw pubkey bytes (not Amino encoded!). + // Compatible with Bitcoin addresses. + return RIPEMD160(SHA256(pubkey[:])) +}`) fmt.Println("```") fmt.Println("") - fmt.Printf("For example, the 33-byte Secp256k1 pubkey `%X` would be encoded as `%X`\n", pub[:], pub.Bytes()) + fmt.Printf("For example, the 33-byte Secp256k1 pubkey `%X` would be encoded as `%X`\n\n", pub[:], pub.Bytes()) + fmt.Printf("The address would then be `RIPEMD160(SHA256(0x%X))` or `%X`\n", pub[:], pub.Address()) fmt.Println("") name = "tendermint/SignatureKeySecp256k1" @@ -104,5 +120,6 @@ func printSecp() { func main() { printEd() + fmt.Println("") printSecp() } From e5951acfb47fa55f3e932170854e518e2ad5b5f7 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 26 Apr 2018 11:46:50 -0400 Subject: [PATCH 126/143] SHA256 -> RIPEMD160 --- docs/specification/new-spec/encoding.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/specification/new-spec/encoding.md b/docs/specification/new-spec/encoding.md index e15c370b3..abecf5926 100644 --- a/docs/specification/new-spec/encoding.md +++ b/docs/specification/new-spec/encoding.md @@ -201,7 +201,7 @@ func MakeParts(obj interface{}, partSize int) []Part Simple Merkle trees are used in numerous places in Tendermint to compute a cryptographic digest of a data structure. -SHA256 is always used as the hashing function. +RIPEMD160 is always used as the hashing function. ### Simple Merkle Root @@ -224,7 +224,7 @@ func SimpleMerkleRoot(hashes [][]byte) []byte{ func SimpleConcatHash(left, right []byte) []byte{ left = encodeByteSlice(left) right = encodeByteSlice(right) - return SHA256(append(left, right)) + return RIPEMD160 (append(left, right)) } ``` From f1ead2df70f5e17848219393ae34a4fab4eaaf22 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 26 Apr 2018 11:50:45 -0400 Subject: [PATCH 127/143] typo --- docs/specification/new-spec/encoding.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/specification/new-spec/encoding.md b/docs/specification/new-spec/encoding.md index abecf5926..ae6e003d8 100644 --- a/docs/specification/new-spec/encoding.md +++ b/docs/specification/new-spec/encoding.md @@ -3,7 +3,7 @@ ## Amino Tendermint uses the Protobuf3 derrivative [Amino]() for all data structures. -Thik of Amino as an object-oriented Protobuf3 with native JSON support. +Think of Amino as an object-oriented Protobuf3 with native JSON support. The goal of the Amino encoding protocol is to bring parity between application logic objects and persistence objects. From d48a6f930dc1e121c63613f210337b0bcc6ba88f Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 26 Apr 2018 20:35:04 -0400 Subject: [PATCH 128/143] document pre-amino pubkeys/addresses --- CHANGELOG.md | 7 + README.md | 9 +- docs/specification/new-spec/abci.md | 3 + docs/specification/new-spec/pre-amino.md | 246 +++++++++++++++++++++++ docs/specification/new-spec/state.md | 3 - version/version.go | 9 +- 6 files changed, 268 insertions(+), 9 deletions(-) create mode 100644 docs/specification/new-spec/pre-amino.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 967fdef65..0b38873f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,12 @@ BUG FIXES: - Graceful handling/recovery for apps that have non-determinism or fail to halt - Graceful handling/recovery for violations of safety, or liveness +## 0.19.1 (TBD) + +BUG FIXES + +- [spec] Document address format and pubkey encoding + ## 0.19.0 (April 13th, 2018) BREAKING: @@ -42,6 +48,7 @@ genesis/priv_validator/node_key JSON files. FEATURES: - [cmd] added `gen_node_key` command + ## 0.18.0 (April 6th, 2018) BREAKING: diff --git a/README.md b/README.md index 1a4cc00cb..89c28e9c2 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,9 @@ _NOTE: This is alpha software. Please contact us if you intend to run it in prod Tendermint Core is Byzantine Fault Tolerant (BFT) middleware that takes a state transition machine - written in any programming language - and securely replicates it on many machines. -For more information, from introduction to install to application development, [Read The Docs](https://tendermint.readthedocs.io/en/master/). +For more information, from introduction to installation and application development, [Read The Docs](https://tendermint.readthedocs.io/en/master/). + +For protocol details, see [the specification](./docs/specification/new-spec). ## Minimum requirements @@ -46,7 +48,8 @@ For more details (or if it fails), [read the docs](https://tendermint.readthedoc ### Tendermint Core -All resources involving the use of, building application on, or developing for, tendermint, can be found at [Read The Docs](https://tendermint.readthedocs.io/en/master/). Additional information about some - and eventually all - of the sub-projects below, can be found at Read The Docs. +To use Tendermint, build apps on it, or develop it, [Read The Docs](https://tendermint.readthedocs.io/en/master/). +Additional information about some - and eventually all - of the sub-projects below, can be found at Read The Docs. ### Sub-projects @@ -61,8 +64,8 @@ All resources involving the use of, building application on, or developing for, ### Applications -* [Ethermint](http://github.com/tendermint/ethermint); Ethereum on Tendermint * [Cosmos SDK](http://github.com/cosmos/cosmos-sdk); a cryptocurrency application framework +* [Ethermint](http://github.com/tendermint/ethermint); Ethereum on Tendermint * [Many more](https://tendermint.readthedocs.io/en/master/ecosystem.html#abci-applications) ### More diff --git a/docs/specification/new-spec/abci.md b/docs/specification/new-spec/abci.md index 75eed6b58..813aac787 100644 --- a/docs/specification/new-spec/abci.md +++ b/docs/specification/new-spec/abci.md @@ -33,6 +33,9 @@ For Ed25519 pubkeys, the Amino prefix is always "1624DE6220". For example, the 3 Amino encoded as `1624DE622076852933A4686A721442E931A8415F62F5F1AEDF4910F1F252FB393F74C40C85` +(Note: in old versions of Tendermint (pre-v0.19.0), the pubkey is just prefixed with a +single type byte, so for ED25519 we'd have `pub_key = 0x1 | pub`) + The `power` is the new voting power for the validator, with the following rules: diff --git a/docs/specification/new-spec/pre-amino.md b/docs/specification/new-spec/pre-amino.md new file mode 100644 index 000000000..edddeff46 --- /dev/null +++ b/docs/specification/new-spec/pre-amino.md @@ -0,0 +1,246 @@ +# Tendermint Encoding (Pre-Amino) + +## PubKeys and Addresses + +PubKeys are prefixed with a type-byte, followed by the raw bytes of the public +key. + +Two keys are supported with the following type bytes: + +``` +TypeByteEd25519 = 0x1 +TypeByteSecp256k1 = 0x2 +``` + +``` +// TypeByte: 0x1 +type PubKeyEd25519 [32]byte + +func (pub PubKeyEd25519) Encode() []byte { + return 0x1 | pub +} + +func (pub PubKeyEd25519) Address() []byte { + // NOTE: the length (0x0120) is also included + return RIPEMD160(0x1 | 0x0120 | pub) +} + +// TypeByte: 0x2 +// NOTE: OpenSSL compressed pubkey (x-cord with 0x2 or 0x3) +type PubKeySecp256k1 [33]byte + +func (pub PubKeySecp256k1) Encode() []byte { + return 0x2 | pub +} + +func (pub PubKeySecp256k1) Address() []byte { + return RIPEMD160(SHA256(pub)) +} +``` + +See https://github.com/tendermint/go-crypto/blob/v0.5.0/pub_key.go for more. + +## Binary Serialization (go-wire) + +Tendermint aims to encode data structures in a manner similar to how the corresponding Go structs +are laid out in memory. +Variable length items are length-prefixed. +While the encoding was inspired by Go, it is easily implemented in other languages as well, given its intuitive design. + +XXX: This is changing to use real varints and 4-byte-prefixes. +See https://github.com/tendermint/go-wire/tree/sdk2. + +### Fixed Length Integers + +Fixed length integers are encoded in Big-Endian using the specified number of bytes. +So `uint8` and `int8` use one byte, `uint16` and `int16` use two bytes, +`uint32` and `int32` use 3 bytes, and `uint64` and `int64` use 4 bytes. + +Negative integers are encoded via twos-complement. + +Examples: + +```go +encode(uint8(6)) == [0x06] +encode(uint32(6)) == [0x00, 0x00, 0x00, 0x06] + +encode(int8(-6)) == [0xFA] +encode(int32(-6)) == [0xFF, 0xFF, 0xFF, 0xFA] +``` + +### Variable Length Integers + +Variable length integers are encoded as length-prefixed Big-Endian integers. +The length-prefix consists of a single byte and corresponds to the length of the encoded integer. + +Negative integers are encoded by flipping the leading bit of the length-prefix to a `1`. + +Zero is encoded as `0x00`. It is not length-prefixed. + +Examples: + +```go +encode(uint(6)) == [0x01, 0x06] +encode(uint(70000)) == [0x03, 0x01, 0x11, 0x70] + +encode(int(-6)) == [0xF1, 0x06] +encode(int(-70000)) == [0xF3, 0x01, 0x11, 0x70] + +encode(int(0)) == [0x00] +``` + +### Strings + +An encoded string is length-prefixed followed by the underlying bytes of the string. +The length-prefix is itself encoded as an `int`. + +The empty string is encoded as `0x00`. It is not length-prefixed. + +Examples: + +```go +encode("") == [0x00] +encode("a") == [0x01, 0x01, 0x61] +encode("hello") == [0x01, 0x05, 0x68, 0x65, 0x6C, 0x6C, 0x6F] +encode("¥") == [0x01, 0x02, 0xC2, 0xA5] +``` + +### Arrays (fixed length) + +An encoded fix-lengthed array is the concatenation of the encoding of its elements. +There is no length-prefix. + +Examples: + +```go +encode([4]int8{1, 2, 3, 4}) == [0x01, 0x02, 0x03, 0x04] +encode([4]int16{1, 2, 3, 4}) == [0x00, 0x01, 0x00, 0x02, 0x00, 0x03, 0x00, 0x04] +encode([4]int{1, 2, 3, 4}) == [0x01, 0x01, 0x01, 0x02, 0x01, 0x03, 0x01, 0x04] +encode([2]string{"abc", "efg"}) == [0x01, 0x03, 0x61, 0x62, 0x63, 0x01, 0x03, 0x65, 0x66, 0x67] +``` + +### Slices (variable length) + +An encoded variable-length array is length-prefixed followed by the concatenation of the encoding of +its elements. +The length-prefix is itself encoded as an `int`. + +An empty slice is encoded as `0x00`. It is not length-prefixed. + +Examples: + +```go +encode([]int8{}) == [0x00] +encode([]int8{1, 2, 3, 4}) == [0x01, 0x04, 0x01, 0x02, 0x03, 0x04] +encode([]int16{1, 2, 3, 4}) == [0x01, 0x04, 0x00, 0x01, 0x00, 0x02, 0x00, 0x03, 0x00, 0x04] +encode([]int{1, 2, 3, 4}) == [0x01, 0x04, 0x01, 0x01, 0x01, 0x02, 0x01, 0x03, 0x01, 0x4] +encode([]string{"abc", "efg"}) == [0x01, 0x02, 0x01, 0x03, 0x61, 0x62, 0x63, 0x01, 0x03, 0x65, 0x66, 0x67] +``` + +### BitArray + +BitArray is encoded as an `int` of the number of bits, and with an array of `uint64` to encode +value of each array element. + +```go +type BitArray struct { + Bits int + Elems []uint64 +} +``` + +### Time + +Time is encoded as an `int64` of the number of nanoseconds since January 1, 1970, +rounded to the nearest millisecond. + +Times before then are invalid. + +Examples: + +```go +encode(time.Time("Jan 1 00:00:00 UTC 1970")) == [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] +encode(time.Time("Jan 1 00:00:01 UTC 1970")) == [0x00, 0x00, 0x00, 0x00, 0x3B, 0x9A, 0xCA, 0x00] // 1,000,000,000 ns +encode(time.Time("Mon Jan 2 15:04:05 -0700 MST 2006")) == [0x0F, 0xC4, 0xBB, 0xC1, 0x53, 0x03, 0x12, 0x00] +``` + +### Structs + +An encoded struct is the concatenation of the encoding of its elements. +There is no length-prefix. + +Examples: + +```go +type MyStruct struct{ + A int + B string + C time.Time +} +encode(MyStruct{4, "hello", time.Time("Mon Jan 2 15:04:05 -0700 MST 2006")}) == + [0x01, 0x04, 0x01, 0x05, 0x68, 0x65, 0x6C, 0x6C, 0x6F, 0x0F, 0xC4, 0xBB, 0xC1, 0x53, 0x03, 0x12, 0x00] +``` + +## Merkle Trees + +Simple Merkle trees are used in numerous places in Tendermint to compute a cryptographic digest of a data structure. + +RIPEMD160 is always used as the hashing function. + +The function `SimpleMerkleRoot` is a simple recursive function defined as follows: + +```go +func SimpleMerkleRoot(hashes [][]byte) []byte{ + switch len(hashes) { + case 0: + return nil + case 1: + return hashes[0] + default: + left := SimpleMerkleRoot(hashes[:(len(hashes)+1)/2]) + right := SimpleMerkleRoot(hashes[(len(hashes)+1)/2:]) + return RIPEMD160(append(left, right)) + } +} +``` + +Note: we abuse notion and call `SimpleMerkleRoot` with arguments of type `struct` or type `[]struct`. +For `struct` arguments, we compute a `[][]byte` by sorting elements of the `struct` according to +field name and then hashing them. +For `[]struct` arguments, we compute a `[][]byte` by hashing the individual `struct` elements. + +## JSON (TMJSON) + +Signed messages (eg. votes, proposals) in the consensus are encoded in TMJSON, rather than TMBIN. +TMJSON is JSON where `[]byte` are encoded as uppercase hex, rather than base64. + +When signing, the elements of a message are sorted by key and the sorted message is embedded in an +outer JSON that includes a `chain_id` field. +We call this encoding the CanonicalSignBytes. For instance, CanonicalSignBytes for a vote would look +like: + +```json +{"chain_id":"my-chain-id","vote":{"block_id":{"hash":DEADBEEF,"parts":{"hash":BEEFDEAD,"total":3}},"height":3,"round":2,"timestamp":1234567890, "type":2} +``` + +Note how the fields within each level are sorted. + +## Other + +### MakeParts + +Encode an object using TMBIN and slice it into parts. + +```go +MakeParts(object, partSize) +``` + +### Part + +```go +type Part struct { + Index int + Bytes byte[] + Proof byte[] +} +``` diff --git a/docs/specification/new-spec/state.md b/docs/specification/new-spec/state.md index abd32edba..c7de007f3 100644 --- a/docs/specification/new-spec/state.md +++ b/docs/specification/new-spec/state.md @@ -74,9 +74,6 @@ func TotalVotingPower(vals []Validators) int64{ } ``` -### PubKey - -TODO: ### ConsensusParams diff --git a/version/version.go b/version/version.go index b0123eb1f..f07690271 100644 --- a/version/version.go +++ b/version/version.go @@ -1,8 +1,11 @@ package version -const Maj = "0" -const Min = "19" -const Fix = "0" +// Version components +const ( + Maj = "0" + Min = "19" + Fix = "0" +) var ( // Version is the current version of Tendermint From 97f3ada9c2bcd37694bbbd4953247640f400fd88 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 26 Apr 2018 21:31:48 -0400 Subject: [PATCH 129/143] update godep --- Gopkg.lock | 21 +++++++++++---------- Gopkg.toml | 2 +- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 0701a1faa..cfb6f6c0b 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -5,7 +5,7 @@ branch = "master" name = "github.com/btcsuite/btcd" packages = ["btcec"] - revision = "2be2f12b358dc57d70b8f501b00be450192efbc3" + revision = "675abc5df3c5531bc741b56a765e35623459da6d" [[projects]] name = "github.com/davecgh/go-spew" @@ -254,8 +254,8 @@ [[projects]] name = "github.com/tendermint/go-amino" packages = ["."] - revision = "42246108ff925a457fb709475070a03dfd3e2b5c" - version = "0.9.6" + revision = "3668c02a8feace009f80754a5e5a8541e5d7b996" + version = "0.9.8" [[projects]] name = "github.com/tendermint/go-crypto" @@ -285,8 +285,8 @@ "pubsub/query", "test" ] - revision = "97e1f1ad3f510048929a51475811a18686c894df" - version = "0.8.2-rc0" + revision = "d94e312673e16a11ea55d742cefb3e331228f898" + version = "v0.8.2" [[projects]] branch = "master" @@ -301,13 +301,14 @@ "ripemd160", "salsa20/salsa" ] - revision = "d6449816ce06963d9d136eee5a56fca5b0616e7e" + revision = "b49d69b5da943f7ef3c9cf91c8777c1f78a0cc3c" [[projects]] branch = "master" name = "golang.org/x/net" packages = [ "context", + "http/httpguts", "http2", "http2/hpack", "idna", @@ -315,13 +316,13 @@ "lex/httplex", "trace" ] - revision = "61147c48b25b599e5b561d2e9c4f3e1ef489ca41" + revision = "5f9ae10d9af5b1c89ae6904293b14b064d4ada23" [[projects]] branch = "master" name = "golang.org/x/sys" packages = ["unix"] - revision = "3b87a42e500a6dc65dae1a55d0b641295971163e" + revision = "bb9c189858d91f42db229b04d45a4c3d23a7662a" [[projects]] name = "golang.org/x/text" @@ -348,7 +349,7 @@ branch = "master" name = "google.golang.org/genproto" packages = ["googleapis/rpc/status"] - revision = "51d0944304c3cbce4afe9e5247e21100037bff78" + revision = "7fd901a49ba6a7f87732eb344f6e3c5b19d1b200" [[projects]] name = "google.golang.org/grpc" @@ -383,6 +384,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "e70f8692c825e80ae8510546e297840b9560d00e11b2272749a55cc2ffd147f0" + inputs-digest = "94cb2543199b0f4b6e9ac0e5b6469bdb77391da1c9f79f5b9792d7af936008ff" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 8e65a41a3..4bf45ecd8 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -79,7 +79,7 @@ [[constraint]] name = "github.com/tendermint/go-amino" - version = "~0.9.6" + version = "~0.9.7" [[constraint]] name = "github.com/tendermint/tmlibs" From 94c016a04e65611846a5c36b482961350a300d58 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 26 Apr 2018 21:32:14 -0400 Subject: [PATCH 130/143] use MarshalJSONIndent for init files. closes #1506 --- consensus/replay_file.go | 3 ++- consensus/types/state.go | 11 ++++++----- types/genesis.go | 2 +- types/priv_validator/priv_validator.go | 2 +- types/validator_set.go | 5 ++++- 5 files changed, 14 insertions(+), 9 deletions(-) diff --git a/consensus/replay_file.go b/consensus/replay_file.go index 1fd4f415d..58f022274 100644 --- a/consensus/replay_file.go +++ b/consensus/replay_file.go @@ -29,6 +29,7 @@ const ( //-------------------------------------------------------- // replay messages interactively or all at once +// replay the wal file func RunReplayFile(config cfg.BaseConfig, csConfig *cfg.ConsensusConfig, console bool) { consensusState := newConsensusStateForReplay(config, csConfig) @@ -262,7 +263,7 @@ func (pb *playback) replayConsoleLoop() int { case "locked_block": fmt.Printf("%v %v\n", rs.LockedBlockParts.StringShort(), rs.LockedBlock.StringShort()) case "votes": - fmt.Println(rs.Votes.StringIndented(" ")) + fmt.Println(rs.Votes.StringIndented(" ")) default: fmt.Println("Unknown option", tokens[1]) diff --git a/consensus/types/state.go b/consensus/types/state.go index 255595e54..65c40781e 100644 --- a/consensus/types/state.go +++ b/consensus/types/state.go @@ -13,6 +13,7 @@ import ( // RoundStepType enumerates the state of the consensus state machine type RoundStepType uint8 // These must be numeric, ordered. +// RoundStepType const ( RoundStepNewHeight = RoundStepType(0x01) // Wait til CommitTime + timeoutCommit RoundStepNewRound = RoundStepType(0x02) // Setup new round and go to RoundStepPropose @@ -80,12 +81,12 @@ type RoundState struct { func (rs *RoundState) RoundStateEvent() types.EventDataRoundState { // XXX: copy the RoundState // if we want to avoid this, we may need synchronous events after all - rs_ := *rs + rsCopy := *rs edrs := types.EventDataRoundState{ Height: rs.Height, Round: rs.Round, Step: rs.Step.String(), - RoundState: &rs_, + RoundState: &rsCopy, } return edrs } @@ -115,16 +116,16 @@ func (rs *RoundState) StringIndented(indent string) string { indent, rs.Height, rs.Round, rs.Step, indent, rs.StartTime, indent, rs.CommitTime, - indent, rs.Validators.StringIndented(indent+" "), + indent, rs.Validators.StringIndented(indent+" "), indent, rs.Proposal, indent, rs.ProposalBlockParts.StringShort(), rs.ProposalBlock.StringShort(), indent, rs.LockedRound, indent, rs.LockedBlockParts.StringShort(), rs.LockedBlock.StringShort(), indent, rs.ValidRound, indent, rs.ValidBlockParts.StringShort(), rs.ValidBlock.StringShort(), - indent, rs.Votes.StringIndented(indent+" "), + indent, rs.Votes.StringIndented(indent+" "), indent, rs.LastCommit.StringShort(), - indent, rs.LastValidators.StringIndented(indent+" "), + indent, rs.LastValidators.StringIndented(indent+" "), indent) } diff --git a/types/genesis.go b/types/genesis.go index faf511196..aee8e0767 100644 --- a/types/genesis.go +++ b/types/genesis.go @@ -41,7 +41,7 @@ func (genDoc *GenesisDoc) AppState() json.RawMessage { // SaveAs is a utility method for saving GenensisDoc as a JSON file. func (genDoc *GenesisDoc) SaveAs(file string) error { - genDocBytes, err := cdc.MarshalJSON(genDoc) + genDocBytes, err := cdc.MarshalJSONIndent(genDoc, "", " ") if err != nil { return err } diff --git a/types/priv_validator/priv_validator.go b/types/priv_validator/priv_validator.go index baff28f64..2bb5ef323 100644 --- a/types/priv_validator/priv_validator.go +++ b/types/priv_validator/priv_validator.go @@ -120,7 +120,7 @@ func (pv *FilePV) save() { if outFile == "" { panic("Cannot save PrivValidator: filePath not set") } - jsonBytes, err := cdc.MarshalJSON(pv) + jsonBytes, err := cdc.MarshalJSONIndent(pv, "", " ") if err != nil { panic(err) } diff --git a/types/validator_set.go b/types/validator_set.go index fa6a2ad9c..28d954f35 100644 --- a/types/validator_set.go +++ b/types/validator_set.go @@ -69,6 +69,7 @@ func (valSet *ValidatorSet) IncrementAccum(times int) { } } +// Copy each validator into a new ValidatorSet func (valSet *ValidatorSet) Copy() *ValidatorSet { validators := make([]*Validator, len(valSet.Validators)) for i, val := range valSet.Validators { @@ -368,6 +369,7 @@ func (valSet *ValidatorSet) String() string { return valSet.StringIndented("") } +// String func (valSet *ValidatorSet) StringIndented(indent string) string { if valSet == nil { return "nil-ValidatorSet" @@ -384,7 +386,7 @@ func (valSet *ValidatorSet) StringIndented(indent string) string { %s}`, indent, valSet.GetProposer().String(), indent, - indent, strings.Join(valStrings, "\n"+indent+" "), + indent, strings.Join(valStrings, "\n"+indent+" "), indent) } @@ -392,6 +394,7 @@ func (valSet *ValidatorSet) StringIndented(indent string) string { //------------------------------------- // Implements sort for sorting validators by address. +// Sort validators by address type ValidatorsByAddress []*Validator func (vs ValidatorsByAddress) Len() int { From 9ab1fafdf134ec24cead0902f7a92b6ace1abc84 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 26 Apr 2018 21:57:59 -0400 Subject: [PATCH 131/143] some amino json for #1504 --- docs/specification/new-spec/encoding.md | 34 +++++++++++++++++++++---- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/docs/specification/new-spec/encoding.md b/docs/specification/new-spec/encoding.md index ae6e003d8..f897bb1dc 100644 --- a/docs/specification/new-spec/encoding.md +++ b/docs/specification/new-spec/encoding.md @@ -277,9 +277,36 @@ func computeHashFromAunts(index, total int, leafHash []byte, innerHashes [][]byt } ``` -## AminoJSON +## JSON -Signed messages (eg. votes, proposals) in the consensus are encoded in AminoJSON, rather than binary Amino. +### Amino + +TODO: improve this + +Amino also supports JSON encoding - registered types are simply encoded as: + +``` +{ + "type": "", + "value": +} + +For instance, an ED25519 PubKey would look like: + +``` +{ + "type": "AC26791624DE60", + "value": "uZ4h63OFWuQ36ZZ4Bd6NF+/w9fWUwrOncrQsackrsTk=" +} +``` + +Where the `"value"` is the base64 encoding of the raw pubkey bytes, and the +`"type"` is the full disfix bytes for Ed25519 pubkeys. + + +### Signed Messages + +Signed messages (eg. votes, proposals) in the consensus are encoded using Amino-JSON, rather than in the standard binary format. When signing, the elements of a message are sorted by key and the sorted message is embedded in an outer JSON that includes a `chain_id` field. @@ -291,6 +318,3 @@ like: ``` Note how the fields within each level are sorted. - - - From a2d77cbe4e02c4b8843a3cb7301f6c9fd23523ae Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 26 Apr 2018 23:11:20 -0400 Subject: [PATCH 132/143] add MarshalJSON methods to fix dump_consensus_state --- consensus/types/height_vote_set.go | 24 ++++++++++++++++++++++++ types/part_set.go | 17 +++++++++++++++++ types/vote_set.go | 20 ++++++++++++++++++++ 3 files changed, 61 insertions(+) diff --git a/consensus/types/height_vote_set.go b/consensus/types/height_vote_set.go index a155bce08..f65f365b8 100644 --- a/consensus/types/height_vote_set.go +++ b/consensus/types/height_vote_set.go @@ -207,6 +207,30 @@ func (hvs *HeightVoteSet) StringIndented(indent string) string { indent) } +type roundVoteBitArrays struct { + Round int `json:"round"` + Prevotes *cmn.BitArray `json:"prevotes"` + Precommits *cmn.BitArray `json:"precommits"` +} + +func (hvs *HeightVoteSet) MarshalJSON() ([]byte, error) { + hvs.mtx.Lock() + defer hvs.mtx.Unlock() + totalRounds := hvs.round + 1 + roundsVotes := make([]roundVoteBitArrays, totalRounds) + // rounds 0 ~ hvs.round inclusive + for round := 0; round < totalRounds; round++ { + roundsVotes[round] = roundVoteBitArrays{ + Round: round, + Prevotes: hvs.roundVoteSets[round].Prevotes.BitArray(), + Precommits: hvs.roundVoteSets[round].Precommits.BitArray(), + } + } + // TODO: all other peer catchup rounds + + return cdc.MarshalJSON(roundsVotes) +} + // If a peer claims that it has 2/3 majority for given blockKey, call this. // NOTE: if there are too many peers, or too much peer churn, // this can cause memory issues. diff --git a/types/part_set.go b/types/part_set.go index 749943291..e4f2b9e97 100644 --- a/types/part_set.go +++ b/types/part_set.go @@ -264,3 +264,20 @@ func (ps *PartSet) StringShort() string { defer ps.mtx.Unlock() return fmt.Sprintf("(%v of %v)", ps.Count(), ps.Total()) } + +func (ps *PartSet) MarshalJSON() ([]byte, error) { + if ps == nil { + return []byte("nil-PartSet"), nil + } + + ps.mtx.Lock() + defer ps.mtx.Unlock() + + return cdc.MarshalJSON(struct { + CountTotal string `json:"count/total"` + PartsBitArray *cmn.BitArray `json:"parts_bit_array"` + }{ + fmt.Sprintf("%d/%d", ps.Count(), ps.Total()), + ps.partsBitArray.Copy(), + }) +} diff --git a/types/vote_set.go b/types/vote_set.go index e255488d6..07ef60ec6 100644 --- a/types/vote_set.go +++ b/types/vote_set.go @@ -445,6 +445,26 @@ func (voteSet *VoteSet) StringIndented(indent string) string { indent) } +// Marshal the VoteSet to JSON. Same as String(), just in JSON, +// and without the height/round/type_ (since its already included in the votes). +func (voteSet *VoteSet) MarshalJSON() ([]byte, error) { + voteStrings := make([]string, len(voteSet.votes)) + for i, vote := range voteSet.votes { + if vote == nil { + voteStrings[i] = "nil-Vote" + } else { + voteStrings[i] = vote.String() + } + } + return cdc.MarshalJSON(struct { + Votes []string `json:"votes"` + VotesBitArray *cmn.BitArray `json:"votes_bit_array"` + PeerMaj23s map[P2PID]BlockID `json:"peer_maj_23s"` + }{ + voteStrings, voteSet.votesBitArray, voteSet.peerMaj23s, + }) +} + func (voteSet *VoteSet) StringShort() string { if voteSet == nil { return "nil-VoteSet" From 47e4d64973fe4efb201c17a2dad2cea4f20efa13 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 26 Apr 2018 23:13:56 -0400 Subject: [PATCH 133/143] add validator address to /status --- rpc/core/status.go | 16 ++++++++++------ rpc/core/types/responses.go | 1 + 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/rpc/core/status.go b/rpc/core/status.go index 9bdf9dd64..32647add5 100644 --- a/rpc/core/status.go +++ b/rpc/core/status.go @@ -79,6 +79,11 @@ func Status() (*ctypes.ResultStatus, error) { latestBlockTime := time.Unix(0, latestBlockTimeNano) + var votingPower int64 + if val := validatorAtHeight(latestHeight); val != nil { + votingPower = val.VotingPower + } + result := &ctypes.ResultStatus{ NodeInfo: p2pSwitch.NodeInfo(), SyncInfo: ctypes.SyncInfo{ @@ -88,12 +93,11 @@ func Status() (*ctypes.ResultStatus, error) { LatestBlockTime: latestBlockTime, Syncing: consensusReactor.FastSync(), }, - ValidatorInfo: ctypes.ValidatorInfo{PubKey: pubKey}, - } - - // add ValidatorStatus if node is a validator - if val := validatorAtHeight(latestHeight); val != nil { - result.ValidatorInfo.VotingPower = val.VotingPower + ValidatorInfo: ctypes.ValidatorInfo{ + Address: pubKey.Address(), + PubKey: pubKey, + VotingPower: votingPower, + }, } return result, nil diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index 82bfceaa7..ef4d72614 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -63,6 +63,7 @@ type SyncInfo struct { } type ValidatorInfo struct { + Address cmn.HexBytes `json:"address"` PubKey crypto.PubKey `json:"pub_key"` VotingPower int64 `json:"voting_power"` } From 94e823cc91c4d0b9103f1086ede772f14e33a1ce Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 26 Apr 2018 23:40:29 -0400 Subject: [PATCH 134/143] p2p: NodeInfo.Channels is HexBytes --- p2p/node_info.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/p2p/node_info.go b/p2p/node_info.go index 930f9bfea..e46236188 100644 --- a/p2p/node_info.go +++ b/p2p/node_info.go @@ -3,6 +3,8 @@ package p2p import ( "fmt" "strings" + + cmn "github.com/tendermint/tmlibs/common" ) const ( @@ -10,6 +12,7 @@ const ( maxNumChannels = 16 // plenty of room for upgrades, for now ) +// Max size of the NodeInfo struct func MaxNodeInfoSize() int { return maxNodeInfoSize } @@ -21,10 +24,11 @@ type NodeInfo struct { ID ID `json:"id"` // authenticated identifier ListenAddr string `json:"listen_addr"` // accepting incoming - // Check compatibility - Network string `json:"network"` // network/chain ID - Version string `json:"version"` // major.minor.revision - Channels []byte `json:"channels"` // channels this node knows about + // Check compatibility. + // Channels are HexBytes so easier to read as JSON + Network string `json:"network"` // network/chain ID + Version string `json:"version"` // major.minor.revision + Channels cmn.HexBytes `json:"channels"` // channels this node knows about // Sanitize Moniker string `json:"moniker"` // arbitrary moniker From 593a785ae278c525b9879ef4579bf2fd3fe269f8 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 26 Apr 2018 23:49:48 -0400 Subject: [PATCH 135/143] set NodeInfo on switch before starting RPC server should fix #1199 --- node/node.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/node/node.go b/node/node.go index 976066964..d3d8968de 100644 --- a/node/node.go +++ b/node/node.go @@ -382,16 +382,6 @@ func (n *Node) OnStart() error { return err } - // Run the RPC server first - // so we can eg. receive txs for the first block - if n.config.RPC.ListenAddress != "" { - listeners, err := n.startRPC() - if err != nil { - return err - } - n.rpcListeners = listeners - } - // Create & add listener protocol, address := cmn.ProtocolAndAddress(n.config.P2P.ListenAddress) l := p2p.NewDefaultListener(protocol, address, n.config.P2P.SkipUPNP, n.Logger.With("module", "p2p")) @@ -412,6 +402,16 @@ func (n *Node) OnStart() error { // Add ourselves to addrbook to prevent dialing ourselves n.addrBook.AddOurAddress(nodeInfo.NetAddress()) + // Run the RPC server first + // so we can eg. receive txs for the first block + if n.config.RPC.ListenAddress != "" { + listeners, err := n.startRPC() + if err != nil { + return err + } + n.rpcListeners = listeners + } + // Start the switch err = n.sw.Start() if err != nil { From f67c5a9e7b914ed7bd13a0a8f10822ce57ec4bbb Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 27 Apr 2018 10:00:34 -0400 Subject: [PATCH 136/143] forgot wire.go file --- consensus/types/wire.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 consensus/types/wire.go diff --git a/consensus/types/wire.go b/consensus/types/wire.go new file mode 100644 index 000000000..bd5c4497d --- /dev/null +++ b/consensus/types/wire.go @@ -0,0 +1,12 @@ +package types + +import ( + "github.com/tendermint/go-amino" + "github.com/tendermint/go-crypto" +) + +var cdc = amino.NewCodec() + +func init() { + crypto.RegisterAmino(cdc) +} From c45ba2967af8232d0ce7897231e47c607acb758b Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 27 Apr 2018 10:29:05 -0400 Subject: [PATCH 137/143] fixes from review --- node/node.go | 4 ++-- types/part_set.go | 4 ++-- types/vote_set.go | 8 +++++++- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/node/node.go b/node/node.go index d3d8968de..205e672f3 100644 --- a/node/node.go +++ b/node/node.go @@ -402,7 +402,7 @@ func (n *Node) OnStart() error { // Add ourselves to addrbook to prevent dialing ourselves n.addrBook.AddOurAddress(nodeInfo.NetAddress()) - // Run the RPC server first + // Start the RPC server before the P2P server // so we can eg. receive txs for the first block if n.config.RPC.ListenAddress != "" { listeners, err := n.startRPC() @@ -412,7 +412,7 @@ func (n *Node) OnStart() error { n.rpcListeners = listeners } - // Start the switch + // Start the switch (the P2P server). err = n.sw.Start() if err != nil { return err diff --git a/types/part_set.go b/types/part_set.go index e4f2b9e97..cad3a03fe 100644 --- a/types/part_set.go +++ b/types/part_set.go @@ -267,7 +267,7 @@ func (ps *PartSet) StringShort() string { func (ps *PartSet) MarshalJSON() ([]byte, error) { if ps == nil { - return []byte("nil-PartSet"), nil + return []byte("{}"), nil } ps.mtx.Lock() @@ -278,6 +278,6 @@ func (ps *PartSet) MarshalJSON() ([]byte, error) { PartsBitArray *cmn.BitArray `json:"parts_bit_array"` }{ fmt.Sprintf("%d/%d", ps.Count(), ps.Total()), - ps.partsBitArray.Copy(), + ps.partsBitArray, }) } diff --git a/types/vote_set.go b/types/vote_set.go index 07ef60ec6..c5c72d435 100644 --- a/types/vote_set.go +++ b/types/vote_set.go @@ -56,9 +56,9 @@ type VoteSet struct { height int64 round int type_ byte + valSet *ValidatorSet mtx sync.Mutex - valSet *ValidatorSet votesBitArray *cmn.BitArray votes []*Vote // Primary votes to share sum int64 // Sum of voting power for seen votes, discounting conflicts @@ -399,6 +399,8 @@ func (voteSet *VoteSet) HasTwoThirdsAny() bool { } func (voteSet *VoteSet) HasAll() bool { + voteSet.mtx.Lock() + defer voteSet.mtx.Unlock() return voteSet.sum == voteSet.valSet.TotalVotingPower() } @@ -424,6 +426,8 @@ func (voteSet *VoteSet) String() string { } func (voteSet *VoteSet) StringIndented(indent string) string { + voteSet.mtx.Lock() + defer voteSet.mtx.Unlock() voteStrings := make([]string, len(voteSet.votes)) for i, vote := range voteSet.votes { if vote == nil { @@ -448,6 +452,8 @@ func (voteSet *VoteSet) StringIndented(indent string) string { // Marshal the VoteSet to JSON. Same as String(), just in JSON, // and without the height/round/type_ (since its already included in the votes). func (voteSet *VoteSet) MarshalJSON() ([]byte, error) { + voteSet.mtx.Lock() + defer voteSet.mtx.Unlock() voteStrings := make([]string, len(voteSet.votes)) for i, vote := range voteSet.votes { if vote == nil { From 65ebbccb74d715dcee579460ac7c604c88decd8a Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Thu, 26 Apr 2018 03:55:57 -0700 Subject: [PATCH 138/143] Add GoAmino DeepCopy() benchmark for RoundState --- .../types/{reactor.go => peer_round_state.go} | 0 consensus/types/{state.go => round_state.go} | 0 consensus/types/round_state_test.go | 95 +++++++++++++++++++ 3 files changed, 95 insertions(+) rename consensus/types/{reactor.go => peer_round_state.go} (100%) rename consensus/types/{state.go => round_state.go} (100%) create mode 100644 consensus/types/round_state_test.go diff --git a/consensus/types/reactor.go b/consensus/types/peer_round_state.go similarity index 100% rename from consensus/types/reactor.go rename to consensus/types/peer_round_state.go diff --git a/consensus/types/state.go b/consensus/types/round_state.go similarity index 100% rename from consensus/types/state.go rename to consensus/types/round_state.go diff --git a/consensus/types/round_state_test.go b/consensus/types/round_state_test.go new file mode 100644 index 000000000..dc88c3a13 --- /dev/null +++ b/consensus/types/round_state_test.go @@ -0,0 +1,95 @@ +package types + +import ( + "testing" + "time" + + "github.com/tendermint/go-amino" + "github.com/tendermint/go-crypto" + "github.com/tendermint/tendermint/types" + cmn "github.com/tendermint/tmlibs/common" +) + +func BenchmarkRoundStateDeepCopy(b *testing.B) { + b.StopTimer() + + // Random validators + nval, ntxs := 100, 100 + vset, _ := types.RandValidatorSet(nval, 1) + precommits := make([]*types.Vote, nval) + blockID := types.BlockID{ + Hash: cmn.RandBytes(20), + PartsHeader: types.PartSetHeader{ + Hash: cmn.RandBytes(20), + }, + } + sig := crypto.SignatureEd25519{} + for i := 0; i < nval; i++ { + precommits[i] = &types.Vote{ + ValidatorAddress: types.Address(cmn.RandBytes(20)), + Timestamp: time.Now(), + BlockID: blockID, + Signature: sig, + } + } + txs := make([]types.Tx, ntxs) + for i := 0; i < ntxs; i++ { + txs[i] = cmn.RandBytes(100) + } + // Random block + block := &types.Block{ + Header: &types.Header{ + ChainID: cmn.RandStr(12), + Time: time.Now(), + LastBlockID: blockID, + LastCommitHash: cmn.RandBytes(20), + DataHash: cmn.RandBytes(20), + ValidatorsHash: cmn.RandBytes(20), + ConsensusHash: cmn.RandBytes(20), + AppHash: cmn.RandBytes(20), + LastResultsHash: cmn.RandBytes(20), + EvidenceHash: cmn.RandBytes(20), + }, + Data: &types.Data{ + Txs: txs, + }, + Evidence: types.EvidenceData{}, + LastCommit: &types.Commit{ + BlockID: blockID, + Precommits: precommits, + }, + } + parts := block.MakePartSet(4096) + // Random Proposal + proposal := &types.Proposal{ + Timestamp: time.Now(), + BlockPartsHeader: types.PartSetHeader{ + Hash: cmn.RandBytes(20), + }, + POLBlockID: blockID, + Signature: sig, + } + // Random HeightVoteSet + // TODO: hvs := + + rs := &RoundState{ + StartTime: time.Now(), + CommitTime: time.Now(), + Validators: vset, + Proposal: proposal, + ProposalBlock: block, + ProposalBlockParts: parts, + LockedBlock: block, + LockedBlockParts: parts, + ValidBlock: block, + ValidBlockParts: parts, + Votes: nil, // TODO + LastCommit: nil, // TODO + LastValidators: vset, + } + b.StartTimer() + + for i := 0; i < b.N; i++ { + amino.DeepCopy(rs) + } +} From 1ab89e6cbfb59681a7a5dc8503020e6d661a1f0d Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 27 Apr 2018 12:23:00 -0400 Subject: [PATCH 139/143] update changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d6562096..0c2701e57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,12 @@ FEATURES: BUG FIXES +- [rpc] Lower case JSON field names +- [rpc] Fix missing entries and lower case the fields in `/dump_consensus_state` +- [rpc] Fix NodeInfo.Channels format to hex +- [rpc] Add Validator address to `/status` +- [rpc] Fix `prove` in ABCIQuery +- [cmd] MarshalJSONIndent on init - [spec] Document address format and pubkey encoding ## 0.19.0 (April 13th, 2018) From ffe81a0206699c28a99abb2f423e2a36bf3a0c94 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 27 Apr 2018 12:35:21 -0400 Subject: [PATCH 140/143] changelog and version --- CHANGELOG.md | 13 ++++++++++++- version/version.go | 4 ++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c2701e57..0a5a70811 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,23 +26,30 @@ BUG FIXES: ## 0.19.1 (April 27th, 2018) +Note this release includes some small breaking changes in the RPC and one in the +config that are really bug fixes. v0.19.1 will work with existing chains, and make Tendermint +easier to use and debug. With <3 + BREAKING (MINOR) - [config] Removed `wal_light` setting. If you really needed this, let us know FEATURES: +- [networks] moved in tooling from devops repo: terraform and ansible scripts for deploying testnets ! - [cmd] Added `gen_node_key` command BUG FIXES +Some of these are breaking in the RPC response, but they're really bugs! + +- [spec] Document address format and pubkey encoding pre and post Amino - [rpc] Lower case JSON field names - [rpc] Fix missing entries and lower case the fields in `/dump_consensus_state` - [rpc] Fix NodeInfo.Channels format to hex - [rpc] Add Validator address to `/status` - [rpc] Fix `prove` in ABCIQuery - [cmd] MarshalJSONIndent on init -- [spec] Document address format and pubkey encoding ## 0.19.0 (April 13th, 2018) @@ -59,6 +66,10 @@ See github.com/tendermint/go-amino for details on the new format. See `scripts/wire2amino.go` for a tool to upgrade genesis/priv_validator/node_key JSON files. +FEATURES + +- [test] docker-compose for local testnet setup (thanks Greg!) + ## 0.18.0 (April 6th, 2018) BREAKING: diff --git a/version/version.go b/version/version.go index f07690271..58e3f5e79 100644 --- a/version/version.go +++ b/version/version.go @@ -4,13 +4,13 @@ package version const ( Maj = "0" Min = "19" - Fix = "0" + Fix = "1" ) var ( // Version is the current version of Tendermint // Must be a string because scripts like dist.sh read this file. - Version = "0.19.0" + Version = "0.19.1" // GitCommit is the current HEAD set using ldflags. GitCommit string From f33da8817a20bcdb29115fb989f476d9f80e54f2 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 27 Apr 2018 23:00:09 -0400 Subject: [PATCH 141/143] rpc: lower_case peer_round_states, use a list, add the node_address --- consensus/state.go | 1 + consensus/types/peer_round_state.go | 30 ++++++++++++++--------------- rpc/core/consensus.go | 14 ++++++++------ rpc/core/types/responses.go | 9 +++++++-- 4 files changed, 31 insertions(+), 23 deletions(-) diff --git a/consensus/state.go b/consensus/state.go index bd069f70c..40b8f16d5 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -990,6 +990,7 @@ func (cs *ConsensusState) enterPrecommit(height int64, round int) { cs.newStep() }() + // check for a polka blockID, ok := cs.Votes.Prevotes(round).TwoThirdsMajority() // If we don't have a polka, we must precommit nil diff --git a/consensus/types/peer_round_state.go b/consensus/types/peer_round_state.go index 7dfeed923..dcb6c8e02 100644 --- a/consensus/types/peer_round_state.go +++ b/consensus/types/peer_round_state.go @@ -13,21 +13,21 @@ import ( // PeerRoundState contains the known state of a peer. // NOTE: Read-only when returned by PeerState.GetRoundState(). type PeerRoundState struct { - Height int64 // Height peer is at - Round int // Round peer is at, -1 if unknown. - Step RoundStepType // Step peer is at - StartTime time.Time // Estimated start of round 0 at this height - Proposal bool // True if peer has proposal for this round - ProposalBlockPartsHeader types.PartSetHeader // - ProposalBlockParts *cmn.BitArray // - ProposalPOLRound int // Proposal's POL round. -1 if none. - ProposalPOL *cmn.BitArray // nil until ProposalPOLMessage received. - Prevotes *cmn.BitArray // All votes peer has for this round - Precommits *cmn.BitArray // All precommits peer has for this round - LastCommitRound int // Round of commit for last height. -1 if none. - LastCommit *cmn.BitArray // All commit precommits of commit for last height. - CatchupCommitRound int // Round that we have commit for. Not necessarily unique. -1 if none. - CatchupCommit *cmn.BitArray // All commit precommits peer has for this height & CatchupCommitRound + Height int64 `json:"height"` // Height peer is at + Round int `json:"round"` // Round peer is at, -1 if unknown. + Step RoundStepType `json:"step"` // Step peer is at + StartTime time.Time `json:"start_time"` // Estimated start of round 0 at this height + Proposal bool `json:"proposal"` // True if peer has proposal for this round + ProposalBlockPartsHeader types.PartSetHeader `json:"proposal_block_parts_header"` // + ProposalBlockParts *cmn.BitArray `json:"proposal_block_parts"` // + ProposalPOLRound int `json:"proposal_pol_round"` // Proposal's POL round. -1 if none. + ProposalPOL *cmn.BitArray `json:"proposal_pol"` // nil until ProposalPOLMessage received. + Prevotes *cmn.BitArray `json:"prevotes"` // All votes peer has for this round + Precommits *cmn.BitArray `json:"precommits"` // All precommits peer has for this round + LastCommitRound int `json:"last_commit_round"` // Round of commit for last height. -1 if none. + LastCommit *cmn.BitArray `json:"last_commit"` // All commit precommits of commit for last height. + CatchupCommitRound int `json:"catchup_commit_round"` // Round that we have commit for. Not necessarily unique. -1 if none. + CatchupCommit *cmn.BitArray `json:"catchup_commit"` // All commit precommits peer has for this height & CatchupCommitRound } // String returns a string representation of the PeerRoundState diff --git a/rpc/core/consensus.go b/rpc/core/consensus.go index 7647aef7d..191b658f0 100644 --- a/rpc/core/consensus.go +++ b/rpc/core/consensus.go @@ -1,10 +1,8 @@ package core import ( - "encoding/json" - cm "github.com/tendermint/tendermint/consensus" - p2p "github.com/tendermint/tendermint/p2p" + "github.com/tendermint/tendermint/p2p" ctypes "github.com/tendermint/tendermint/rpc/core/types" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" @@ -84,14 +82,18 @@ func Validators(heightPtr *int64) (*ctypes.ResultValidators, error) { // } // ``` func DumpConsensusState() (*ctypes.ResultDumpConsensusState, error) { - peerRoundStates := make(map[p2p.ID]json.RawMessage) - for _, peer := range p2pSwitch.Peers().List() { + peers := p2pSwitch.Peers().List() + peerRoundStates := make([]ctypes.PeerRoundState, len(peers)) + for i, peer := range peers { peerState := peer.Get(types.PeerStateKey).(*cm.PeerState) peerRoundState, err := peerState.GetRoundStateJSON() if err != nil { return nil, err } - peerRoundStates[peer.ID()] = peerRoundState + peerRoundStates[i] = ctypes.PeerRoundState{ + NodeAddress: p2p.IDAddressString(peer.ID(), peer.NodeInfo().ListenAddr), + PeerRoundState: peerRoundState, + } } roundState, err := consensusState.GetRoundStateJSON() if err != nil { diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index ef4d72614..3a60be827 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -113,8 +113,13 @@ type ResultValidators struct { } type ResultDumpConsensusState struct { - RoundState json.RawMessage `json:"round_state"` - PeerRoundStates map[p2p.ID]json.RawMessage `json:"peer_round_states"` + RoundState json.RawMessage `json:"round_state"` + PeerRoundStates []PeerRoundState `json:"peer_round_states"` +} + +type PeerRoundState struct { + NodeAddress string `json:"node_address"` + PeerRoundState json.RawMessage `json:"peer_round_state"` } type ResultBroadcastTx struct { From 79bfbebfff2be69252cf7a620f5cdc6274fcaa6d Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 27 Apr 2018 23:19:40 -0400 Subject: [PATCH 142/143] rpc: docs/comments --- CHANGELOG.md | 2 +- rpc/core/consensus.go | 113 +++++++++++++++++++++++++++++++++--- rpc/core/status.go | 73 +++++++++++------------ rpc/core/types/responses.go | 42 +++++++++++--- 4 files changed, 175 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a5a70811..facc69635 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,7 +45,7 @@ Some of these are breaking in the RPC response, but they're really bugs! - [spec] Document address format and pubkey encoding pre and post Amino - [rpc] Lower case JSON field names -- [rpc] Fix missing entries and lower case the fields in `/dump_consensus_state` +- [rpc] Fix missing entries, improve, and lower case the fields in `/dump_consensus_state` - [rpc] Fix NodeInfo.Channels format to hex - [rpc] Add Validator address to `/status` - [rpc] Fix `prove` in ABCIQuery diff --git a/rpc/core/consensus.go b/rpc/core/consensus.go index 191b658f0..98b0699d0 100644 --- a/rpc/core/consensus.go +++ b/rpc/core/consensus.go @@ -71,16 +71,111 @@ func Validators(heightPtr *int64) (*ctypes.ResultValidators, error) { // > The above command returns JSON structured like this: // // ```json -// { -// "error": "", -// "result": { -// "peer_round_states": [], -// "round_state": "RoundState{\n H:3537 R:0 S:RoundStepNewHeight\n StartTime: 2017-05-31 12:32:31.178653883 +0000 UTC\n CommitTime: 2017-05-31 12:32:30.178653883 +0000 UTC\n Validators: ValidatorSet{\n Proposer: Validator{E89A51D60F68385E09E716D353373B11F8FACD62 {PubKeyEd25519{68DFDA7E50F82946E7E8546BED37944A422CD1B831E70DF66BA3B8430593944D}} VP:10 A:0}\n Validators:\n Validator{E89A51D60F68385E09E716D353373B11F8FACD62 {PubKeyEd25519{68DFDA7E50F82946E7E8546BED37944A422CD1B831E70DF66BA3B8430593944D}} VP:10 A:0}\n }\n Proposal: \n ProposalBlock: nil-PartSet nil-Block\n LockedRound: 0\n LockedBlock: nil-PartSet nil-Block\n Votes: HeightVoteSet{H:3537 R:0~0\n VoteSet{H:3537 R:0 T:1 +2/3: BA{1:_} map[]}\n VoteSet{H:3537 R:0 T:2 +2/3: BA{1:_} map[]}\n }\n LastCommit: VoteSet{H:3536 R:0 T:2 +2/3:B7F988FBCDC68F9320E346EECAA76E32F6054654:1:673BE7C01F74 BA{1:X} map[]}\n LastValidators: ValidatorSet{\n Proposer: Validator{E89A51D60F68385E09E716D353373B11F8FACD62 {PubKeyEd25519{68DFDA7E50F82946E7E8546BED37944A422CD1B831E70DF66BA3B8430593944D}} VP:10 A:0}\n Validators:\n Validator{E89A51D60F68385E09E716D353373B11F8FACD62 {PubKeyEd25519{68DFDA7E50F82946E7E8546BED37944A422CD1B831E70DF66BA3B8430593944D}} VP:10 A:0}\n }\n}" -// }, -// "id": "", -// "jsonrpc": "2.0" -// } +//{ +// "jsonrpc": "2.0", +// "id": "", +// "result": { +// "round_state": { +// "height": 138, +// "round": 0, +// "step": 1, +// "start_time": "2018-04-27T23:16:34.472087096-04:00", +// "commit_time": "2018-04-27T23:16:33.472087096-04:00", +// "validators": { +// "validators": [ +// { +// "address": "5875562FF0FFDECC895C20E32FC14988952E99E7", +// "pub_key": { +// "type": "AC26791624DE60", +// "value": "PpDJRUrLG2RgFqYYjawfn/AcAgacSXpLFrmfYYQnuzE=" +// }, +// "voting_power": 10, +// "accum": 0 +// } +// ], +// "proposer": { +// "address": "5875562FF0FFDECC895C20E32FC14988952E99E7", +// "pub_key": { +// "type": "AC26791624DE60", +// "value": "PpDJRUrLG2RgFqYYjawfn/AcAgacSXpLFrmfYYQnuzE=" +// }, +// "voting_power": 10, +// "accum": 0 +// } +// }, +// "proposal": null, +// "proposal_block": null, +// "proposal_block_parts": null, +// "locked_round": 0, +// "locked_block": null, +// "locked_block_parts": null, +// "valid_round": 0, +// "valid_block": null, +// "valid_block_parts": null, +// "votes": [ +// { +// "round": 0, +// "prevotes": "_", +// "precommits": "_" +// } +// ], +// "commit_round": -1, +// "last_commit": { +// "votes": [ +// "Vote{0:5875562FF0FF 137/00/2(Precommit) 5701C93659EA /ED3588D7AF29.../ @ 2018-04-28T03:16:33.469Z}" +// ], +// "votes_bit_array": "x", +// "peer_maj_23s": {} +// }, +// "last_validators": { +// "validators": [ +// { +// "address": "5875562FF0FFDECC895C20E32FC14988952E99E7", +// "pub_key": { +// "type": "AC26791624DE60", +// "value": "PpDJRUrLG2RgFqYYjawfn/AcAgacSXpLFrmfYYQnuzE=" +// }, +// "voting_power": 10, +// "accum": 0 +// } +// ], +// "proposer": { +// "address": "5875562FF0FFDECC895C20E32FC14988952E99E7", +// "pub_key": { +// "type": "AC26791624DE60", +// "value": "PpDJRUrLG2RgFqYYjawfn/AcAgacSXpLFrmfYYQnuzE=" +// }, +// "voting_power": 10, +// "accum": 0 +// } +// } +// }, +// "peer_round_states": { +// "d4bf26bfa5e390b94d98106ab858abf64db26d48": { +// "Height": 136, +// "Round": 0, +// "Step": 1, +// "StartTime": "2018-04-27T23:16:33.841163812-04:00", +// "Proposal": false, +// "ProposalBlockPartsHeader": { +// "total": 1, +// "hash": "E27F2D13298F7CB14090EE60CD9AB214D2F5161F" +// }, +// "ProposalBlockParts": "x", +// "ProposalPOLRound": -1, +// "ProposalPOL": "_", +// "Prevotes": "_", +// "Precommits": "x", +// "LastCommitRound": 0, +// "LastCommit": null, +// "CatchupCommitRound": 0, +// "CatchupCommit": "_" +// } +// } +// } +//} // ``` +// UNSTABLE func DumpConsensusState() (*ctypes.ResultDumpConsensusState, error) { peers := p2pSwitch.Peers().List() peerRoundStates := make([]ctypes.PeerRoundState, len(peers)) diff --git a/rpc/core/status.go b/rpc/core/status.go index 32647add5..4b2ceca2d 100644 --- a/rpc/core/status.go +++ b/rpc/core/status.go @@ -25,42 +25,43 @@ import ( // > The above command returns JSON structured like this: // // ```json -// { -// "result": { -// "sync_info": { -// "syncing": false, -// "latest_block_time": "2017-12-07T18:19:47.617Z", -// "latest_block_height": 6, -// "latest_app_hash": "", -// "latest_block_hash": "A63D0C3307DEDCCFCC82ED411AE9108B70B29E02", -// } -// "validator_info": { -// "pub_key": { -// "data": "8C9A68070CBE33F9C445862BA1E9D96A75CEB68C0CF6ADD3652D07DCAC5D0380", -// "type": "ed25519" -// }, -// "voting_power": 10 -// } -// "node_info": { -// "other": [ -// "wire_version=0.7.2", -// "p2p_version=0.5.0", -// "consensus_version=v1/0.2.2", -// "rpc_version=0.7.0/3", -// "tx_index=on", -// "rpc_addr=tcp://0.0.0.0:46657" -// ], -// "version": "0.13.0-14ccc8b", -// "listen_addr": "10.0.2.15:46656", -// "remote_addr": "", -// "network": "test-chain-qhVCa2", -// "moniker": "vagrant-ubuntu-trusty-64", -// "pub_key": "844981FE99ABB19F7816F2D5E94E8A74276AB1153760A7799E925C75401856C6", -// } -// }, -// "id": "", -// "jsonrpc": "2.0" -// } +//{ +// "jsonrpc": "2.0", +// "id": "", +// "result": { +// "node_info": { +// "id": "562dd7f579f0ecee8c94a11a3c1e378c1876f433", +// "listen_addr": "192.168.1.2:46656", +// "network": "test-chain-I6zScH", +// "version": "0.19.0", +// "channels": "4020212223303800", +// "moniker": "Ethans-MacBook-Pro.local", +// "other": [ +// "amino_version=0.9.8", +// "p2p_version=0.5.0", +// "consensus_version=v1/0.2.2", +// "rpc_version=0.7.0/3", +// "tx_index=on", +// "rpc_addr=tcp://0.0.0.0:46657" +// ] +// }, +// "sync_info": { +// "latest_block_hash": "2D4D7055BE685E3CB2410603C92AD37AE557AC59", +// "latest_app_hash": "0000000000000000", +// "latest_block_height": 231, +// "latest_block_time": "2018-04-27T23:18:08.459766485-04:00", +// "syncing": false +// }, +// "validator_info": { +// "address": "5875562FF0FFDECC895C20E32FC14988952E99E7", +// "pub_key": { +// "type": "AC26791624DE60", +// "value": "PpDJRUrLG2RgFqYYjawfn/AcAgacSXpLFrmfYYQnuzE=" +// }, +// "voting_power": 10 +// } +// } +//} // ``` func Status() (*ctypes.ResultStatus, error) { latestHeight := blockStore.Height() diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index 3a60be827..be7d51f20 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -14,20 +14,24 @@ import ( "github.com/tendermint/tendermint/types" ) +// List of blocks type ResultBlockchainInfo struct { LastHeight int64 `json:"last_height"` BlockMetas []*types.BlockMeta `json:"block_metas"` } +// Genesis file type ResultGenesis struct { Genesis *types.GenesisDoc `json:"genesis"` } +// Single block (with meta) type ResultBlock struct { BlockMeta *types.BlockMeta `json:"block_meta"` Block *types.Block `json:"block"` } +// Commit and Header type ResultCommit struct { // SignedHeader is header and commit, embedded so we only have // one level in the json output @@ -35,6 +39,7 @@ type ResultCommit struct { CanonicalCommit bool `json:"canonical"` } +// ABCI results from a block type ResultBlockResults struct { Height int64 `json:"height"` Results *state.ABCIResponses `json:"results"` @@ -54,6 +59,7 @@ func NewResultCommit(header *types.Header, commit *types.Commit, } } +// Info about the node's syncing state type SyncInfo struct { LatestBlockHash cmn.HexBytes `json:"latest_block_hash"` LatestAppHash cmn.HexBytes `json:"latest_app_hash"` @@ -62,18 +68,21 @@ type SyncInfo struct { Syncing bool `json:"syncing"` } +// Info about the node's validator type ValidatorInfo struct { Address cmn.HexBytes `json:"address"` PubKey crypto.PubKey `json:"pub_key"` VotingPower int64 `json:"voting_power"` } +// Node Status type ResultStatus struct { NodeInfo p2p.NodeInfo `json:"node_info"` SyncInfo SyncInfo `json:"sync_info"` ValidatorInfo ValidatorInfo `json:"validator_info"` } +// Is TxIndexing enabled func (s *ResultStatus) TxIndexEnabled() bool { if s == nil { return false @@ -87,41 +96,51 @@ func (s *ResultStatus) TxIndexEnabled() bool { return false } +// Info about peer connections type ResultNetInfo struct { Listening bool `json:"listening"` Listeners []string `json:"listeners"` Peers []Peer `json:"peers"` } +// Log from dialing seeds type ResultDialSeeds struct { Log string `json:"log"` } +// Log from dialing peers type ResultDialPeers struct { Log string `json:"log"` } +// A peer type Peer struct { p2p.NodeInfo `json:"node_info"` IsOutbound bool `json:"is_outbound"` ConnectionStatus p2p.ConnectionStatus `json:"connection_status"` } +// Validators for a height type ResultValidators struct { BlockHeight int64 `json:"block_height"` Validators []*types.Validator `json:"validators"` } +// Info about the consensus state. +// Unstable type ResultDumpConsensusState struct { RoundState json.RawMessage `json:"round_state"` PeerRoundStates []PeerRoundState `json:"peer_round_states"` } +// Raw JSON for the PeerRoundState +// Unstable type PeerRoundState struct { NodeAddress string `json:"node_address"` PeerRoundState json.RawMessage `json:"peer_round_state"` } +// CheckTx result type ResultBroadcastTx struct { Code uint32 `json:"code"` Data cmn.HexBytes `json:"data"` @@ -130,6 +149,7 @@ type ResultBroadcastTx struct { Hash cmn.HexBytes `json:"hash"` } +// CheckTx and DeliverTx results type ResultBroadcastTxCommit struct { CheckTx abci.ResponseCheckTx `json:"check_tx"` DeliverTx abci.ResponseDeliverTx `json:"deliver_tx"` @@ -137,6 +157,7 @@ type ResultBroadcastTxCommit struct { Height int64 `json:"height"` } +// Result of querying for a tx type ResultTx struct { Hash cmn.HexBytes `json:"hash"` Height int64 `json:"height"` @@ -146,30 +167,33 @@ type ResultTx struct { Proof types.TxProof `json:"proof,omitempty"` } +// List of mempool txs type ResultUnconfirmedTxs struct { N int `json:"n_txs"` Txs []types.Tx `json:"txs"` } +// Info abci msg type ResultABCIInfo struct { Response abci.ResponseInfo `json:"response"` } +// Query abci msg type ResultABCIQuery struct { Response abci.ResponseQuery `json:"response"` } -type ResultUnsafeFlushMempool struct{} - -type ResultUnsafeProfile struct{} - -type ResultSubscribe struct{} - -type ResultUnsubscribe struct{} +// empty results +type ( + ResultUnsafeFlushMempool struct{} + ResultUnsafeProfile struct{} + ResultSubscribe struct{} + ResultUnsubscribe struct{} + ResultHealth struct{} +) +// Event data from a subscription type ResultEvent struct { Query string `json:"query"` Data types.TMEventData `json:"data"` } - -type ResultHealth struct{} From d92def4b6099ef864c2453c3402f6c1ffaac5de8 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sat, 28 Apr 2018 00:06:05 -0400 Subject: [PATCH 143/143] version bump --- version/version.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/version/version.go b/version/version.go index 58e3f5e79..31479b32b 100644 --- a/version/version.go +++ b/version/version.go @@ -4,13 +4,13 @@ package version const ( Maj = "0" Min = "19" - Fix = "1" + Fix = "2" ) var ( // Version is the current version of Tendermint // Must be a string because scripts like dist.sh read this file. - Version = "0.19.1" + Version = "0.19.2-dev" // GitCommit is the current HEAD set using ldflags. GitCommit string