From ac1243260391b0db7f4acc100253a32735132079 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 27 Jun 2018 11:52:21 +0400 Subject: [PATCH 1/9] [rpc/client/http] set codec on rpc client --- rpc/client/httpclient.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rpc/client/httpclient.go b/rpc/client/httpclient.go index 1414edce3..79967bd0c 100644 --- a/rpc/client/httpclient.go +++ b/rpc/client/httpclient.go @@ -29,12 +29,13 @@ type HTTP struct { *WSEvents } -// New takes a remote endpoint in the form tcp://: +// NewHTTP takes a remote endpoint in the form tcp://: // and the websocket path (which always seems to be "/websocket") func NewHTTP(remote, wsEndpoint string) *HTTP { rc := rpcclient.NewJSONRPCClient(remote) cdc := rc.Codec() ctypes.RegisterAmino(cdc) + rc.SetCodec(cdc) return &HTTP{ rpc: rc, From 9563927bbd75b38421fd64ec1fbf216c7c74645c Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 27 Jun 2018 11:52:47 +0400 Subject: [PATCH 2/9] print only tx's hash and size when logging blocks Closes #1799 --- types/block.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/types/block.go b/types/block.go index 6adc0c4c4..43856f1f4 100644 --- a/types/block.go +++ b/types/block.go @@ -464,7 +464,7 @@ func (data *Data) StringIndented(indent string) string { txStrings[i] = fmt.Sprintf("... (%v total)", len(data.Txs)) break } - txStrings[i] = fmt.Sprintf("Tx:%v", tx) + txStrings[i] = fmt.Sprintf("%X (%d bytes)", tx.Hash(), len(tx)) } return fmt.Sprintf(`Data{ %s %v @@ -504,7 +504,7 @@ func (data *EvidenceData) StringIndented(indent string) string { } evStrings[i] = fmt.Sprintf("Evidence:%v", ev) } - return fmt.Sprintf(`Data{ + return fmt.Sprintf(`EvidenceData{ %s %v %s}#%v`, indent, strings.Join(evStrings, "\n"+indent+" "), From 2a7602c4ed4eccb42cc7e87928df785828d1fad9 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 27 Jun 2018 14:12:52 +0400 Subject: [PATCH 3/9] [tendermint init] add default consensus params to genesis --- cmd/tendermint/commands/init.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/tendermint/commands/init.go b/cmd/tendermint/commands/init.go index 45812b9ed..ad39cd20b 100644 --- a/cmd/tendermint/commands/init.go +++ b/cmd/tendermint/commands/init.go @@ -52,8 +52,9 @@ func initFilesWithConfig(config *cfg.Config) error { logger.Info("Found genesis file", "path", genFile) } else { genDoc := types.GenesisDoc{ - ChainID: cmn.Fmt("test-chain-%v", cmn.RandStr(6)), - GenesisTime: time.Now(), + ChainID: cmn.Fmt("test-chain-%v", cmn.RandStr(6)), + GenesisTime: time.Now(), + ConsensusParams: types.DefaultConsensusParams(), } genDoc.Validators = []types.GenesisValidator{{ PubKey: pv.GetPubKey(), From f760c24ff0df7cd150ca1e10a16cdc70110fac79 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 27 Jun 2018 14:14:48 +0400 Subject: [PATCH 4/9] [config] remove MaxBlockSizeTxs and MaxBlockSizeBytes in favor of consensus params --- config/config.go | 6 ------ config/toml.go | 4 ---- consensus/state.go | 2 +- docs/specification/configuration.md | 4 ---- 4 files changed, 1 insertion(+), 15 deletions(-) diff --git a/config/config.go b/config/config.go index e081056fa..f5361c22b 100644 --- a/config/config.go +++ b/config/config.go @@ -454,10 +454,6 @@ type ConsensusConfig struct { // Make progress as soon as we have all the precommits (as if TimeoutCommit = 0) SkipTimeoutCommit bool `mapstructure:"skip_timeout_commit"` - // BlockSize - MaxBlockSizeTxs int `mapstructure:"max_block_size_txs"` - MaxBlockSizeBytes int `mapstructure:"max_block_size_bytes"` - // EmptyBlocks mode and possible interval between empty blocks in seconds CreateEmptyBlocks bool `mapstructure:"create_empty_blocks"` CreateEmptyBlocksInterval int `mapstructure:"create_empty_blocks_interval"` @@ -479,8 +475,6 @@ func DefaultConsensusConfig() *ConsensusConfig { TimeoutPrecommitDelta: 500, TimeoutCommit: 1000, SkipTimeoutCommit: false, - MaxBlockSizeTxs: 10000, - MaxBlockSizeBytes: 1, // TODO CreateEmptyBlocks: true, CreateEmptyBlocksInterval: 0, PeerGossipSleepDuration: 100, diff --git a/config/toml.go b/config/toml.go index 4d892339a..b3745d3c8 100644 --- a/config/toml.go +++ b/config/toml.go @@ -217,10 +217,6 @@ timeout_commit = {{ .Consensus.TimeoutCommit }} # Make progress as soon as we have all the precommits (as if TimeoutCommit = 0) skip_timeout_commit = {{ .Consensus.SkipTimeoutCommit }} -# BlockSize -max_block_size_txs = {{ .Consensus.MaxBlockSizeTxs }} -max_block_size_bytes = {{ .Consensus.MaxBlockSizeBytes }} - # EmptyBlocks mode and possible interval between empty blocks in seconds create_empty_blocks = {{ .Consensus.CreateEmptyBlocks }} create_empty_blocks_interval = {{ .Consensus.CreateEmptyBlocksInterval }} diff --git a/consensus/state.go b/consensus/state.go index 386501c8d..78d528d09 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -932,7 +932,7 @@ func (cs *ConsensusState) createProposalBlock() (block *types.Block, blockParts } // Mempool validated transactions - txs := cs.mempool.Reap(cs.config.MaxBlockSizeTxs) + txs := cs.mempool.Reap(cs.state.ConsensusParams.BlockSize.MaxTxs) block, parts := cs.state.MakeBlock(cs.Height, txs, commit) evidence := cs.evpool.PendingEvidence() block.AddEvidence(evidence) diff --git a/docs/specification/configuration.md b/docs/specification/configuration.md index 1cf661200..9ceb8be72 100644 --- a/docs/specification/configuration.md +++ b/docs/specification/configuration.md @@ -170,10 +170,6 @@ timeout_commit = 1000 # Make progress as soon as we have all the precommits (as if TimeoutCommit = 0) skip_timeout_commit = false -# BlockSize -max_block_size_txs = 10000 -max_block_size_bytes = 1 - # EmptyBlocks mode and possible interval between empty blocks in seconds create_empty_blocks = true create_empty_blocks_interval = 0 From 297cd4cfe80b86ba7870d7d5ee64624ebd551ecb Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 27 Jun 2018 14:15:37 +0400 Subject: [PATCH 5/9] limit HTTP request body and WS read msg size to 1MB --- rpc/lib/server/handlers.go | 8 +++++++- rpc/lib/server/http_server.go | 20 ++++++++++++++++++-- types/params.go | 1 + 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/rpc/lib/server/handlers.go b/rpc/lib/server/handlers.go index 6cc03012a..dcacfb663 100644 --- a/rpc/lib/server/handlers.go +++ b/rpc/lib/server/handlers.go @@ -406,7 +406,13 @@ type wsConnection struct { // description of how to configure ping period and pong wait time. NOTE: if the // write buffer is full, pongs may be dropped, which may cause clients to // disconnect. see https://github.com/gorilla/websocket/issues/97 -func NewWSConnection(baseConn *websocket.Conn, funcMap map[string]*RPCFunc, cdc *amino.Codec, options ...func(*wsConnection)) *wsConnection { +func NewWSConnection( + baseConn *websocket.Conn, + funcMap map[string]*RPCFunc, + cdc *amino.Codec, + options ...func(*wsConnection), +) *wsConnection { + baseConn.SetReadLimit(maxBodyBytes) wsc := &wsConnection{ remoteAddr: baseConn.RemoteAddr().String(), baseConn: baseConn, diff --git a/rpc/lib/server/http_server.go b/rpc/lib/server/http_server.go index 9bdb4dffa..6223d205c 100644 --- a/rpc/lib/server/http_server.go +++ b/rpc/lib/server/http_server.go @@ -23,6 +23,12 @@ type Config struct { MaxOpenConnections int } +const ( + // maxBodyBytes controls the maximum number of bytes the + // server will read parsing the request body. + maxBodyBytes = int64(1000000) // 1MB +) + // StartHTTPServer starts an HTTP server on listenAddr with the given handler. // It wraps handler with RecoverAndLogHandler. func StartHTTPServer( @@ -53,7 +59,7 @@ func StartHTTPServer( go func() { err := http.Serve( listener, - RecoverAndLogHandler(handler, logger), + RecoverAndLogHandler(maxBytesHandler{h: handler, n: maxBodyBytes}, logger), ) logger.Error("RPC HTTP server stopped", "err", err) }() @@ -99,7 +105,7 @@ func StartHTTPAndTLSServer( go func() { err := http.ServeTLS( listener, - RecoverAndLogHandler(handler, logger), + RecoverAndLogHandler(maxBytesHandler{h: handler, n: maxBodyBytes}, logger), certFile, keyFile, ) @@ -202,3 +208,13 @@ func (w *ResponseWriterWrapper) WriteHeader(status int) { func (w *ResponseWriterWrapper) Hijack() (net.Conn, *bufio.ReadWriter, error) { return w.ResponseWriter.(http.Hijacker).Hijack() } + +type maxBytesHandler struct { + h http.Handler + n int64 +} + +func (h maxBytesHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + r.Body = http.MaxBytesReader(w, r.Body, h.n) + h.h.ServeHTTP(w, r) +} diff --git a/types/params.go b/types/params.go index d068342c6..6cbac47a6 100644 --- a/types/params.go +++ b/types/params.go @@ -7,6 +7,7 @@ import ( ) const ( + // MaxBlockSizeBytes is the maximum permitted size of the blocks. MaxBlockSizeBytes = 104857600 // 100MB ) From ab04201c3d658d4eb0689b2c6d48d95b672560a3 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 27 Jun 2018 18:07:26 +0400 Subject: [PATCH 6/9] fix empty packet size calculation Fixes #1762#issuecomment-398940107 --- mempool/reactor.go | 2 +- p2p/conn/connection.go | 15 +++++++-------- p2p/conn/connection_test.go | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/mempool/reactor.go b/mempool/reactor.go index fab9480ba..066118036 100644 --- a/mempool/reactor.go +++ b/mempool/reactor.go @@ -5,8 +5,8 @@ import ( "reflect" "time" - abci "github.com/tendermint/tendermint/abci/types" amino "github.com/tendermint/go-amino" + abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tmlibs/clist" "github.com/tendermint/tmlibs/log" diff --git a/p2p/conn/connection.go b/p2p/conn/connection.go index c56507296..1392c8051 100644 --- a/p2p/conn/connection.go +++ b/p2p/conn/connection.go @@ -156,7 +156,7 @@ func NewMConnectionWithConfig(conn net.Conn, chDescs []*ChannelDescriptor, onRec onReceive: onReceive, onError: onError, config: config, - emptyPacketMsgSize: emptyPacketMsgSize(), + emptyPacketMsgSize: emptyPacketMsgSize(config.MaxPacketMsgSize), } // Create channels @@ -631,7 +631,7 @@ func newChannel(conn *MConnection, desc ChannelDescriptor) *Channel { desc: desc, sendQueue: make(chan []byte, desc.SendQueueCapacity), recving: make([]byte, 0, desc.RecvBufferCapacity), - maxPacketMsgPayloadSize: conn.config.MaxPacketMsgSize, + maxPacketMsgPayloadSize: conn.config.MaxPacketMsgSize - conn.emptyPacketMsgSize, } } @@ -694,7 +694,7 @@ func (ch *Channel) isSendPending() bool { func (ch *Channel) nextPacketMsg() PacketMsg { packet := PacketMsg{} packet.ChannelID = byte(ch.desc.ID) - maxSize := ch.maxPacketMsgPayloadSize - ch.conn.emptyPacketMsgSize + maxSize := ch.maxPacketMsgPayloadSize packet.Bytes = ch.sending[:cmn.MinInt(maxSize, len(ch.sending))] if len(ch.sending) <= maxSize { packet.EOF = byte(0x01) @@ -788,17 +788,16 @@ func (mp PacketMsg) String() string { // - EOF field key + byte = 2 bytes // - Bytes field key = 1 bytes // - Uvarint length of MustMarshalBinary(bytes) = 1 or 2 bytes -// - Struct terminator = 1 byte -// = up to 14 bytes overhead for the packet. +// = up to 13 bytes overhead for the packet. -func emptyPacketMsgSize() int { +func emptyPacketMsgSize(maxPayloadSize int) int { emptyPacketMsgSize := len(cdc.MustMarshalBinary(PacketMsg{ ChannelID: 0x01, EOF: 1, - Bytes: make([]byte, 1), + Bytes: make([]byte, maxPayloadSize), })) // -1 byte of data // +1 byte because uvarint length of MustMarshalBinary(bytes) will be 2 bytes for big packets // +1 byte because uvarint length of MustMarshalBinary(packet) will be 2 bytes for big packets - return emptyPacketMsgSize - 1 + 1 + 1 + return emptyPacketMsgSize - maxPayloadSize + 1 + 1 } diff --git a/p2p/conn/connection_test.go b/p2p/conn/connection_test.go index 34b37ab8a..6e4c1a090 100644 --- a/p2p/conn/connection_test.go +++ b/p2p/conn/connection_test.go @@ -426,7 +426,7 @@ func TestMConnectionReadErrorLongMessage(t *testing.T) { var packet = PacketMsg{ ChannelID: 0x01, EOF: 1, - Bytes: make([]byte, mconnClient.config.MaxPacketMsgSize-emptyPacketMsgSize()), + Bytes: make([]byte, mconnClient.config.MaxPacketMsgSize-emptyPacketMsgSize(mconnClient.config.MaxPacketMsgSize)), } _, err = cdc.MarshalBinaryWriter(buf, packet) assert.Nil(t, err) From 61c5791fa3aa0d16451285f705f48526d26ce7be Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Fri, 29 Jun 2018 12:17:26 +0400 Subject: [PATCH 7/9] revert back to Jae's original payload size limit except now we calculate the max size using the maxPacketMsgSize() function, which frees developers from having to know amino encoding details. plus, 10 additional bytes are added to leave the room for amino upgrades (both making it more efficient / less efficient) --- benchmarks/codec_test.go | 2 +- config/config.go | 39 +++++++------- config/toml.go | 5 +- crypto/merkle/doc.go | 2 +- crypto/merkle/simple_tree.go | 8 +-- crypto/merkle/simple_tree_test.go | 2 +- docs/running-in-production.md | 2 +- docs/specification/configuration.md | 2 +- evidence/wire.go | 2 +- p2p/conn/connection.go | 81 +++++++++++++---------------- p2p/conn/connection_test.go | 4 +- p2p/switch.go | 2 +- privval/priv_validator.go | 2 +- privval/priv_validator_test.go | 2 +- rpc/core/pipe.go | 2 +- 15 files changed, 73 insertions(+), 84 deletions(-) diff --git a/benchmarks/codec_test.go b/benchmarks/codec_test.go index ee61cc9a4..53cbf632c 100644 --- a/benchmarks/codec_test.go +++ b/benchmarks/codec_test.go @@ -6,8 +6,8 @@ import ( "github.com/tendermint/go-amino" - "github.com/tendermint/tendermint/crypto" proto "github.com/tendermint/tendermint/benchmarks/proto" + "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/p2p" ctypes "github.com/tendermint/tendermint/rpc/core/types" ) diff --git a/config/config.go b/config/config.go index f5361c22b..e01819305 100644 --- a/config/config.go +++ b/config/config.go @@ -299,9 +299,8 @@ type P2PConfig struct { // Time to wait before flushing messages out on the connection, in ms FlushThrottleTimeout int `mapstructure:"flush_throttle_timeout"` - // Maximum size of a message packet, in bytes - // Includes a header, which is ~13 bytes - MaxPacketMsgSize int `mapstructure:"max_packet_msg_size"` + // Maximum size of a message packet payload, in bytes + MaxPacketMsgPayloadSize int `mapstructure:"max_packet_msg_payload_size"` // Rate at which packets can be sent, in bytes/second SendRate int64 `mapstructure:"send_rate"` @@ -340,23 +339,23 @@ type P2PConfig struct { // DefaultP2PConfig returns a default configuration for the peer-to-peer layer func DefaultP2PConfig() *P2PConfig { return &P2PConfig{ - ListenAddress: "tcp://0.0.0.0:26656", - UPNP: false, - AddrBook: defaultAddrBookPath, - AddrBookStrict: true, - MaxNumPeers: 50, - FlushThrottleTimeout: 100, - MaxPacketMsgSize: 1024, // 1 kB - SendRate: 512000, // 500 kB/s - RecvRate: 512000, // 500 kB/s - PexReactor: true, - SeedMode: false, - AllowDuplicateIP: true, // so non-breaking yet - HandshakeTimeout: 20 * time.Second, - DialTimeout: 3 * time.Second, - TestDialFail: false, - TestFuzz: false, - TestFuzzConfig: DefaultFuzzConnConfig(), + ListenAddress: "tcp://0.0.0.0:26656", + UPNP: false, + AddrBook: defaultAddrBookPath, + AddrBookStrict: true, + MaxNumPeers: 50, + FlushThrottleTimeout: 100, + MaxPacketMsgPayloadSize: 1024, // 1 kB + SendRate: 512000, // 500 kB/s + RecvRate: 512000, // 500 kB/s + PexReactor: true, + SeedMode: false, + AllowDuplicateIP: true, // so non-breaking yet + HandshakeTimeout: 20 * time.Second, + DialTimeout: 3 * time.Second, + TestDialFail: false, + TestFuzz: false, + TestFuzzConfig: DefaultFuzzConnConfig(), } } diff --git a/config/toml.go b/config/toml.go index b3745d3c8..4569291d4 100644 --- a/config/toml.go +++ b/config/toml.go @@ -164,9 +164,8 @@ flush_throttle_timeout = {{ .P2P.FlushThrottleTimeout }} # Maximum number of peers to connect to max_num_peers = {{ .P2P.MaxNumPeers }} -# Maximum size of a message packet, in bytes -# Includes a header, which is ~13 bytes -max_packet_msg_size = {{ .P2P.MaxPacketMsgSize }} +# Maximum size of a message packet payload, in bytes +max_packet_msg_payload_size = {{ .P2P.MaxPacketMsgPayloadSize }} # Rate at which packets can be sent, in bytes/second send_rate = {{ .P2P.SendRate }} diff --git a/crypto/merkle/doc.go b/crypto/merkle/doc.go index da65dd858..865c30217 100644 --- a/crypto/merkle/doc.go +++ b/crypto/merkle/doc.go @@ -28,4 +28,4 @@ https://bitcointalk.org/?topic=102395 TODO(ismail): add 2nd pre-image protection or clarify further on how we use this and why this secure. */ -package merkle \ No newline at end of file +package merkle diff --git a/crypto/merkle/simple_tree.go b/crypto/merkle/simple_tree.go index 35a6eaa7c..46a075909 100644 --- a/crypto/merkle/simple_tree.go +++ b/crypto/merkle/simple_tree.go @@ -9,12 +9,12 @@ func SimpleHashFromTwoHashes(left, right []byte) []byte { var hasher = tmhash.New() err := encodeByteSlice(hasher, left) if err != nil { - panic(err) - } + panic(err) + } err = encodeByteSlice(hasher, right) if err != nil { - panic(err) - } + panic(err) + } return hasher.Sum(nil) } diff --git a/crypto/merkle/simple_tree_test.go b/crypto/merkle/simple_tree_test.go index a721bccea..6eef93623 100644 --- a/crypto/merkle/simple_tree_test.go +++ b/crypto/merkle/simple_tree_test.go @@ -6,8 +6,8 @@ import ( cmn "github.com/tendermint/tmlibs/common" . "github.com/tendermint/tmlibs/test" - "testing" "github.com/tendermint/tendermint/crypto/tmhash" + "testing" ) type testItem []byte diff --git a/docs/running-in-production.md b/docs/running-in-production.md index 3ceded499..225a97853 100644 --- a/docs/running-in-production.md +++ b/docs/running-in-production.md @@ -21,7 +21,7 @@ to prevent Denial-of-service attacks. You can read more about it ### P2P The core of the Tendermint peer-to-peer system is `MConnection`. Each -connection has `MaxPacketMsgSize`, which is the maximum packet +connection has `MaxPacketMsgPayloadSize`, which is the maximum packet size and bounded send & receive queues. One can impose restrictions on send & receive rate per connection (`SendRate`, `RecvRate`). diff --git a/docs/specification/configuration.md b/docs/specification/configuration.md index 9ceb8be72..214757b9f 100644 --- a/docs/specification/configuration.md +++ b/docs/specification/configuration.md @@ -119,7 +119,7 @@ flush_throttle_timeout = 100 max_num_peers = 50 # Maximum size of a message packet payload, in bytes -max_msg_packet_payload_size = 1024 +max_packet_msg_payload_size = 1024 # Rate at which packets can be sent, in bytes/second send_rate = 512000 diff --git a/evidence/wire.go b/evidence/wire.go index d4db37c54..fb3a177cc 100644 --- a/evidence/wire.go +++ b/evidence/wire.go @@ -2,8 +2,8 @@ package evidence import ( "github.com/tendermint/go-amino" - "github.com/tendermint/tendermint/types" "github.com/tendermint/tendermint/crypto" + "github.com/tendermint/tendermint/types" ) var cdc = amino.NewCodec() diff --git a/p2p/conn/connection.go b/p2p/conn/connection.go index 1392c8051..b19a1ca19 100644 --- a/p2p/conn/connection.go +++ b/p2p/conn/connection.go @@ -18,7 +18,7 @@ import ( ) const ( - defaultMaxPacketMsgSize = 1024 + defaultMaxPacketMsgPayloadSize = 1024 numBatchPacketMsgs = 10 minReadBufferSize = 1024 @@ -96,7 +96,7 @@ type MConnection struct { created time.Time // time of creation - emptyPacketMsgSize int + _maxPacketMsgSize int } // MConnConfig is a MConnection configuration. @@ -105,7 +105,7 @@ type MConnConfig struct { RecvRate int64 `mapstructure:"recv_rate"` // Maximum payload size - MaxPacketMsgSize int `mapstructure:"max_packet_msg_size"` + MaxPacketMsgPayloadSize int `mapstructure:"max_packet_msg_payload_size"` // Interval to flush writes (throttled) FlushThrottle time.Duration `mapstructure:"flush_throttle"` @@ -120,12 +120,12 @@ type MConnConfig struct { // DefaultMConnConfig returns the default config. func DefaultMConnConfig() MConnConfig { return MConnConfig{ - SendRate: defaultSendRate, - RecvRate: defaultRecvRate, - MaxPacketMsgSize: defaultMaxPacketMsgSize, - FlushThrottle: defaultFlushThrottle, - PingInterval: defaultPingInterval, - PongTimeout: defaultPongTimeout, + SendRate: defaultSendRate, + RecvRate: defaultRecvRate, + MaxPacketMsgPayloadSize: defaultMaxPacketMsgPayloadSize, + FlushThrottle: defaultFlushThrottle, + PingInterval: defaultPingInterval, + PongTimeout: defaultPongTimeout, } } @@ -146,17 +146,16 @@ func NewMConnectionWithConfig(conn net.Conn, chDescs []*ChannelDescriptor, onRec } mconn := &MConnection{ - conn: conn, - bufConnReader: bufio.NewReaderSize(conn, minReadBufferSize), - bufConnWriter: bufio.NewWriterSize(conn, minWriteBufferSize), - sendMonitor: flow.New(0, 0), - recvMonitor: flow.New(0, 0), - send: make(chan struct{}, 1), - pong: make(chan struct{}, 1), - onReceive: onReceive, - onError: onError, - config: config, - emptyPacketMsgSize: emptyPacketMsgSize(config.MaxPacketMsgSize), + conn: conn, + bufConnReader: bufio.NewReaderSize(conn, minReadBufferSize), + bufConnWriter: bufio.NewWriterSize(conn, minWriteBufferSize), + sendMonitor: flow.New(0, 0), + recvMonitor: flow.New(0, 0), + send: make(chan struct{}, 1), + pong: make(chan struct{}, 1), + onReceive: onReceive, + onError: onError, + config: config, } // Create channels @@ -173,6 +172,9 @@ func NewMConnectionWithConfig(conn net.Conn, chDescs []*ChannelDescriptor, onRec mconn.BaseService = *cmn.NewBaseService(nil, "MConnection", mconn) + // maxPacketMsgSize() is a bit heavy, so call just once + mconn._maxPacketMsgSize = mconn.maxPacketMsgSize() + return mconn } @@ -397,7 +399,7 @@ func (c *MConnection) sendSomePacketMsgs() bool { // Block until .sendMonitor says we can write. // Once we're ready we send more than we asked for, // but amortized it should even out. - c.sendMonitor.Limit(c.config.MaxPacketMsgSize, atomic.LoadInt64(&c.config.SendRate), true) + c.sendMonitor.Limit(c._maxPacketMsgSize, atomic.LoadInt64(&c.config.SendRate), true) // Now send some PacketMsgs. for i := 0; i < numBatchPacketMsgs; i++ { @@ -455,7 +457,7 @@ func (c *MConnection) recvRoutine() { FOR_LOOP: for { // Block until .recvMonitor says we can read. - c.recvMonitor.Limit(c.config.MaxPacketMsgSize, atomic.LoadInt64(&c.config.RecvRate), true) + c.recvMonitor.Limit(c._maxPacketMsgSize, atomic.LoadInt64(&c.config.RecvRate), true) // Peek into bufConnReader for debugging /* @@ -475,7 +477,7 @@ FOR_LOOP: var packet Packet var _n int64 var err error - _n, err = cdc.UnmarshalBinaryReader(c.bufConnReader, &packet, int64(c.config.MaxPacketMsgSize)) + _n, err = cdc.UnmarshalBinaryReader(c.bufConnReader, &packet, int64(c._maxPacketMsgSize)) c.recvMonitor.Update(int(_n)) if err != nil { if c.IsRunning() { @@ -548,6 +550,16 @@ func (c *MConnection) stopPongTimer() { } } +// maxPacketMsgSize returns a maximum size of PacketMsg, including the overhead +// of amino encoding. +func (c *MConnection) maxPacketMsgSize() int { + return len(cdc.MustMarshalBinary(PacketMsg{ + ChannelID: 0x01, + EOF: 1, + Bytes: make([]byte, c.config.MaxPacketMsgPayloadSize), + })) + 10 // leave room for changes in amino +} + type ConnectionStatus struct { Duration time.Duration SendMonitor flow.Status @@ -631,7 +643,7 @@ func newChannel(conn *MConnection, desc ChannelDescriptor) *Channel { desc: desc, sendQueue: make(chan []byte, desc.SendQueueCapacity), recving: make([]byte, 0, desc.RecvBufferCapacity), - maxPacketMsgPayloadSize: conn.config.MaxPacketMsgSize - conn.emptyPacketMsgSize, + maxPacketMsgPayloadSize: conn.config.MaxPacketMsgPayloadSize, } } @@ -780,24 +792,3 @@ type PacketMsg struct { func (mp PacketMsg) String() string { return fmt.Sprintf("PacketMsg{%X:%X T:%X}", mp.ChannelID, mp.Bytes, mp.EOF) } - -// - Uvarint length of MustMarshalBinary(packet) = 1 or 2 bytes -// (as long as it's less than 16,384 bytes) -// - Prefix bytes = 4 bytes -// - ChannelID field key + byte = 2 bytes -// - EOF field key + byte = 2 bytes -// - Bytes field key = 1 bytes -// - Uvarint length of MustMarshalBinary(bytes) = 1 or 2 bytes -// = up to 13 bytes overhead for the packet. - -func emptyPacketMsgSize(maxPayloadSize int) int { - emptyPacketMsgSize := len(cdc.MustMarshalBinary(PacketMsg{ - ChannelID: 0x01, - EOF: 1, - Bytes: make([]byte, maxPayloadSize), - })) - // -1 byte of data - // +1 byte because uvarint length of MustMarshalBinary(bytes) will be 2 bytes for big packets - // +1 byte because uvarint length of MustMarshalBinary(packet) will be 2 bytes for big packets - return emptyPacketMsgSize - maxPayloadSize + 1 + 1 -} diff --git a/p2p/conn/connection_test.go b/p2p/conn/connection_test.go index 6e4c1a090..8006b37a8 100644 --- a/p2p/conn/connection_test.go +++ b/p2p/conn/connection_test.go @@ -426,7 +426,7 @@ func TestMConnectionReadErrorLongMessage(t *testing.T) { var packet = PacketMsg{ ChannelID: 0x01, EOF: 1, - Bytes: make([]byte, mconnClient.config.MaxPacketMsgSize-emptyPacketMsgSize(mconnClient.config.MaxPacketMsgSize)), + Bytes: make([]byte, mconnClient.config.MaxPacketMsgPayloadSize), } _, err = cdc.MarshalBinaryWriter(buf, packet) assert.Nil(t, err) @@ -440,7 +440,7 @@ func TestMConnectionReadErrorLongMessage(t *testing.T) { packet = PacketMsg{ ChannelID: 0x01, EOF: 1, - Bytes: make([]byte, mconnClient.config.MaxPacketMsgSize+1), + Bytes: make([]byte, mconnClient.config.MaxPacketMsgPayloadSize+100), } _, err = cdc.MarshalBinaryWriter(buf, packet) assert.Nil(t, err) diff --git a/p2p/switch.go b/p2p/switch.go index ae322b54f..bf5f9747f 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -100,7 +100,7 @@ func NewSwitch(cfg *config.P2PConfig, options ...SwitchOption) *Switch { mConfig.FlushThrottle = time.Duration(cfg.FlushThrottleTimeout) * time.Millisecond mConfig.SendRate = cfg.SendRate mConfig.RecvRate = cfg.RecvRate - mConfig.MaxPacketMsgSize = cfg.MaxPacketMsgSize + mConfig.MaxPacketMsgPayloadSize = cfg.MaxPacketMsgPayloadSize sw.mConfig = mConfig diff --git a/privval/priv_validator.go b/privval/priv_validator.go index 9f02482ab..8a54b5ccf 100644 --- a/privval/priv_validator.go +++ b/privval/priv_validator.go @@ -287,7 +287,7 @@ func (pv *FilePV) saveSigned(height int64, round int, step int8, func (pv *FilePV) SignHeartbeat(chainID string, heartbeat *types.Heartbeat) error { pv.mtx.Lock() defer pv.mtx.Unlock() - sig, err:= pv.PrivKey.Sign(heartbeat.SignBytes(chainID)) + sig, err := pv.PrivKey.Sign(heartbeat.SignBytes(chainID)) if err != nil { return err } diff --git a/privval/priv_validator_test.go b/privval/priv_validator_test.go index 4fc8f97fc..345b51438 100644 --- a/privval/priv_validator_test.go +++ b/privval/priv_validator_test.go @@ -9,8 +9,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/tendermint/tendermint/types" "github.com/tendermint/tendermint/crypto" + "github.com/tendermint/tendermint/types" cmn "github.com/tendermint/tmlibs/common" ) diff --git a/rpc/core/pipe.go b/rpc/core/pipe.go index 9fcb75e19..bf32c9c66 100644 --- a/rpc/core/pipe.go +++ b/rpc/core/pipe.go @@ -3,8 +3,8 @@ package core import ( "time" - crypto "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/consensus" + crypto "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/proxy" sm "github.com/tendermint/tendermint/state" From 399e2fbdac5bfda42b0f34305046ca81410a6dbf Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Fri, 29 Jun 2018 12:37:06 +0400 Subject: [PATCH 8/9] update changelog --- CHANGELOG.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 24ec19d79..00276a0bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,22 +2,22 @@ ## TBD -IMPROVEMENTS: - - [crypto] Make public key size into public constants - BUG FIXES: -- [rpc] limited number of HTTP/WebSocket connections +- [rpc] Limit maximum number of HTTP/WebSocket connections (`rpc.max_open_connections`) and gRPC connections - (`rpc.grpc_max_open_connections`). Check out [Running In - Production](https://tendermint.readthedocs.io/en/master/running-in-production.html) - guide if you want to increase them. + (`rpc.grpc_max_open_connections`). Check out "Running In Production" guide if + you want to increase them. +- [rpc] Limit maximum request body size to 1MB (header is limited to 1MB). BREAKING CHANGES: - [config] Rename `skip_upnp` to `upnp`, and turn it off by default. +- [config] Rename `max_packet_msg_size` to `max_packet_msg_payload_size`. IMPROVEMENT -- [rpc/client] Supports https and wss now. +- [crypto] Make public key size into public constants - [p2p] Add IPv6 support to peering. +- [rpc/client] Supports https and wss now. +- [stdout] Txs inside blocks are now logged as hashes (plus size in bytes). ## 0.21.0 From 2d98899b9b57817b892ff1c9f2643e0b2850cadc Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Sun, 1 Jul 2018 09:44:12 +0400 Subject: [PATCH 9/9] set MaxTxs to 10000 (the same that was used in the config before) --- types/params.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/types/params.go b/types/params.go index 6cbac47a6..0654d07b9 100644 --- a/types/params.go +++ b/types/params.go @@ -57,7 +57,7 @@ func DefaultConsensusParams() *ConsensusParams { func DefaultBlockSize() BlockSize { return BlockSize{ MaxBytes: 22020096, // 21MB - MaxTxs: 100000, + MaxTxs: 10000, MaxGas: -1, } }