From f0c2c9ba5abdd7e1fd5287e96eba14b69bd6435a Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 1 Apr 2019 19:45:57 -0400 Subject: [PATCH 1/5] Fixes tendermint/tendermint#3439 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * make sure we create valid private keys: - genPrivKey samples and rejects invalid fieldelems (like libsecp256k1) - GenPrivKeySecp256k1 uses `(sha(secret) mod (n − 1)) + 1` - fix typo, rename test file: s/secpk256k1/secp256k1/ * Update crypto/secp256k1/secp256k1.go --- crypto/secp256k1/secp256k1.go | 52 +++++++++++++++---- crypto/secp256k1/secp256k1_cgo_test.go | 39 ++++++++++++++ crypto/secp256k1/secp256k1_internal_test.go | 45 ++++++++++++++++ crypto/secp256k1/secp256k1_nocgo_test.go | 1 - .../{secpk256k1_test.go => secp256k1_test.go} | 26 ++++++++++ 5 files changed, 151 insertions(+), 12 deletions(-) create mode 100644 crypto/secp256k1/secp256k1_cgo_test.go create mode 100644 crypto/secp256k1/secp256k1_internal_test.go rename crypto/secp256k1/{secpk256k1_test.go => secp256k1_test.go} (74%) diff --git a/crypto/secp256k1/secp256k1.go b/crypto/secp256k1/secp256k1.go index 78857c45c..fc64a0b0f 100644 --- a/crypto/secp256k1/secp256k1.go +++ b/crypto/secp256k1/secp256k1.go @@ -6,6 +6,7 @@ import ( "crypto/subtle" "fmt" "io" + "math/big" "golang.org/x/crypto/ripemd160" @@ -65,32 +66,61 @@ func (privKey PrivKeySecp256k1) Equals(other crypto.PrivKey) bool { } // GenPrivKey generates a new ECDSA private key on curve secp256k1 private key. -// It uses OS randomness in conjunction with the current global random seed -// in tendermint/libs/common to generate the private key. +// It uses OS randomness to generate the private key. func GenPrivKey() PrivKeySecp256k1 { return genPrivKey(crypto.CReader()) } // genPrivKey generates a new secp256k1 private key using the provided reader. func genPrivKey(rand io.Reader) PrivKeySecp256k1 { - privKeyBytes := [32]byte{} - _, err := io.ReadFull(rand, privKeyBytes[:]) - if err != nil { - panic(err) + var privKeyBytes [32]byte + d := new(big.Int) + for { + privKeyBytes = [32]byte{} + _, err := io.ReadFull(rand, privKeyBytes[:]) + if err != nil { + panic(err) + } + + d.SetBytes(privKeyBytes[:]) + // break if we found a valid point (i.e. > 0 and < N == curverOrder) + isValidFieldElement := 0 < d.Sign() && d.Cmp(secp256k1.S256().N) < 0 + if isValidFieldElement { + break + } } - // crypto.CRandBytes is guaranteed to be 32 bytes long, so it can be - // casted to PrivKeySecp256k1. + return PrivKeySecp256k1(privKeyBytes) } +var one = new(big.Int).SetInt64(1) + // GenPrivKeySecp256k1 hashes the secret with SHA2, and uses // that 32 byte output to create the private key. +// +// It makes sure the private key is a valid field element by setting: +// +// c = sha256(secret) +// k = (c mod (n − 1)) + 1, where n = curve order. +// // NOTE: secret should be the output of a KDF like bcrypt, // if it's derived from user input. func GenPrivKeySecp256k1(secret []byte) PrivKeySecp256k1 { - privKey32 := sha256.Sum256(secret) - // sha256.Sum256() is guaranteed to be 32 bytes long, so it can be - // casted to PrivKeySecp256k1. + secHash := sha256.Sum256(secret) + // to guarantee that we have a valid field element, we use the approach of: + // "Suite B Implementer’s Guide to FIPS 186-3", A.2.1 + // https://apps.nsa.gov/iaarchive/library/ia-guidance/ia-solutions-for-classified/algorithm-guidance/suite-b-implementers-guide-to-fips-186-3-ecdsa.cfm + // see also https://github.com/golang/go/blob/0380c9ad38843d523d9c9804fe300cb7edd7cd3c/src/crypto/ecdsa/ecdsa.go#L89-L101 + fe := new(big.Int).SetBytes(secHash[:]) + n := new(big.Int).Sub(secp256k1.S256().N, one) + fe.Mod(fe, n) + fe.Add(fe, one) + + feB := fe.Bytes() + var privKey32 [32]byte + // copy feB over to fixed 32 byte privKey32 and pad (if necessary) + copy(privKey32[32-len(feB):32], feB) + return PrivKeySecp256k1(privKey32) } diff --git a/crypto/secp256k1/secp256k1_cgo_test.go b/crypto/secp256k1/secp256k1_cgo_test.go new file mode 100644 index 000000000..edb207b53 --- /dev/null +++ b/crypto/secp256k1/secp256k1_cgo_test.go @@ -0,0 +1,39 @@ +// +build libsecp256k1 + +package secp256k1 + +import ( + "github.com/magiconair/properties/assert" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestPrivKeySecp256k1SignVerify(t *testing.T) { + msg := []byte("A.1.2 ECC Key Pair Generation by Testing Candidates") + priv := GenPrivKey() + tests := []struct { + name string + privKey PrivKeySecp256k1 + wantSignErr bool + wantVerifyPasses bool + }{ + {name: "valid sign-verify round", privKey: priv, wantSignErr: false, wantVerifyPasses: true}, + {name: "invalid private key", privKey: [32]byte{}, wantSignErr: true, wantVerifyPasses: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.privKey.Sign(msg) + if tt.wantSignErr { + require.Error(t, err) + t.Logf("Got error: %s", err) + return + } + require.NoError(t, err) + require.NotNil(t, got) + + pub := tt.privKey.PubKey() + assert.Equal(t, tt.wantVerifyPasses, pub.VerifyBytes(msg, got)) + }) + } +} diff --git a/crypto/secp256k1/secp256k1_internal_test.go b/crypto/secp256k1/secp256k1_internal_test.go new file mode 100644 index 000000000..305f12020 --- /dev/null +++ b/crypto/secp256k1/secp256k1_internal_test.go @@ -0,0 +1,45 @@ +package secp256k1 + +import ( + "bytes" + "math/big" + "testing" + + "github.com/stretchr/testify/require" + + underlyingSecp256k1 "github.com/btcsuite/btcd/btcec" +) + +func Test_genPrivKey(t *testing.T) { + + empty := make([]byte, 32) + oneB := big.NewInt(1).Bytes() + onePadded := make([]byte, 32) + copy(onePadded[32-len(oneB):32], oneB) + t.Logf("one padded: %v, len=%v", onePadded, len(onePadded)) + + validOne := append(empty, onePadded...) + tests := []struct { + name string + notSoRand []byte + shouldPanic bool + }{ + {"empty bytes (panics because 1st 32 bytes are zero and 0 is not a valid field element)", empty, true}, + {"curve order: N", underlyingSecp256k1.S256().N.Bytes(), true}, + {"valid because 0 < 1 < N", validOne, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.shouldPanic { + require.Panics(t, func() { + genPrivKey(bytes.NewReader(tt.notSoRand)) + }) + return + } + got := genPrivKey(bytes.NewReader(tt.notSoRand)) + fe := new(big.Int).SetBytes(got[:]) + require.True(t, fe.Cmp(underlyingSecp256k1.S256().N) < 0) + require.True(t, fe.Sign() > 0) + }) + } +} diff --git a/crypto/secp256k1/secp256k1_nocgo_test.go b/crypto/secp256k1/secp256k1_nocgo_test.go index a06a0e3d1..17cb75815 100644 --- a/crypto/secp256k1/secp256k1_nocgo_test.go +++ b/crypto/secp256k1/secp256k1_nocgo_test.go @@ -6,7 +6,6 @@ import ( "testing" secp256k1 "github.com/btcsuite/btcd/btcec" - "github.com/stretchr/testify/require" ) diff --git a/crypto/secp256k1/secpk256k1_test.go b/crypto/secp256k1/secp256k1_test.go similarity index 74% rename from crypto/secp256k1/secpk256k1_test.go rename to crypto/secp256k1/secp256k1_test.go index 0f0b5adce..2488b5399 100644 --- a/crypto/secp256k1/secpk256k1_test.go +++ b/crypto/secp256k1/secp256k1_test.go @@ -2,6 +2,7 @@ package secp256k1_test import ( "encoding/hex" + "math/big" "testing" "github.com/btcsuite/btcutil/base58" @@ -84,3 +85,28 @@ func TestSecp256k1LoadPrivkeyAndSerializeIsIdentity(t *testing.T) { require.Equal(t, privKeyBytes[:], serializedBytes) } } + +func TestGenPrivKeySecp256k1(t *testing.T) { + // curve oder N + N := underlyingSecp256k1.S256().N + tests := []struct { + name string + secret []byte + }{ + {"empty secret", []byte{}}, + {"some long secret", []byte("We live in a society exquisitely dependent on science and technology, in which hardly anyone knows anything about science and technology.")}, + {"another seed used in cosmos tests #1", []byte{0}}, + {"another seed used in cosmos tests #2", []byte("mySecret")}, + {"another seed used in cosmos tests #3", []byte("")}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotPrivKey := secp256k1.GenPrivKeySecp256k1(tt.secret) + require.NotNil(t, gotPrivKey) + // interpret as a big.Int and make sure it is a valid field element: + fe := new(big.Int).SetBytes(gotPrivKey[:]) + require.True(t, fe.Cmp(N) < 0) + require.True(t, fe.Sign() > 0) + }) + } +} From df2f25fe823f13dff9fec46e977a5ea6dae94e4d Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 1 Apr 2019 19:59:57 -0400 Subject: [PATCH 2/5] Fixes tendermint/tendermint#3522 * OriginalAddr -> SocketAddr OriginalAddr records the originally dialed address for outbound peers, rather than the peer's self reported address. For inbound peers, it was nil. Here, we rename it to SocketAddr and for inbound peers, set it to the RemoteAddr of the connection. * use SocketAddr Numerous places in the code call peer.NodeInfo().NetAddress(). However, this call to NetAddress() may perform a DNS lookup if the reported NodeInfo.ListenAddr includes a name. Failure of this lookup returns a nil address, which can lead to panics in the code. Instead, call peer.SocketAddr() to return the static address of the connection. * remove nodeInfo.NetAddress() Expose `transport.NetAddress()`, a static result determined when the transport is created. Removing NetAddress() from the nodeInfo prevents accidental DNS lookups. * fixes from review * linter * fixes from review --- node/node.go | 2 +- p2p/mock/peer.go | 68 +++++++++++++++++++++++++++++++++++++ p2p/node_info.go | 10 ++---- p2p/peer.go | 27 +++++++-------- p2p/peer_set_test.go | 2 +- p2p/peer_test.go | 14 ++++---- p2p/pex/pex_reactor.go | 4 +-- p2p/pex/pex_reactor_test.go | 27 ++++++++------- p2p/switch.go | 22 ++++++------ p2p/switch_test.go | 8 ++--- p2p/test_util.go | 19 ++++++----- p2p/transport.go | 25 +++++++++++--- p2p/transport_test.go | 62 +++++++++++++++++---------------- rpc/core/consensus.go | 2 +- 14 files changed, 187 insertions(+), 105 deletions(-) create mode 100644 p2p/mock/peer.go diff --git a/node/node.go b/node/node.go index e5ddd09cc..f431d2422 100644 --- a/node/node.go +++ b/node/node.go @@ -488,7 +488,7 @@ func NewNode(config *cfg.Config, addrBook := pex.NewAddrBook(config.P2P.AddrBookFile(), config.P2P.AddrBookStrict) // Add ourselves to addrbook to prevent dialing ourselves - addrBook.AddOurAddress(nodeInfo.NetAddress()) + addrBook.AddOurAddress(sw.NetAddress()) addrBook.SetLogger(p2pLogger.With("book", config.P2P.AddrBookFile())) if config.P2P.PexReactor { diff --git a/p2p/mock/peer.go b/p2p/mock/peer.go new file mode 100644 index 000000000..bdcf012de --- /dev/null +++ b/p2p/mock/peer.go @@ -0,0 +1,68 @@ +package mock + +import ( + "net" + + "github.com/tendermint/tendermint/crypto/ed25519" + cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/p2p" + "github.com/tendermint/tendermint/p2p/conn" +) + +type Peer struct { + *cmn.BaseService + ip net.IP + id p2p.ID + addr *p2p.NetAddress + kv map[string]interface{} + Outbound, Persistent bool +} + +// NewPeer creates and starts a new mock peer. If the ip +// is nil, random routable address is used. +func NewPeer(ip net.IP) *Peer { + var netAddr *p2p.NetAddress + if ip == nil { + _, netAddr = p2p.CreateRoutableAddr() + } else { + netAddr = p2p.NewNetAddressIPPort(ip, 26656) + } + nodeKey := p2p.NodeKey{PrivKey: ed25519.GenPrivKey()} + netAddr.ID = nodeKey.ID() + mp := &Peer{ + ip: ip, + id: nodeKey.ID(), + addr: netAddr, + kv: make(map[string]interface{}), + } + mp.BaseService = cmn.NewBaseService(nil, "MockPeer", mp) + mp.Start() + return mp +} + +func (mp *Peer) FlushStop() { mp.Stop() } +func (mp *Peer) TrySend(chID byte, msgBytes []byte) bool { return true } +func (mp *Peer) Send(chID byte, msgBytes []byte) bool { return true } +func (mp *Peer) NodeInfo() p2p.NodeInfo { + return p2p.DefaultNodeInfo{ + ID_: mp.addr.ID, + ListenAddr: mp.addr.DialString(), + } +} +func (mp *Peer) Status() conn.ConnectionStatus { return conn.ConnectionStatus{} } +func (mp *Peer) ID() p2p.ID { return mp.id } +func (mp *Peer) IsOutbound() bool { return mp.Outbound } +func (mp *Peer) IsPersistent() bool { return mp.Persistent } +func (mp *Peer) Get(key string) interface{} { + if value, ok := mp.kv[key]; ok { + return value + } + return nil +} +func (mp *Peer) Set(key string, value interface{}) { + mp.kv[key] = value +} +func (mp *Peer) RemoteIP() net.IP { return mp.ip } +func (mp *Peer) SocketAddr() *p2p.NetAddress { return mp.addr } +func (mp *Peer) RemoteAddr() net.Addr { return &net.TCPAddr{IP: mp.ip, Port: 8800} } +func (mp *Peer) CloseConn() error { return nil } diff --git a/p2p/node_info.go b/p2p/node_info.go index 699fd7f1e..e80f1e1b7 100644 --- a/p2p/node_info.go +++ b/p2p/node_info.go @@ -23,14 +23,8 @@ func MaxNodeInfoSize() int { // NodeInfo exposes basic info of a node // and determines if we're compatible. type NodeInfo interface { - nodeInfoAddress - nodeInfoTransport -} - -// nodeInfoAddress exposes just the core info of a node. -type nodeInfoAddress interface { ID() ID - NetAddress() *NetAddress + nodeInfoTransport } // nodeInfoTransport validates a nodeInfo and checks @@ -221,7 +215,7 @@ func (info DefaultNodeInfo) NetAddress() *NetAddress { if err != nil { switch err.(type) { case ErrNetAddressLookup: - // XXX If the peer provided a host name and the lookup fails here + // XXX If the peer provided a host name and the lookup fails here // we're out of luck. // TODO: use a NetAddress in DefaultNodeInfo default: diff --git a/p2p/peer.go b/p2p/peer.go index 73332a2aa..fab3b42d4 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -29,7 +29,7 @@ type Peer interface { NodeInfo() NodeInfo // peer's info Status() tmconn.ConnectionStatus - OriginalAddr() *NetAddress // original address for outbound peers + SocketAddr() *NetAddress // actual address of the socket Send(byte, []byte) bool TrySend(byte, []byte) bool @@ -46,7 +46,7 @@ type peerConn struct { persistent bool conn net.Conn // source connection - originalAddr *NetAddress // nil for inbound connections + socketAddr *NetAddress // cached RemoteIP() ip net.IP @@ -55,14 +55,14 @@ type peerConn struct { func newPeerConn( outbound, persistent bool, conn net.Conn, - originalAddr *NetAddress, + socketAddr *NetAddress, ) peerConn { return peerConn{ - outbound: outbound, - persistent: persistent, - conn: conn, - originalAddr: originalAddr, + outbound: outbound, + persistent: persistent, + conn: conn, + socketAddr: socketAddr, } } @@ -223,13 +223,12 @@ func (p *peer) NodeInfo() NodeInfo { return p.nodeInfo } -// OriginalAddr returns the original address, which was used to connect with -// the peer. Returns nil for inbound peers. -func (p *peer) OriginalAddr() *NetAddress { - if p.peerConn.outbound { - return p.peerConn.originalAddr - } - return nil +// SocketAddr returns the address of the socket. +// For outbound peers, it's the address dialed (after DNS resolution). +// For inbound peers, it's the address returned by the underlying connection +// (not what's reported in the peer's NodeInfo). +func (p *peer) SocketAddr() *NetAddress { + return p.peerConn.socketAddr } // Status returns the peer's ConnectionStatus. diff --git a/p2p/peer_set_test.go b/p2p/peer_set_test.go index 1d2372fb0..4bacb07d0 100644 --- a/p2p/peer_set_test.go +++ b/p2p/peer_set_test.go @@ -29,7 +29,7 @@ func (mp *mockPeer) IsPersistent() bool { return true } func (mp *mockPeer) Get(s string) interface{} { return s } func (mp *mockPeer) Set(string, interface{}) {} func (mp *mockPeer) RemoteIP() net.IP { return mp.ip } -func (mp *mockPeer) OriginalAddr() *NetAddress { return nil } +func (mp *mockPeer) SocketAddr() *NetAddress { return nil } func (mp *mockPeer) RemoteAddr() net.Addr { return &net.TCPAddr{IP: mp.ip, Port: 8800} } func (mp *mockPeer) CloseConn() error { return nil } diff --git a/p2p/peer_test.go b/p2p/peer_test.go index 90be31131..bf61beb4f 100644 --- a/p2p/peer_test.go +++ b/p2p/peer_test.go @@ -109,25 +109,27 @@ func testOutboundPeerConn( persistent bool, ourNodePrivKey crypto.PrivKey, ) (peerConn, error) { + + var pc peerConn conn, err := testDial(addr, config) if err != nil { - return peerConn{}, cmn.ErrorWrap(err, "Error creating peer") + return pc, cmn.ErrorWrap(err, "Error creating peer") } - pc, err := testPeerConn(conn, config, true, persistent, ourNodePrivKey, addr) + pc, err = testPeerConn(conn, config, true, persistent, ourNodePrivKey, addr) if err != nil { if cerr := conn.Close(); cerr != nil { - return peerConn{}, cmn.ErrorWrap(err, cerr.Error()) + return pc, cmn.ErrorWrap(err, cerr.Error()) } - return peerConn{}, err + return pc, err } // ensure dialed ID matches connection ID if addr.ID != pc.ID() { if cerr := conn.Close(); cerr != nil { - return peerConn{}, cmn.ErrorWrap(err, cerr.Error()) + return pc, cmn.ErrorWrap(err, cerr.Error()) } - return peerConn{}, ErrSwitchAuthenticationFailure{addr, pc.ID()} + return pc, ErrSwitchAuthenticationFailure{addr, pc.ID()} } return pc, nil diff --git a/p2p/pex/pex_reactor.go b/p2p/pex/pex_reactor.go index 01d1d8db5..0ce116326 100644 --- a/p2p/pex/pex_reactor.go +++ b/p2p/pex/pex_reactor.go @@ -167,7 +167,7 @@ func (r *PEXReactor) AddPeer(p Peer) { } } else { // inbound peer is its own source - addr := p.NodeInfo().NetAddress() + addr := p.SocketAddr() src := addr // add to book. dont RequestAddrs right away because @@ -309,7 +309,7 @@ func (r *PEXReactor) ReceiveAddrs(addrs []*p2p.NetAddress, src Peer) error { } r.requestsSent.Delete(id) - srcAddr := src.NodeInfo().NetAddress() + srcAddr := src.SocketAddr() for _, netAddr := range addrs { // Validate netAddr. Disconnect from a peer if it sends us invalid data. if netAddr == nil { diff --git a/p2p/pex/pex_reactor_test.go b/p2p/pex/pex_reactor_test.go index 4f4ccb039..8a35f424e 100644 --- a/p2p/pex/pex_reactor_test.go +++ b/p2p/pex/pex_reactor_test.go @@ -101,7 +101,7 @@ func TestPEXReactorRunning(t *testing.T) { } addOtherNodeAddrToAddrBook := func(switchIndex, otherSwitchIndex int) { - addr := switches[otherSwitchIndex].NodeInfo().NetAddress() + addr := switches[otherSwitchIndex].NetAddress() books[switchIndex].AddAddress(addr, addr) } @@ -132,7 +132,7 @@ func TestPEXReactorReceive(t *testing.T) { r.RequestAddrs(peer) size := book.Size() - addrs := []*p2p.NetAddress{peer.NodeInfo().NetAddress()} + addrs := []*p2p.NetAddress{peer.SocketAddr()} msg := cdc.MustMarshalBinaryBare(&pexAddrsMessage{Addrs: addrs}) r.Receive(PexChannel, peer, msg) assert.Equal(t, size+1, book.Size()) @@ -189,7 +189,7 @@ func TestPEXReactorAddrsMessageAbuse(t *testing.T) { assert.True(t, r.requestsSent.Has(id)) assert.True(t, sw.Peers().Has(peer.ID())) - addrs := []*p2p.NetAddress{peer.NodeInfo().NetAddress()} + addrs := []*p2p.NetAddress{peer.SocketAddr()} msg := cdc.MustMarshalBinaryBare(&pexAddrsMessage{Addrs: addrs}) // receive some addrs. should clear the request @@ -234,7 +234,7 @@ func TestCheckSeeds(t *testing.T) { badPeerConfig = &PEXReactorConfig{ Seeds: []string{"ed3dfd27bfc4af18f67a49862f04cc100696e84d@bad.network.addr:26657", "d824b13cb5d40fa1d8a614e089357c7eff31b670@anotherbad.network.addr:26657", - seed.NodeInfo().NetAddress().String()}, + seed.NetAddress().String()}, } peer = testCreatePeerWithConfig(dir, 2, badPeerConfig) require.Nil(t, peer.Start()) @@ -268,12 +268,13 @@ func TestConnectionSpeedForPeerReceivedFromSeed(t *testing.T) { defer os.RemoveAll(dir) // nolint: errcheck // 1. create peer - peer := testCreateDefaultPeer(dir, 1) - require.Nil(t, peer.Start()) - defer peer.Stop() + peerSwitch := testCreateDefaultPeer(dir, 1) + require.Nil(t, peerSwitch.Start()) + defer peerSwitch.Stop() // 2. Create seed which knows about the peer - seed := testCreateSeed(dir, 2, []*p2p.NetAddress{peer.NodeInfo().NetAddress()}, []*p2p.NetAddress{peer.NodeInfo().NetAddress()}) + peerAddr := peerSwitch.NetAddress() + seed := testCreateSeed(dir, 2, []*p2p.NetAddress{peerAddr}, []*p2p.NetAddress{peerAddr}) require.Nil(t, seed.Start()) defer seed.Stop() @@ -300,7 +301,7 @@ func TestPEXReactorCrawlStatus(t *testing.T) { // Create a peer, add it to the peer set and the addrbook. peer := p2p.CreateRandomPeer(false) p2p.AddPeerToSwitch(pexR.Switch, peer) - addr1 := peer.NodeInfo().NetAddress() + addr1 := peer.SocketAddr() pexR.book.AddAddress(addr1, addr1) // Add a non-connected address to the book. @@ -364,7 +365,7 @@ func TestPEXReactorSeedModeFlushStop(t *testing.T) { reactor := switches[0].Reactors()["pex"].(*PEXReactor) peerID := switches[1].NodeInfo().ID() - err = switches[1].DialPeerWithAddress(switches[0].NodeInfo().NetAddress(), false) + err = switches[1].DialPeerWithAddress(switches[0].NetAddress(), false) assert.NoError(t, err) // sleep up to a second while waiting for the peer to send us a message. @@ -402,7 +403,7 @@ func TestPEXReactorDoesNotAddPrivatePeersToAddrBook(t *testing.T) { pexR.RequestAddrs(peer) size := book.Size() - addrs := []*p2p.NetAddress{peer.NodeInfo().NetAddress()} + addrs := []*p2p.NetAddress{peer.SocketAddr()} msg := cdc.MustMarshalBinaryBare(&pexAddrsMessage{Addrs: addrs}) pexR.Receive(PexChannel, peer, msg) assert.Equal(t, size, book.Size()) @@ -419,7 +420,7 @@ func TestPEXReactorDialPeer(t *testing.T) { sw.SetAddrBook(book) peer := newMockPeer() - addr := peer.NodeInfo().NetAddress() + addr := peer.SocketAddr() assert.Equal(t, 0, pexR.AttemptsToDial(addr)) @@ -590,7 +591,7 @@ func testCreateSeed(dir string, id int, knownAddrs, srcAddrs []*p2p.NetAddress) // Starting and stopping the peer is left to the caller func testCreatePeerWithSeed(dir string, id int, seed *p2p.Switch) *p2p.Switch { conf := &PEXReactorConfig{ - Seeds: []string{seed.NodeInfo().NetAddress().String()}, + Seeds: []string{seed.NetAddress().String()}, } return testCreatePeerWithConfig(dir, id, conf) } diff --git a/p2p/switch.go b/p2p/switch.go index 7d2e6c3f3..2228ac071 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -86,6 +86,12 @@ type Switch struct { metrics *Metrics } +// NetAddress returns the address the switch is listening on. +func (sw *Switch) NetAddress() *NetAddress { + addr := sw.transport.NetAddress() + return &addr +} + // SwitchOption sets an optional parameter on the Switch. type SwitchOption func(*Switch) @@ -284,13 +290,7 @@ func (sw *Switch) StopPeerForError(peer Peer, reason interface{}) { sw.stopAndRemovePeer(peer, reason) if peer.IsPersistent() { - addr := peer.OriginalAddr() - if addr == nil { - // FIXME: persistent peers can't be inbound right now. - // self-reported address for inbound persistent peers - addr = peer.NodeInfo().NetAddress() - } - go sw.reconnectToPeer(addr) + go sw.reconnectToPeer(peer.SocketAddr()) } } @@ -378,7 +378,7 @@ func (sw *Switch) SetAddrBook(addrBook AddrBook) { // like contributed to consensus. func (sw *Switch) MarkPeerAsGood(peer Peer) { if sw.addrBook != nil { - sw.addrBook.MarkGood(peer.NodeInfo().NetAddress()) + sw.addrBook.MarkGood(peer.SocketAddr()) } } @@ -395,7 +395,7 @@ func (sw *Switch) DialPeersAsync(addrBook AddrBook, peers []string, persistent b sw.Logger.Error("Error in peer's address", "err", err) } - ourAddr := sw.nodeInfo.NetAddress() + ourAddr := sw.NetAddress() // TODO: this code feels like it's in the wrong place. // The integration tests depend on the addrBook being saved @@ -524,7 +524,7 @@ func (sw *Switch) acceptRoutine() { if in >= sw.config.MaxNumInboundPeers { sw.Logger.Info( "Ignoring inbound connection: already have enough inbound peers", - "address", p.NodeInfo().NetAddress().String(), + "address", p.SocketAddr(), "have", in, "max", sw.config.MaxNumInboundPeers, ) @@ -641,7 +641,7 @@ func (sw *Switch) addPeer(p Peer) error { return err } - p.SetLogger(sw.Logger.With("peer", p.NodeInfo().NetAddress())) + p.SetLogger(sw.Logger.With("peer", p.SocketAddr())) // Handle the shut down case where the switch has stopped but we're // concurrently trying to add a peer. diff --git a/p2p/switch_test.go b/p2p/switch_test.go index 358661616..ed968d738 100644 --- a/p2p/switch_test.go +++ b/p2p/switch_test.go @@ -160,10 +160,6 @@ func assertMsgReceivedWithTimeout(t *testing.T, msgBytes []byte, channel byte, r func TestSwitchFiltersOutItself(t *testing.T) { s1 := MakeSwitch(cfg, 1, "127.0.0.1", "123.123.123", initSwitchFunc) - // addr := s1.NodeInfo().NetAddress() - - // // add ourselves like we do in node.go#427 - // s1.addrBook.AddOurAddress(addr) // simulate s1 having a public IP by creating a remote peer with the same ID rp := &remotePeer{PrivKey: s1.nodeKey.PrivKey, Config: cfg} @@ -495,7 +491,7 @@ func TestSwitchAcceptRoutine(t *testing.T) { rp := &remotePeer{PrivKey: ed25519.GenPrivKey(), Config: cfg} remotePeers = append(remotePeers, rp) rp.Start() - c, err := rp.Dial(sw.NodeInfo().NetAddress()) + c, err := rp.Dial(sw.NetAddress()) require.NoError(t, err) // spawn a reading routine to prevent connection from closing go func(c net.Conn) { @@ -514,7 +510,7 @@ func TestSwitchAcceptRoutine(t *testing.T) { // 2. check we close new connections if we already have MaxNumInboundPeers peers rp := &remotePeer{PrivKey: ed25519.GenPrivKey(), Config: cfg} rp.Start() - conn, err := rp.Dial(sw.NodeInfo().NetAddress()) + conn, err := rp.Dial(sw.NetAddress()) require.NoError(t, err) // check conn is closed one := make([]byte, 1) diff --git a/p2p/test_util.go b/p2p/test_util.go index 2d320df85..df60539ba 100644 --- a/p2p/test_util.go +++ b/p2p/test_util.go @@ -35,7 +35,8 @@ func CreateRandomPeer(outbound bool) *peer { addr, netAddr := CreateRoutableAddr() p := &peer{ peerConn: peerConn{ - outbound: outbound, + outbound: outbound, + socketAddr: netAddr, }, nodeInfo: mockNodeInfo{netAddr}, mconn: &conn.MConnection{}, @@ -174,10 +175,15 @@ func MakeSwitch( PrivKey: ed25519.GenPrivKey(), } nodeInfo := testNodeInfo(nodeKey.ID(), fmt.Sprintf("node%d", i)) + addr, err := NewNetAddressString( + IDAddressString(nodeKey.ID(), nodeInfo.(DefaultNodeInfo).ListenAddr), + ) + if err != nil { + panic(err) + } t := NewMultiplexTransport(nodeInfo, nodeKey, MConnConfig(cfg)) - addr := nodeInfo.NetAddress() if err := t.Listen(*addr); err != nil { panic(err) } @@ -214,7 +220,7 @@ func testPeerConn( cfg *config.P2PConfig, outbound, persistent bool, ourNodePrivKey crypto.PrivKey, - originalAddr *NetAddress, + socketAddr *NetAddress, ) (pc peerConn, err error) { conn := rawConn @@ -231,12 +237,7 @@ func testPeerConn( } // Only the information we already have - return peerConn{ - outbound: outbound, - persistent: persistent, - conn: conn, - originalAddr: originalAddr, - }, nil + return newPeerConn(outbound, persistent, conn, socketAddr), nil } //---------------------------------------------------------------- diff --git a/p2p/transport.go b/p2p/transport.go index d1bccf9b8..f07253c7f 100644 --- a/p2p/transport.go +++ b/p2p/transport.go @@ -24,6 +24,7 @@ type IPResolver interface { // accept is the container to carry the upgraded connection and NodeInfo from an // asynchronously running routine to the Accept method. type accept struct { + netAddr *NetAddress conn net.Conn nodeInfo NodeInfo err error @@ -47,6 +48,9 @@ type peerConfig struct { // the transport. Each transport is also responsible to filter establishing // peers specific to its domain. type Transport interface { + // Listening address. + NetAddress() NetAddress + // Accept returns a newly connected Peer. Accept(peerConfig) (Peer, error) @@ -115,6 +119,7 @@ func MultiplexTransportResolver(resolver IPResolver) MultiplexTransportOption { // MultiplexTransport accepts and dials tcp connections and upgrades them to // multiplexed peers. type MultiplexTransport struct { + netAddr NetAddress listener net.Listener acceptc chan accept @@ -161,6 +166,11 @@ func NewMultiplexTransport( } } +// NetAddress implements Transport. +func (mt *MultiplexTransport) NetAddress() NetAddress { + return mt.netAddr +} + // Accept implements Transport. func (mt *MultiplexTransport) Accept(cfg peerConfig) (Peer, error) { select { @@ -173,7 +183,7 @@ func (mt *MultiplexTransport) Accept(cfg peerConfig) (Peer, error) { cfg.outbound = false - return mt.wrapPeer(a.conn, a.nodeInfo, cfg, nil), nil + return mt.wrapPeer(a.conn, a.nodeInfo, cfg, a.netAddr), nil case <-mt.closec: return nil, &ErrTransportClosed{} } @@ -224,6 +234,7 @@ func (mt *MultiplexTransport) Listen(addr NetAddress) error { return err } + mt.netAddr = addr mt.listener = ln go mt.acceptPeers() @@ -258,15 +269,21 @@ func (mt *MultiplexTransport) acceptPeers() { var ( nodeInfo NodeInfo secretConn *conn.SecretConnection + netAddr *NetAddress ) err := mt.filterConn(c) if err == nil { secretConn, nodeInfo, err = mt.upgrade(c, nil) + if err == nil { + addr := c.RemoteAddr() + id := PubKeyToID(secretConn.RemotePubKey()) + netAddr = NewNetAddress(id, addr) + } } select { - case mt.acceptc <- accept{secretConn, nodeInfo, err}: + case mt.acceptc <- accept{netAddr, secretConn, nodeInfo, err}: // Make the upgraded peer available. case <-mt.closec: // Give up if the transport was closed. @@ -426,14 +443,14 @@ func (mt *MultiplexTransport) wrapPeer( c net.Conn, ni NodeInfo, cfg peerConfig, - dialedAddr *NetAddress, + socketAddr *NetAddress, ) Peer { peerConn := newPeerConn( cfg.outbound, cfg.persistent, c, - dialedAddr, + socketAddr, ) p := newPeer( diff --git a/p2p/transport_test.go b/p2p/transport_test.go index 81f9d1b8e..35fd9c66b 100644 --- a/p2p/transport_test.go +++ b/p2p/transport_test.go @@ -8,6 +8,8 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" + "github.com/tendermint/tendermint/crypto/ed25519" "github.com/tendermint/tendermint/p2p/conn" ) @@ -142,43 +144,23 @@ func TestTransportMultiplexConnFilterTimeout(t *testing.T) { func TestTransportMultiplexAcceptMultiple(t *testing.T) { mt := testSetupMultiplexTransport(t) + id, addr := mt.nodeKey.ID(), mt.listener.Addr().String() + laddr, err := NewNetAddressStringWithOptionalID(IDAddressString(id, addr)) + require.NoError(t, err) var ( - seed = rand.New(rand.NewSource(time.Now().UnixNano())) - errc = make(chan error, seed.Intn(64)+64) + seed = rand.New(rand.NewSource(time.Now().UnixNano())) + nDialers = seed.Intn(64) + 64 + errc = make(chan error, nDialers) ) // Setup dialers. - for i := 0; i < cap(errc); i++ { - go func() { - var ( - pv = ed25519.GenPrivKey() - dialer = newMultiplexTransport( - testNodeInfo(PubKeyToID(pv.PubKey()), defaultNodeName), - NodeKey{ - PrivKey: pv, - }, - ) - ) - addr, err := NewNetAddressStringWithOptionalID(IDAddressString(mt.nodeKey.ID(), mt.listener.Addr().String())) - if err != nil { - errc <- err - return - } - - _, err = dialer.Dial(*addr, peerConfig{}) - if err != nil { - errc <- err - return - } - - // Signal that the connection was established. - errc <- nil - }() + for i := 0; i < nDialers; i++ { + go testDialer(*laddr, errc) } // Catch connection errors. - for i := 0; i < cap(errc); i++ { + for i := 0; i < nDialers; i++ { if err := <-errc; err != nil { t.Fatal(err) } @@ -216,6 +198,27 @@ func TestTransportMultiplexAcceptMultiple(t *testing.T) { } } +func testDialer(dialAddr NetAddress, errc chan error) { + var ( + pv = ed25519.GenPrivKey() + dialer = newMultiplexTransport( + testNodeInfo(PubKeyToID(pv.PubKey()), defaultNodeName), + NodeKey{ + PrivKey: pv, + }, + ) + ) + + _, err := dialer.Dial(dialAddr, peerConfig{}) + if err != nil { + errc <- err + return + } + + // Signal that the connection was established. + errc <- nil +} + func TestTransportMultiplexAcceptNonBlocking(t *testing.T) { mt := testSetupMultiplexTransport(t) @@ -591,6 +594,7 @@ func TestTransportHandshake(t *testing.T) { } } +// create listener func testSetupMultiplexTransport(t *testing.T) *MultiplexTransport { var ( pv = ed25519.GenPrivKey() diff --git a/rpc/core/consensus.go b/rpc/core/consensus.go index 81694b7ee..08f989799 100644 --- a/rpc/core/consensus.go +++ b/rpc/core/consensus.go @@ -215,7 +215,7 @@ func DumpConsensusState() (*ctypes.ResultDumpConsensusState, error) { } peerStates[i] = ctypes.PeerStateInfo{ // Peer basic info. - NodeAddress: peer.NodeInfo().NetAddress().String(), + NodeAddress: peer.SocketAddr().String(), // Peer consensus state. PeerState: peerStateJSON, } From 5a7a97541b3216364b4b37424a1d00c5bd6a0abe Mon Sep 17 00:00:00 2001 From: Ismail Khoffi Date: Tue, 2 Apr 2019 21:38:50 +0200 Subject: [PATCH 3/5] delete dummy mock, mockPeer, and use mock.Peer instead * delete dummy mock and use mock mock instead: - this fixes the build/test/linter issues in #77 - this replicates some of the changes of https://github.com/tendermint/tendermint/commit/d2f3a306be5ca4e6890fcab18b14785cceaffb65 (see https://github.com/tendermint/tendermint/pull/3487#discussion_r269620510) * delete obsolete mockPeer (how often did we mock this?) --- consensus/state_test.go | 6 +-- p2p/dummy/peer.go | 100 ------------------------------------ p2p/pex/pex_reactor_test.go | 51 ++---------------- 3 files changed, 8 insertions(+), 149 deletions(-) delete mode 100644 p2p/dummy/peer.go diff --git a/consensus/state_test.go b/consensus/state_test.go index 69de825e7..a11809c5f 100644 --- a/consensus/state_test.go +++ b/consensus/state_test.go @@ -14,7 +14,7 @@ import ( cmn "github.com/tendermint/tendermint/libs/common" "github.com/tendermint/tendermint/libs/log" tmpubsub "github.com/tendermint/tendermint/libs/pubsub" - p2pdummy "github.com/tendermint/tendermint/p2p/dummy" + p2pmock "github.com/tendermint/tendermint/p2p/mock" "github.com/tendermint/tendermint/types" ) @@ -1548,7 +1548,7 @@ func TestStateHalt1(t *testing.T) { func TestStateOutputsBlockPartsStats(t *testing.T) { // create dummy peer cs, _ := randConsensusState(1) - peer := p2pdummy.NewPeer() + peer := p2pmock.NewPeer(nil) // 1) new block part parts := types.NewPartSetFromData(cmn.RandBytes(100), 10) @@ -1591,7 +1591,7 @@ func TestStateOutputsBlockPartsStats(t *testing.T) { func TestStateOutputVoteStats(t *testing.T) { cs, vss := randConsensusState(2) // create dummy peer - peer := p2pdummy.NewPeer() + peer := p2pmock.NewPeer(nil) vote := signVote(vss[1], types.PrecommitType, []byte("test"), types.PartSetHeader{}) diff --git a/p2p/dummy/peer.go b/p2p/dummy/peer.go deleted file mode 100644 index 57edafc67..000000000 --- a/p2p/dummy/peer.go +++ /dev/null @@ -1,100 +0,0 @@ -package dummy - -import ( - "net" - - cmn "github.com/tendermint/tendermint/libs/common" - p2p "github.com/tendermint/tendermint/p2p" - tmconn "github.com/tendermint/tendermint/p2p/conn" -) - -type peer struct { - cmn.BaseService - kv map[string]interface{} -} - -var _ p2p.Peer = (*peer)(nil) - -// NewPeer creates new dummy peer. -func NewPeer() *peer { - p := &peer{ - kv: make(map[string]interface{}), - } - p.BaseService = *cmn.NewBaseService(nil, "peer", p) - - return p -} - -// FlushStop just calls Stop. -func (p *peer) FlushStop() { - p.Stop() -} - -// ID always returns dummy. -func (p *peer) ID() p2p.ID { - return p2p.ID("dummy") -} - -// IsOutbound always returns false. -func (p *peer) IsOutbound() bool { - return false -} - -// IsPersistent always returns false. -func (p *peer) IsPersistent() bool { - return false -} - -// NodeInfo always returns empty node info. -func (p *peer) NodeInfo() p2p.NodeInfo { - return p2p.DefaultNodeInfo{} -} - -// RemoteIP always returns localhost. -func (p *peer) RemoteIP() net.IP { - return net.ParseIP("127.0.0.1") -} - -// Addr always returns tcp://localhost:8800. -func (p *peer) RemoteAddr() net.Addr { - return &net.TCPAddr{IP: net.ParseIP("127.0.0.1"), Port: 8800} -} - -// CloseConn always returns nil. -func (p *peer) CloseConn() error { - return nil -} - -// Status always returns empry connection status. -func (p *peer) Status() tmconn.ConnectionStatus { - return tmconn.ConnectionStatus{} -} - -// Send does not do anything and just returns true. -func (p *peer) Send(byte, []byte) bool { - return true -} - -// TrySend does not do anything and just returns true. -func (p *peer) TrySend(byte, []byte) bool { - return true -} - -// Set records value under key specified in the map. -func (p *peer) Set(key string, value interface{}) { - p.kv[key] = value -} - -// Get returns a value associated with the key. Nil is returned if no value -// found. -func (p *peer) Get(key string) interface{} { - if value, ok := p.kv[key]; ok { - return value - } - return nil -} - -// OriginalAddr always returns nil. -func (p *peer) OriginalAddr() *p2p.NetAddress { - return nil -} diff --git a/p2p/pex/pex_reactor_test.go b/p2p/pex/pex_reactor_test.go index 8a35f424e..be0669703 100644 --- a/p2p/pex/pex_reactor_test.go +++ b/p2p/pex/pex_reactor_test.go @@ -2,8 +2,8 @@ package pex import ( "fmt" + "io/ioutil" - "net" "os" "path/filepath" "testing" @@ -12,14 +12,11 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/tendermint/tendermint/crypto" - "github.com/tendermint/tendermint/crypto/ed25519" - cmn "github.com/tendermint/tendermint/libs/common" "github.com/tendermint/tendermint/libs/log" + "github.com/tendermint/tendermint/p2p/mock" "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/p2p" - "github.com/tendermint/tendermint/p2p/conn" ) var ( @@ -148,7 +145,7 @@ func TestPEXReactorRequestMessageAbuse(t *testing.T) { sw := createSwitchAndAddReactors(r) sw.SetAddrBook(book) - peer := newMockPeer() + peer := mock.NewPeer(nil) p2p.AddPeerToSwitch(sw, peer) assert.True(t, sw.Peers().Has(peer.ID())) @@ -178,7 +175,7 @@ func TestPEXReactorAddrsMessageAbuse(t *testing.T) { sw := createSwitchAndAddReactors(r) sw.SetAddrBook(book) - peer := newMockPeer() + peer := mock.NewPeer(nil) p2p.AddPeerToSwitch(sw, peer) assert.True(t, sw.Peers().Has(peer.ID())) @@ -419,7 +416,7 @@ func TestPEXReactorDialPeer(t *testing.T) { sw := createSwitchAndAddReactors(pexR) sw.SetAddrBook(book) - peer := newMockPeer() + peer := mock.NewPeer(nil) addr := peer.SocketAddr() assert.Equal(t, 0, pexR.AttemptsToDial(addr)) @@ -445,44 +442,6 @@ func TestPEXReactorDialPeer(t *testing.T) { } } -type mockPeer struct { - *cmn.BaseService - pubKey crypto.PubKey - addr *p2p.NetAddress - outbound, persistent bool -} - -func newMockPeer() mockPeer { - _, netAddr := p2p.CreateRoutableAddr() - mp := mockPeer{ - addr: netAddr, - pubKey: ed25519.GenPrivKey().PubKey(), - } - mp.BaseService = cmn.NewBaseService(nil, "MockPeer", mp) - mp.Start() - return mp -} - -func (mp mockPeer) FlushStop() { mp.Stop() } -func (mp mockPeer) ID() p2p.ID { return mp.addr.ID } -func (mp mockPeer) IsOutbound() bool { return mp.outbound } -func (mp mockPeer) IsPersistent() bool { return mp.persistent } -func (mp mockPeer) NodeInfo() p2p.NodeInfo { - return p2p.DefaultNodeInfo{ - ID_: mp.addr.ID, - ListenAddr: mp.addr.DialString(), - } -} -func (mockPeer) RemoteIP() net.IP { return net.ParseIP("127.0.0.1") } -func (mockPeer) Status() conn.ConnectionStatus { return conn.ConnectionStatus{} } -func (mockPeer) Send(byte, []byte) bool { return false } -func (mockPeer) TrySend(byte, []byte) bool { return false } -func (mockPeer) Set(string, interface{}) {} -func (mockPeer) Get(string) interface{} { return nil } -func (mockPeer) OriginalAddr() *p2p.NetAddress { return nil } -func (mockPeer) RemoteAddr() net.Addr { return &net.TCPAddr{IP: net.ParseIP("127.0.0.1"), Port: 8800} } -func (mockPeer) CloseConn() error { return nil } - func assertPeersWithTimeout( t *testing.T, switches []*p2p.Switch, From eb7af9a765561ad0b31fe600db09979a1db1604a Mon Sep 17 00:00:00 2001 From: Ismail Khoffi Date: Tue, 2 Apr 2019 21:47:46 +0200 Subject: [PATCH 4/5] Backport release automation scripts for v0.30.3 * Release management using CircleCI (#3498) * Release management using CircleCI * Changelog updated * Fix for wrong version tag (#3517) * Fix for wrong version tag (tag on the release branch instead of master) * Release message changelog link fix (#3519) --- .circleci/config.yml | 149 +++++++++++++++++- CHANGELOG_PENDING.md | 2 + DOCKER/Dockerfile | 4 +- Makefile | 11 +- scripts/release_management/README.md | 65 ++++++++ scripts/release_management/bump-semver.py | 37 +++++ scripts/release_management/github-draft.py | 61 +++++++ scripts/release_management/github-openpr.py | 52 ++++++ .../github-public-newbranch.bash | 28 ++++ scripts/release_management/github-publish.py | 53 +++++++ scripts/release_management/github-upload.py | 68 ++++++++ scripts/release_management/sha-files.py | 35 ++++ scripts/release_management/zip-file.py | 44 ++++++ 13 files changed, 597 insertions(+), 12 deletions(-) create mode 100644 scripts/release_management/README.md create mode 100755 scripts/release_management/bump-semver.py create mode 100755 scripts/release_management/github-draft.py create mode 100755 scripts/release_management/github-openpr.py create mode 100644 scripts/release_management/github-public-newbranch.bash create mode 100755 scripts/release_management/github-publish.py create mode 100755 scripts/release_management/github-upload.py create mode 100755 scripts/release_management/sha-files.py create mode 100755 scripts/release_management/zip-file.py diff --git a/.circleci/config.yml b/.circleci/config.yml index d6440de1e..7ad793549 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,17 +3,20 @@ version: 2 defaults: &defaults working_directory: /go/src/github.com/tendermint/tendermint docker: - - image: circleci/golang:1.11.4 + - image: circleci/golang environment: GOBIN: /tmp/workspace/bin docs_update_config: &docs_update_config working_directory: ~/repo docker: - - image: tendermint/docs_deployment + - image: tendermintdev/jq_curl environment: AWS_REGION: us-east-1 +release_management_docker: &release_management_docker + machine: true + jobs: setup_dependencies: <<: *defaults @@ -154,7 +157,7 @@ jobs: for pkg in $(go list github.com/tendermint/tendermint/... | circleci tests split --split-by=timings); do id=$(basename "$pkg") - GOCACHE=off go test -v -timeout 5m -race -coverprofile=/tmp/workspace/profiles/$id.out -covermode=atomic "$pkg" | tee "/tmp/logs/$id-$RANDOM.log" + go test -v -timeout 5m -race -coverprofile=/tmp/workspace/profiles/$id.out -covermode=atomic "$pkg" | tee "/tmp/logs/$id-$RANDOM.log" done - persist_to_workspace: root: /tmp/workspace @@ -192,7 +195,7 @@ jobs: name: run localnet and exit on failure command: | set -x - docker run --rm -v "$PWD":/go/src/github.com/tendermint/tendermint -w /go/src/github.com/tendermint/tendermint golang:1.11.4 make build-linux + docker run --rm -v "$PWD":/go/src/github.com/tendermint/tendermint -w /go/src/github.com/tendermint/tendermint golang make build-linux make localnet-start & ./scripts/localnet-blocks-test.sh 40 5 10 localhost @@ -239,7 +242,121 @@ jobs: - run: name: Trigger website build command: | - chamber exec tendermint -- start_website_build + curl --silent \ + --show-error \ + -X POST \ + --header "Content-Type: application/json" \ + -d "{\"branch\": \"$CIRCLE_BRANCH\"}" \ + "https://circleci.com/api/v1.1/project/github/$CIRCLE_PROJECT_USERNAME/$WEBSITE_REPO_NAME/build?circle-token=$TENDERBOT_API_TOKEN" > response.json + + RESULT=`jq -r '.status' response.json` + MESSAGE=`jq -r '.message' response.json` + + if [[ ${RESULT} == "null" ]] || [[ ${RESULT} -ne "200" ]]; then + echo "CircleCI API call failed: $MESSAGE" + exit 1 + else + echo "Website build started" + fi + + prepare_build: + <<: *defaults + steps: + - checkout + - run: + name: Get next release number + command: | + export LAST_TAG="`git describe --tags --abbrev=0 --match "${CIRCLE_BRANCH}.*"`" + echo "Last tag: ${LAST_TAG}" + if [ -z "${LAST_TAG}" ]; then + export LAST_TAG="${CIRCLE_BRANCH}" + echo "Last tag not found. Possibly fresh branch or feature branch. Setting ${LAST_TAG} as tag." + fi + export NEXT_TAG="`python -u scripts/release_management/bump-semver.py --version "${LAST_TAG}"`" + echo "Next tag: ${NEXT_TAG}" + echo "export CIRCLE_TAG=\"${NEXT_TAG}\"" > release-version.source + - run: + name: Build dependencies + command: | + make get_tools get_vendor_deps + - persist_to_workspace: + root: . + paths: + - "release-version.source" + - save_cache: + key: v1-release-deps-{{ .Branch }}-{{ .Revision }} + paths: + - "vendor" + + build_artifacts: + <<: *defaults + parallelism: 4 + steps: + - checkout + - restore_cache: + keys: + - v1-release-deps-{{ .Branch }}-{{ .Revision }} + - attach_workspace: + at: /tmp/workspace + - run: + name: Build artifact + command: | + # Setting CIRCLE_TAG because we do not tag the release ourselves. + source /tmp/workspace/release-version.source + if test ${CIRCLE_NODE_INDEX:-0} == 0 ;then export GOOS=linux GOARCH=amd64 && export OUTPUT=build/tendermint_${GOOS}_${GOARCH} && make build && python -u scripts/release_management/zip-file.py ;fi + if test ${CIRCLE_NODE_INDEX:-0} == 1 ;then export GOOS=darwin GOARCH=amd64 && export OUTPUT=build/tendermint_${GOOS}_${GOARCH} && make build && python -u scripts/release_management/zip-file.py ;fi + if test ${CIRCLE_NODE_INDEX:-0} == 2 ;then export GOOS=windows GOARCH=amd64 && export OUTPUT=build/tendermint_${GOOS}_${GOARCH} && make build && python -u scripts/release_management/zip-file.py ;fi + if test ${CIRCLE_NODE_INDEX:-0} == 3 ;then export GOOS=linux GOARCH=arm && export OUTPUT=build/tendermint_${GOOS}_${GOARCH} && make build && python -u scripts/release_management/zip-file.py ;fi + - persist_to_workspace: + root: build + paths: + - "*.zip" + - "tendermint_linux_amd64" + + release_artifacts: + <<: *defaults + steps: + - checkout + - attach_workspace: + at: /tmp/workspace + - run: + name: Deploy to GitHub + command: | + # Setting CIRCLE_TAG because we do not tag the release ourselves. + source /tmp/workspace/release-version.source + echo "---" + ls -la /tmp/workspace/*.zip + echo "---" + python -u scripts/release_management/sha-files.py + echo "---" + cat /tmp/workspace/SHA256SUMS + echo "---" + export RELEASE_ID="`python -u scripts/release_management/github-draft.py`" + echo "Release ID: ${RELEASE_ID}" + #Todo: Parallelize uploads + export GOOS=linux GOARCH=amd64 && python -u scripts/release_management/github-upload.py --id "${RELEASE_ID}" + export GOOS=darwin GOARCH=amd64 && python -u scripts/release_management/github-upload.py --id "${RELEASE_ID}" + export GOOS=windows GOARCH=amd64 && python -u scripts/release_management/github-upload.py --id "${RELEASE_ID}" + export GOOS=linux GOARCH=arm && python -u scripts/release_management/github-upload.py --id "${RELEASE_ID}" + python -u scripts/release_management/github-upload.py --file "/tmp/workspace/SHA256SUMS" --id "${RELEASE_ID}" + python -u scripts/release_management/github-publish.py --id "${RELEASE_ID}" + + release_docker: + <<: *release_management_docker + steps: + - checkout + - attach_workspace: + at: /tmp/workspace + - run: + name: Deploy to Docker Hub + command: | + # Setting CIRCLE_TAG because we do not tag the release ourselves. + source /tmp/workspace/release-version.source + cp /tmp/workspace/tendermint_linux_amd64 DOCKER/tendermint + docker build --label="tendermint" --tag="tendermint/tendermint:${CIRCLE_TAG}" --tag="tendermint/tendermint:latest" "DOCKER" + docker login -u "${DOCKERHUB_USER}" --password-stdin <<< "${DOCKERHUB_PASS}" + docker push "tendermint/tendermint" + docker logout workflows: version: 2 @@ -277,3 +394,25 @@ workflows: - upload_coverage: requires: - test_cover + release: + jobs: + - prepare_build + - build_artifacts: + requires: + - prepare_build + - release_artifacts: + requires: + - prepare_build + - build_artifacts + filters: + branches: + only: + - /v[0-9]+\.[0-9]+/ + - release_docker: + requires: + - prepare_build + - build_artifacts + filters: + branches: + only: + - /v[0-9]+\.[0-9]+/ diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 640985836..a7548ee71 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -20,4 +20,6 @@ Special thanks to external contributors on this release: ### IMPROVEMENTS: +- [CircleCI] \#3497 Move release management to CircleCI + ### BUG FIXES: diff --git a/DOCKER/Dockerfile b/DOCKER/Dockerfile index 4a855f425..6a7f289f5 100644 --- a/DOCKER/Dockerfile +++ b/DOCKER/Dockerfile @@ -1,5 +1,5 @@ -FROM alpine:3.7 -MAINTAINER Greg Szabo +FROM alpine:3.9 +LABEL maintainer="hello@tendermint.com" # Tendermint will be looking for the genesis file in /tendermint/config/genesis.json # (unless you change `genesis_file` in config.toml). You can put your config.toml and diff --git a/Makefile b/Makefile index a1ba06aa9..50e7d3b58 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ GOTOOLS = \ github.com/square/certstrap GOBIN?=${GOPATH}/bin PACKAGES=$(shell go list ./...) +OUTPUT?=build/tendermint INCLUDE = -I=. -I=${GOPATH}/src -I=${GOPATH}/src/github.com/gogo/protobuf/protobuf BUILD_TAGS?='tendermint' @@ -19,13 +20,13 @@ check: check_tools get_vendor_deps ### Build Tendermint build: - CGO_ENABLED=0 go build $(BUILD_FLAGS) -tags $(BUILD_TAGS) -o build/tendermint ./cmd/tendermint/ + CGO_ENABLED=0 go build $(BUILD_FLAGS) -tags $(BUILD_TAGS) -o $(OUTPUT) ./cmd/tendermint/ build_c: - CGO_ENABLED=1 go build $(BUILD_FLAGS) -tags "$(BUILD_TAGS) gcc" -o build/tendermint ./cmd/tendermint/ + CGO_ENABLED=1 go build $(BUILD_FLAGS) -tags "$(BUILD_TAGS) gcc" -o $(OUTPUT) ./cmd/tendermint/ build_race: - CGO_ENABLED=0 go build -race $(BUILD_FLAGS) -tags $(BUILD_TAGS) -o build/tendermint ./cmd/tendermint + CGO_ENABLED=0 go build -race $(BUILD_FLAGS) -tags $(BUILD_TAGS) -o $(OUTPUT) ./cmd/tendermint install: CGO_ENABLED=0 go install $(BUILD_FLAGS) -tags $(BUILD_TAGS) ./cmd/tendermint @@ -109,7 +110,7 @@ draw_deps: get_deps_bin_size: @# Copy of build recipe with additional flags to perform binary size analysis - $(eval $(shell go build -work -a $(BUILD_FLAGS) -tags $(BUILD_TAGS) -o build/tendermint ./cmd/tendermint/ 2>&1)) + $(eval $(shell go build -work -a $(BUILD_FLAGS) -tags $(BUILD_TAGS) -o $(OUTPUT) ./cmd/tendermint/ 2>&1)) @find $(WORK) -type f -name "*.a" | xargs -I{} du -hxs "{}" | sort -rh | sed -e s:${WORK}/::g > deps_bin_size.log @echo "Results can be found here: $(CURDIR)/deps_bin_size.log" @@ -261,7 +262,7 @@ check_dep: ### Docker image build-docker: - cp build/tendermint DOCKER/tendermint + cp $(OUTPUT) DOCKER/tendermint docker build --label=tendermint --tag="tendermint/tendermint" DOCKER rm -rf DOCKER/tendermint diff --git a/scripts/release_management/README.md b/scripts/release_management/README.md new file mode 100644 index 000000000..e92f1ccf6 --- /dev/null +++ b/scripts/release_management/README.md @@ -0,0 +1,65 @@ +# Release management scripts + +## Overview +The scripts in this folder are used for release management in CircleCI. Although the scripts are fully configurable using input parameters, +the default settings were modified to accommodate CircleCI execution. + +# Build scripts +These scripts help during the build process. They prepare the release files. + +## bump-semver.py +Bumps the semantic version of the input `--version`. Versions are expected in vMAJOR.MINOR.PATCH format or vMAJOR.MINOR format. + +In vMAJOR.MINOR format, the result will be patch version 0 of that version, for example `v1.2 -> v1.2.0`. + +In vMAJOR.MINOR.PATCH format, the result will be a bumped PATCH version, for example `v1.2.3 -> v1.2.4`. + +If the PATCH number contains letters, it is considered a development version, in which case, the result is the non-development version of that number. +The patch number will not be bumped, only the "-dev" or similar additional text will be removed. For example: `v1.2.6-rc1 -> v1.2.6`. + +## zip-file.py +Specialized ZIP command for release management. Special features: +1. Uses Python ZIP libaries, so the `zip` command does not need to be installed. +1. Can only zip one file. +1. Optionally gets file version, Go OS and architecture. +1. By default all inputs and output is formatted exactly how CircleCI needs it. + +By default, the command will try to ZIP the file at `build/tendermint_${GOOS}_${GOARCH}`. +This can be changed with the `--file` input parameter. + +By default, the command will output the ZIP file to `build/tendermint_${CIRCLE_TAG}_${GOOS}_${GOARCH}.zip`. +This can be changed with the `--destination` (folder), `--version`, `--goos` and `--goarch` input parameters respectively. + +## sha-files.py +Specialized `shasum` command for release management. Special features: +1. Reads all ZIP files in the given folder. +1. By default all inputs and output is formatted exactly how CircleCI needs it. + +By default, the command will look up all ZIP files in the `build/` folder. + +By default, the command will output results into the `build/SHA256SUMS` file. + +# GitHub management +Uploading build results to GitHub requires at least these steps: +1. Create a new release on GitHub with content +2. Upload all binaries to the release +3. Publish the release +The below scripts help with these steps. + +## github-draft.py +Creates a GitHub release and fills the content with the CHANGELOG.md link. The version number can be changed by the `--version` parameter. + +By default, the command will use the tendermint/tendermint organization/repo, which can be changed using the `--org` and `--repo` parameters. + +By default, the command will get the version number from the `${CIRCLE_TAG}` variable. + +Returns the GitHub release ID. + +## github-upload.py +Upload a file to a GitHub release. The release is defined by the mandatory `--id` (release ID) input parameter. + +By default, the command will upload the file `/tmp/workspace/tendermint_${CIRCLE_TAG}_${GOOS}_${GOARCH}.zip`. This can be changed by the `--file` input parameter. + +## github-publish.py +Publish a GitHub release. The release is defined by the mandatory `--id` (release ID) input parameter. + diff --git a/scripts/release_management/bump-semver.py b/scripts/release_management/bump-semver.py new file mode 100755 index 000000000..b13a10342 --- /dev/null +++ b/scripts/release_management/bump-semver.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python + +# Bump the release number of a semantic version number and print it. --version is required. +# Version is +# - vA.B.C, in which case vA.B.C+1 will be returned +# - vA.B.C-devorwhatnot in which case vA.B.C will be returned +# - vA.B in which case vA.B.0 will be returned + +import re +import argparse + + +def semver(ver): + if re.match('v[0-9]+\.[0-9]+',ver) is None: + ver="v0.0" + #raise argparse.ArgumentTypeError('--version must be a semantic version number with major, minor and patch numbers') + return ver + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--version", help="Version number to bump, e.g.: v1.0.0", required=True, type=semver) + args = parser.parse_args() + + found = re.match('(v[0-9]+\.[0-9]+)(\.(.+))?', args.version) + majorminorprefix = found.group(1) + patch = found.group(3) + if patch is None: + patch = "0-new" + + if re.match('[0-9]+$',patch) is None: + patchfound = re.match('([0-9]+)',patch) + patch = int(patchfound.group(1)) + else: + patch = int(patch) + 1 + + print("{0}.{1}".format(majorminorprefix, patch)) diff --git a/scripts/release_management/github-draft.py b/scripts/release_management/github-draft.py new file mode 100755 index 000000000..8a189d53e --- /dev/null +++ b/scripts/release_management/github-draft.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python + +# Create a draft release on GitHub. By default in the tendermint/tendermint repo. +# Optimized for CircleCI + +import argparse +import httplib +import json +import os +from base64 import b64encode + +def request(org, repo, data): + user_and_pass = b64encode(b"{0}:{1}".format(os.environ['GITHUB_USERNAME'], os.environ['GITHUB_TOKEN'])).decode("ascii") + headers = { + 'User-Agent': 'tenderbot', + 'Accept': 'application/vnd.github.v3+json', + 'Authorization': 'Basic %s' % user_and_pass + } + + conn = httplib.HTTPSConnection('api.github.com', timeout=5) + conn.request('POST', '/repos/{0}/{1}/releases'.format(org,repo), data, headers) + response = conn.getresponse() + if response.status < 200 or response.status > 299: + print("{0}: {1}".format(response.status, response.reason)) + conn.close() + raise IOError(response.reason) + responsedata = response.read() + conn.close() + return json.loads(responsedata) + + +def create_draft(org,repo,branch,version): + draft = { + 'tag_name': version, + 'target_commitish': '{0}'.format(branch), + 'name': '{0} (WARNING: ALPHA SOFTWARE)'.format(version), + 'body': 'https://github.com/{0}/{1}/blob/{2}/CHANGELOG.md#{3}'.format(org,repo,branch,version.replace('.','')), + 'draft': True, + 'prerelease': False + } + data=json.dumps(draft) + return request(org, repo, data) + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--org", default="tendermint", help="GitHub organization") + parser.add_argument("--repo", default="tendermint", help="GitHub repository") + parser.add_argument("--branch", default=os.environ.get('CIRCLE_BRANCH'), help="Branch to build from, e.g.: v1.0") + parser.add_argument("--version", default=os.environ.get('CIRCLE_TAG'), help="Version number for binary, e.g.: v1.0.0") + args = parser.parse_args() + + if not os.environ.has_key('GITHUB_USERNAME'): + raise parser.error('environment variable GITHUB_USERNAME is required') + + if not os.environ.has_key('GITHUB_TOKEN'): + raise parser.error('environment variable GITHUB_TOKEN is required') + + release = create_draft(args.org,args.repo,args.branch,args.version) + + print(release["id"]) + diff --git a/scripts/release_management/github-openpr.py b/scripts/release_management/github-openpr.py new file mode 100755 index 000000000..af0434f02 --- /dev/null +++ b/scripts/release_management/github-openpr.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python + +# Open a PR against the develop branch. --branch required. +# Optimized for CircleCI + +import json +import os +import argparse +import httplib +from base64 import b64encode + + +def request(org, repo, data): + user_and_pass = b64encode(b"{0}:{1}".format(os.environ['GITHUB_USERNAME'], os.environ['GITHUB_TOKEN'])).decode("ascii") + headers = { + 'User-Agent': 'tenderbot', + 'Accept': 'application/vnd.github.v3+json', + 'Authorization': 'Basic %s' % user_and_pass + } + + conn = httplib.HTTPSConnection('api.github.com', timeout=5) + conn.request('POST', '/repos/{0}/{1}/pulls'.format(org,repo), data, headers) + response = conn.getresponse() + if response.status < 200 or response.status > 299: + print(response) + conn.close() + raise IOError(response.reason) + responsedata = response.read() + conn.close() + return json.loads(responsedata) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--org", default="tendermint", help="GitHub organization. Defaults to tendermint.") + parser.add_argument("--repo", default="tendermint", help="GitHub repository. Defaults to tendermint.") + parser.add_argument("--head", help="The name of the branch where your changes are implemented.", required=True) + parser.add_argument("--base", help="The name of the branch you want the changes pulled into.", required=True) + parser.add_argument("--title", default="Security release {0}".format(os.environ.get('CIRCLE_TAG')), help="The title of the pull request.") + args = parser.parse_args() + + if not os.environ.has_key('GITHUB_USERNAME'): + raise parser.error('GITHUB_USERNAME not set.') + + if not os.environ.has_key('GITHUB_TOKEN'): + raise parser.error('GITHUB_TOKEN not set.') + + if os.environ.get('CIRCLE_TAG') is None: + raise parser.error('CIRCLE_TAG not set.') + + result = request(args.org, args.repo, data=json.dumps({'title':"{0}".format(args.title),'head':"{0}".format(args.head),'base':"{0}".format(args.base),'body':""})) + print(result['html_url']) diff --git a/scripts/release_management/github-public-newbranch.bash b/scripts/release_management/github-public-newbranch.bash new file mode 100644 index 000000000..ca2fa1314 --- /dev/null +++ b/scripts/release_management/github-public-newbranch.bash @@ -0,0 +1,28 @@ +#!/bin/sh + +# github-public-newbranch.bash - create public branch from the security repository + +set -euo pipefail + +# Create new branch +BRANCH="${CIRCLE_TAG:-v0.0.0}-security-`date -u +%Y%m%d%H%M%S`" +# Check if the patch release exist already as a branch +if [ -n "`git branch | grep '${BRANCH}'`" ]; then + echo "WARNING: Branch ${BRANCH} already exists." +else + echo "Creating branch ${BRANCH}." + git branch "${BRANCH}" +fi + +# ... and check it out +git checkout "${BRANCH}" + +# Add entry to public repository +git remote add tendermint-origin git@github.com:tendermint/tendermint.git + +# Push branch and tag to public repository +git push tendermint-origin +git push tendermint-origin --tags + +# Create a PR from the public branch to the assumed release branch in public (release branch has to exist) +python -u scripts/release_management/github-openpr.py --head "${BRANCH}" --base "${BRANCH:%.*}" diff --git a/scripts/release_management/github-publish.py b/scripts/release_management/github-publish.py new file mode 100755 index 000000000..31071aecd --- /dev/null +++ b/scripts/release_management/github-publish.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python + +# Publish an existing GitHub draft release. --id required. +# Optimized for CircleCI + +import json +import os +import argparse +import httplib +from base64 import b64encode + + +def request(org, repo, id, data): + user_and_pass = b64encode(b"{0}:{1}".format(os.environ['GITHUB_USERNAME'], os.environ['GITHUB_TOKEN'])).decode("ascii") + headers = { + 'User-Agent': 'tenderbot', + 'Accept': 'application/vnd.github.v3+json', + 'Authorization': 'Basic %s' % user_and_pass + } + + conn = httplib.HTTPSConnection('api.github.com', timeout=5) + conn.request('POST', '/repos/{0}/{1}/releases/{2}'.format(org,repo,id), data, headers) + response = conn.getresponse() + if response.status < 200 or response.status > 299: + print(response) + conn.close() + raise IOError(response.reason) + responsedata = response.read() + conn.close() + return json.loads(responsedata) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--org", default="tendermint", help="GitHub organization") + parser.add_argument("--repo", default="tendermint", help="GitHub repository") + parser.add_argument("--id", help="GitHub release ID", required=True, type=int) + parser.add_argument("--version", default=os.environ.get('CIRCLE_TAG'), help="Version number for the release, e.g.: v1.0.0") + args = parser.parse_args() + + if not os.environ.has_key('GITHUB_USERNAME'): + raise parser.error('GITHUB_USERNAME not set.') + + if not os.environ.has_key('GITHUB_TOKEN'): + raise parser.error('GITHUB_TOKEN not set.') + + try: + result = request(args.org, args.repo, args.id, data=json.dumps({'draft':False,'tag_name':"{0}".format(args.version)})) + except IOError as e: + print(e) + result = request(args.org, args.repo, args.id, data=json.dumps({'draft':False,'tag_name':"{0}-autorelease".format(args.version)})) + + print(result['name']) diff --git a/scripts/release_management/github-upload.py b/scripts/release_management/github-upload.py new file mode 100755 index 000000000..77c76a755 --- /dev/null +++ b/scripts/release_management/github-upload.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python + +# Upload a file to a GitHub draft release. --id and --file are required. +# Optimized for CircleCI + +import json +import os +import re +import argparse +import mimetypes +import httplib +from base64 import b64encode + + +def request(baseurl, path, mimetype, mimeencoding, data): + user_and_pass = b64encode(b"{0}:{1}".format(os.environ['GITHUB_USERNAME'], os.environ['GITHUB_TOKEN'])).decode("ascii") + + headers = { + 'User-Agent': 'tenderbot', + 'Accept': 'application/vnd.github.v3.raw+json', + 'Authorization': 'Basic %s' % user_and_pass, + 'Content-Type': mimetype, + 'Content-Encoding': mimeencoding + } + + conn = httplib.HTTPSConnection(baseurl, timeout=5) + conn.request('POST', path, data, headers) + response = conn.getresponse() + if response.status < 200 or response.status > 299: + print(response) + conn.close() + raise IOError(response.reason) + responsedata = response.read() + conn.close() + return json.loads(responsedata) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--id", help="GitHub release ID", required=True, type=int) + parser.add_argument("--file", default="/tmp/workspace/tendermint_{0}_{1}_{2}.zip".format(os.environ.get('CIRCLE_TAG'),os.environ.get('GOOS'),os.environ.get('GOARCH')), help="File to upload") + parser.add_argument("--return-id-only", help="Return only the release ID after upload to GitHub.", action='store_true') + args = parser.parse_args() + + if not os.environ.has_key('GITHUB_USERNAME'): + raise parser.error('GITHUB_USERNAME not set.') + + if not os.environ.has_key('GITHUB_TOKEN'): + raise parser.error('GITHUB_TOKEN not set.') + + mimetypes.init() + filename = os.path.basename(args.file) + mimetype,mimeencoding = mimetypes.guess_type(filename, strict=False) + if mimetype is None: + mimetype = 'application/zip' + if mimeencoding is None: + mimeencoding = 'utf8' + + with open(args.file,'rb') as f: + asset = f.read() + + result = request('uploads.github.com', '/repos/tendermint/tendermint/releases/{0}/assets?name={1}'.format(args.id, filename), mimetype, mimeencoding, asset) + + if args.return_id_only: + print(result['id']) + else: + print(result['browser_download_url']) + diff --git a/scripts/release_management/sha-files.py b/scripts/release_management/sha-files.py new file mode 100755 index 000000000..2a9ee0d59 --- /dev/null +++ b/scripts/release_management/sha-files.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python + +# Create SHA256 summaries from all ZIP files in a folder +# Optimized for CircleCI + +import re +import os +import argparse +import zipfile +import hashlib + + +BLOCKSIZE = 65536 + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--folder", default="/tmp/workspace", help="Folder to look for, for ZIP files") + parser.add_argument("--shafile", default="/tmp/workspace/SHA256SUMS", help="SHA256 summaries File") + args = parser.parse_args() + + for filename in os.listdir(args.folder): + if re.search('\.zip$',filename) is None: + continue + if not os.path.isfile(os.path.join(args.folder, filename)): + continue + with open(args.shafile,'a+') as shafile: + hasher = hashlib.sha256() + with open(os.path.join(args.folder, filename),'r') as f: + buf = f.read(BLOCKSIZE) + while len(buf) > 0: + hasher.update(buf) + buf = f.read(BLOCKSIZE) + shafile.write("{0} {1}\n".format(hasher.hexdigest(),filename)) + diff --git a/scripts/release_management/zip-file.py b/scripts/release_management/zip-file.py new file mode 100755 index 000000000..5d2f5b2c8 --- /dev/null +++ b/scripts/release_management/zip-file.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python + +# ZIP one file as "tendermint" into a ZIP like tendermint_VERSION_OS_ARCH.zip +# Use environment variables CIRCLE_TAG, GOOS and GOARCH for easy input parameters. +# Optimized for CircleCI + +import os +import argparse +import zipfile +import hashlib + + +BLOCKSIZE = 65536 + + +def zip_asset(file,destination,arcname,version,goos,goarch): + filename = os.path.basename(file) + output = "{0}/{1}_{2}_{3}_{4}.zip".format(destination,arcname,version,goos,goarch) + + with zipfile.ZipFile(output,'w') as f: + f.write(filename=file,arcname=arcname) + f.comment=filename + return output + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--file", default="build/tendermint_{0}_{1}".format(os.environ.get('GOOS'),os.environ.get('GOARCH')), help="File to zip") + parser.add_argument("--destination", default="build", help="Destination folder for files") + parser.add_argument("--version", default=os.environ.get('CIRCLE_TAG'), help="Version number for binary, e.g.: v1.0.0") + parser.add_argument("--goos", default=os.environ.get('GOOS'), help="GOOS parameter") + parser.add_argument("--goarch", default=os.environ.get('GOARCH'), help="GOARCH parameter") + args = parser.parse_args() + + if args.version is None: + raise parser.error("argument --version is required") + if args.goos is None: + raise parser.error("argument --goos is required") + if args.goarch is None: + raise parser.error("argument --goarch is required") + + file = zip_asset(args.file,args.destination,"tendermint",args.version,args.goos,args.goarch) + print(file) + From 1fb900f9f53724e6b0b2d0433301a92f0ae52168 Mon Sep 17 00:00:00 2001 From: Ismail Khoffi Date: Tue, 2 Apr 2019 21:49:05 +0200 Subject: [PATCH 5/5] changelog and version v0.30.3 --- CHANGELOG.md | 24 +++++++++++++++++++++++- version/version.go | 2 +- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 44ecdf388..5e1a9d267 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,27 @@ # Changelog +## v0.30.3 + +*April 1st, 2019* + +This release includes two security sensitive fixes: it ensures generated private +keys are valid, and it prevents certain DNS lookups that would cause the node to +panic if the lookup failed. + +### BUG FIXES: + +- [crypto/secp256k1] [\#3439](https://github.com/tendermint/tendermint/issues/3439) + Ensure generated private keys are valid by randomly sampling until a valid key is found. + Previously, it was possible (though rare!) to generate keys that exceeded the curve order. + Such keys would lead to invalid signatures. +- [p2p] [\#3522](https://github.com/tendermint/tendermint/issues/3522) Memoize + socket address in peer connections to avoid DNS lookups. Previously, failed + DNS lookups could cause the node to panic. + +### IMPROVEMENTS: + +- [circle] [\#3497](https://github.com/tendermint/tendermint/issues/3497) Move release management to CircleCI + ## v0.30.2 *March 10th, 2019* @@ -14,7 +36,7 @@ fix here. ### BREAKING CHANGES: * Go API -- [libs/db] [\#3842](https://github.com/cosmos/cosmos-sdk/issues/3842) Add Close() method to Batch interface (@Stumble) + - [libs/db] [\#3842](https://github.com/cosmos/cosmos-sdk/issues/3842) Add Close() method to Batch interface (@Stumble) ### BUG FIXES: - [libs/db] [\#3842](https://github.com/cosmos/cosmos-sdk/issues/3842) Fix CLevelDB memory leak (@Stumble) diff --git a/version/version.go b/version/version.go index 1b0a36ae1..ab9c4ab35 100644 --- a/version/version.go +++ b/version/version.go @@ -20,7 +20,7 @@ const ( // Must be a string because scripts like dist.sh read this file. // XXX: Don't change the name of this variable or you will break // automation :) - TMCoreSemVer = "0.30.2" + TMCoreSemVer = "0.30.3" // ABCISemVer is the semantic version of the ABCI library ABCISemVer = "0.15.0"