diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 35a27cfe7..950ed1dc2 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -10,4 +10,4 @@ * @ebuchman @cmwaters @tychoish @williambanfield @creachadair @sergio-mena @jmalicevic @thanethomson @ancazamfir # Spec related changes can be approved by the protocol design team -/spec @josef-widder @milosevic @cason +/spec @josef-widder @milosevic @cason @sergio-mena @jmalicevic diff --git a/.github/workflows/janitor.yml b/.github/workflows/janitor.yml index e6bc45ec1..ceb21941d 100644 --- a/.github/workflows/janitor.yml +++ b/.github/workflows/janitor.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 3 steps: - - uses: styfle/cancel-workflow-action@0.9.1 + - uses: styfle/cancel-workflow-action@0.10.0 with: workflow_id: 1041851,1401230,2837803 access_token: ${{ github.token }} diff --git a/.github/workflows/proto-lint.yml b/.github/workflows/proto-lint.yml index f8fff89cc..377e43ca6 100644 --- a/.github/workflows/proto-lint.yml +++ b/.github/workflows/proto-lint.yml @@ -15,7 +15,7 @@ jobs: timeout-minutes: 5 steps: - uses: actions/checkout@v3 - - uses: bufbuild/buf-setup-action@v1.5.0 + - uses: bufbuild/buf-setup-action@v1.6.0 - uses: bufbuild/buf-lint-action@v1 with: input: 'proto' diff --git a/config/config.go b/config/config.go index 804c5fc87..230fed98c 100644 --- a/config/config.go +++ b/config/config.go @@ -671,7 +671,7 @@ func DefaultP2PConfig() *P2PConfig { ExternalAddress: "", UPNP: false, MaxConnections: 64, - MaxOutgoingConnections: 32, + MaxOutgoingConnections: 12, MaxIncomingConnectionAttempts: 100, FlushThrottleTimeout: 100 * time.Millisecond, // The MTU (Maximum Transmission Unit) for Ethernet is 1500 bytes. diff --git a/go.mod b/go.mod index 8dd4c9288..691efb09f 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/snikch/goodman v0.0.0-20171125024755-10e37e294daa github.com/spf13/cobra v1.5.0 github.com/spf13/viper v1.12.0 - github.com/stretchr/testify v1.7.2 + github.com/stretchr/testify v1.8.0 github.com/tendermint/tm-db v0.6.6 golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e golang.org/x/net v0.0.0-20220617184016-355a448f1bc9 @@ -42,7 +42,7 @@ require ( github.com/creachadair/taskgroup v0.3.2 github.com/golangci/golangci-lint v1.46.0 github.com/google/go-cmp v0.5.8 - github.com/vektra/mockery/v2 v2.13.1 + github.com/vektra/mockery/v2 v2.14.0 gotest.tools v2.2.0+incompatible ) @@ -199,7 +199,7 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/ssgreg/nlreturn/v2 v2.2.1 // indirect github.com/stbenjam/no-sprintf-host-port v0.1.1 // indirect - github.com/stretchr/objx v0.1.1 // indirect + github.com/stretchr/objx v0.4.0 // indirect github.com/subosito/gotenv v1.4.0 // indirect github.com/sylvia7788/contextcheck v1.0.4 // indirect github.com/tdakkota/asciicheck v0.1.1 // indirect diff --git a/go.sum b/go.sum index bdb93f91e..ca88c3e8f 100644 --- a/go.sum +++ b/go.sum @@ -1064,8 +1064,9 @@ github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3 github.com/streadway/amqp v1.0.0/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/handy v0.0.0-20200128134331-0f66f006fb2e/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v0.0.0-20170130113145-4d4bfba8f1d1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.1.4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -1075,8 +1076,9 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/subosito/gotenv v1.3.0/go.mod h1:YzJjq/33h7nrwdY+iHMhEOEEbW0ovIz0tB6t6PwAXzs= github.com/subosito/gotenv v1.4.0 h1:yAzM1+SmVcz5R4tXGsNMu1jUl2aOJXoiWUCEwwnGrvs= @@ -1124,8 +1126,8 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC github.com/valyala/fasthttp v1.30.0/go.mod h1:2rsYD01CKFrjjsvFxx75KlEUNpWNBY9JWD3K/7o2Cus= github.com/valyala/quicktemplate v1.7.0/go.mod h1:sqKJnoaOF88V07vkO+9FL8fb9uZg/VPSJnLYn+LmLk8= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= -github.com/vektra/mockery/v2 v2.13.1 h1:Lqs7aZiC7TwZO76fJ/4Zsb3NaO4F7cuuz0mZLYeNwtQ= -github.com/vektra/mockery/v2 v2.13.1/go.mod h1:bnD1T8tExSgPD1ripLkDbr60JA9VtQeu12P3wgLZd7M= +github.com/vektra/mockery/v2 v2.14.0 h1:KZ1p5Hrn8tiY+LErRMr14HHle6khxo+JKOXLBW/yfqs= +github.com/vektra/mockery/v2 v2.14.0/go.mod h1:bnD1T8tExSgPD1ripLkDbr60JA9VtQeu12P3wgLZd7M= github.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8/go.mod h1:dniwbG03GafCjFohMDmz6Zc6oCuiqgH6tGNyXTkHzXE= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= diff --git a/internal/inspect/inspect.go b/internal/inspect/inspect.go index 6381ea888..573b63f40 100644 --- a/internal/inspect/inspect.go +++ b/internal/inspect/inspect.go @@ -10,13 +10,13 @@ import ( "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/internal/eventbus" "github.com/tendermint/tendermint/internal/inspect/rpc" + tmstrings "github.com/tendermint/tendermint/internal/libs/strings" rpccore "github.com/tendermint/tendermint/internal/rpc/core" "github.com/tendermint/tendermint/internal/state" "github.com/tendermint/tendermint/internal/state/indexer" "github.com/tendermint/tendermint/internal/state/indexer/sink" "github.com/tendermint/tendermint/internal/store" "github.com/tendermint/tendermint/libs/log" - tmstrings "github.com/tendermint/tendermint/libs/strings" "github.com/tendermint/tendermint/types" "golang.org/x/sync/errgroup" diff --git a/libs/strings/string.go b/internal/libs/strings/string.go similarity index 100% rename from libs/strings/string.go rename to internal/libs/strings/string.go diff --git a/libs/strings/string_test.go b/internal/libs/strings/string_test.go similarity index 100% rename from libs/strings/string_test.go rename to internal/libs/strings/string_test.go diff --git a/internal/p2p/conn/connection.go b/internal/p2p/conn/connection.go index c8fc21188..8f8453e71 100644 --- a/internal/p2p/conn/connection.go +++ b/internal/p2p/conn/connection.go @@ -100,7 +100,8 @@ type MConnection struct { // used to ensure FlushStop and OnStop // are safe to call concurrently. - stopMtx sync.Mutex + stopMtx sync.Mutex + stopSignal <-chan struct{} cancel context.CancelFunc @@ -207,6 +208,7 @@ func (c *MConnection) OnStart(ctx context.Context) error { c.quitSendRoutine = make(chan struct{}) c.doneSendRoutine = make(chan struct{}) c.quitRecvRoutine = make(chan struct{}) + c.stopSignal = ctx.Done() c.setRecvLastMsgAt(time.Now()) go c.sendRoutine(ctx) go c.recvRoutine(ctx) @@ -681,6 +683,8 @@ func (ch *channel) sendBytes(bytes []byte) bool { return true case <-time.After(defaultSendTimeout): return false + case <-ch.conn.stopSignal: + return false } } diff --git a/internal/p2p/p2ptest/network.go b/internal/p2p/p2ptest/network.go index 5c1b0a218..813344915 100644 --- a/internal/p2p/p2ptest/network.go +++ b/internal/p2p/p2ptest/network.go @@ -252,12 +252,13 @@ func (n *Network) MakeNode(ctx context.Context, t *testing.T, opts NodeOptions) require.NotNil(t, ep, "transport not listening an endpoint") peerManager, err := p2p.NewPeerManager(nodeID, dbm.NewMemDB(), p2p.PeerManagerOptions{ - MinRetryTime: 10 * time.Millisecond, - MaxRetryTime: 100 * time.Millisecond, - RetryTimeJitter: time.Millisecond, - MaxPeers: opts.MaxPeers, - MaxConnected: opts.MaxConnected, - Metrics: p2p.NopMetrics(), + MinRetryTime: 10 * time.Millisecond, + DisconnectCooldownPeriod: 10 * time.Millisecond, + MaxRetryTime: 100 * time.Millisecond, + RetryTimeJitter: time.Millisecond, + MaxPeers: opts.MaxPeers, + MaxConnected: opts.MaxConnected, + Metrics: p2p.NopMetrics(), }) require.NoError(t, err) @@ -269,7 +270,7 @@ func (n *Network) MakeNode(ctx context.Context, t *testing.T, opts NodeOptions) func() *types.NodeInfo { return &nodeInfo }, transport, ep, - p2p.RouterOptions{DialSleep: func(_ context.Context) {}}, + p2p.RouterOptions{}, ) require.NoError(t, err) diff --git a/internal/p2p/peermanager.go b/internal/p2p/peermanager.go index b30c5dbfb..8230b262e 100644 --- a/internal/p2p/peermanager.go +++ b/internal/p2p/peermanager.go @@ -144,6 +144,10 @@ type PeerManagerOptions struct { // retry times, to avoid thundering herds. 0 disables jitter. RetryTimeJitter time.Duration + // DisconnectCooldownPeriod is the amount of time after we + // disconnect from a peer before we'll consider dialing a new peer + DisconnectCooldownPeriod time.Duration + // PeerScores sets fixed scores for specific peers. It is mainly used // for testing. A score of 0 is ignored. PeerScores map[types.NodeID]PeerScore @@ -217,11 +221,11 @@ func (o *PeerManagerOptions) Validate() error { return nil } -// isPersistentPeer checks if a peer is in PersistentPeers. It will panic +// isPersistent checks if a peer is in PersistentPeers. It will panic // if called before optimize(). func (o *PeerManagerOptions) isPersistent(id types.NodeID) bool { if o.persistentPeers == nil { - panic("isPersistentPeer() called before optimize()") + panic("isPersistent() called before optimize()") } return o.persistentPeers[id] } @@ -549,6 +553,10 @@ func (m *PeerManager) TryDialNext() NodeAddress { continue } + if !peer.LastDisconnected.IsZero() && time.Since(peer.LastDisconnected) < m.options.DisconnectCooldownPeriod { + continue + } + for _, addressInfo := range peer.AddressInfo { if time.Since(addressInfo.LastDialFailure) < m.retryDelay(addressInfo.DialFailures, peer.Persistent) { continue @@ -867,6 +875,22 @@ func (m *PeerManager) Disconnected(ctx context.Context, peerID types.NodeID) { delete(m.evicting, peerID) delete(m.ready, peerID) + if peer, ok := m.store.Get(peerID); ok { + peer.LastDisconnected = time.Now() + _ = m.store.Set(peer) + // launch a thread to ping the dialWaker when the + // disconnected peer can be dialed again. + go func() { + timer := time.NewTimer(m.options.DisconnectCooldownPeriod) + defer timer.Stop() + select { + case <-timer.C: + m.dialWaker.Wake() + case <-ctx.Done(): + } + }() + } + if ready { m.broadcast(ctx, PeerUpdate{ NodeID: peerID, @@ -1447,9 +1471,10 @@ func (s *peerStore) Size() int { // peerInfo contains peer information stored in a peerStore. type peerInfo struct { - ID types.NodeID - AddressInfo map[NodeAddress]*peerAddressInfo - LastConnected time.Time + ID types.NodeID + AddressInfo map[NodeAddress]*peerAddressInfo + LastConnected time.Time + LastDisconnected time.Time // These fields are ephemeral, i.e. not persisted to the database. Persistent bool @@ -1489,8 +1514,8 @@ func peerInfoFromProto(msg *p2pproto.PeerInfo) (*peerInfo, error) { func (p *peerInfo) ToProto() *p2pproto.PeerInfo { msg := &p2pproto.PeerInfo{ ID: string(p.ID), - LastConnected: &p.LastConnected, Inactive: p.Inactive, + LastConnected: &p.LastConnected, } for _, addressInfo := range p.AddressInfo { msg.AddressInfo = append(msg.AddressInfo, addressInfo.ToProto()) @@ -1498,6 +1523,7 @@ func (p *peerInfo) ToProto() *p2pproto.PeerInfo { if msg.LastConnected.IsZero() { msg.LastConnected = nil } + return msg } diff --git a/internal/p2p/router.go b/internal/p2p/router.go index d5edd42be..8b7db9a03 100644 --- a/internal/p2p/router.go +++ b/internal/p2p/router.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "io" - "math/rand" "net" "runtime" "sync" @@ -62,13 +61,7 @@ type RouterOptions struct { // return an error to reject the peer. FilterPeerByID func(context.Context, types.NodeID) error - // DialSleep controls the amount of time that the router - // sleeps between dialing peers. If not set, a default value - // is used that sleeps for a (random) amount of time up to 3 - // seconds between submitting each peer to be dialed. - DialSleep func(context.Context) - - // NumConcurrentDials controls how many parallel go routines + // NumConcrruentDials controls how many parallel go routines // are used to dial peers. This defaults to the value of // runtime.NumCPU. NumConcurrentDials func() int @@ -439,33 +432,6 @@ func (r *Router) filterPeersID(ctx context.Context, id types.NodeID) error { return r.options.FilterPeerByID(ctx, id) } -func (r *Router) dialSleep(ctx context.Context) { - if r.options.DialSleep == nil { - // the connTracker (on the other side) only rate - // limits peers for dialing more than once every 10ms, - // so these numbers are safe. - const ( - maxDialerInterval = 500 // ms - minDialerInterval = 100 // ms - ) - - // nolint:gosec // G404: Use of weak random number generator - dur := time.Duration(rand.Int63n(maxDialerInterval-minDialerInterval+1) + minDialerInterval) - - timer := time.NewTimer(dur * time.Millisecond) - defer timer.Stop() - - select { - case <-ctx.Done(): - case <-timer.C: - } - - return - } - - r.options.DialSleep(ctx) -} - // acceptPeers accepts inbound connections from peers on the given transport, // and spawns goroutines that route messages to/from them. func (r *Router) acceptPeers(ctx context.Context, transport Transport) { @@ -590,11 +556,6 @@ LOOP: select { case addresses <- address: - // this jitters the frequency that we call - // DialNext and prevents us from attempting to - // create connections too quickly. - - r.dialSleep(ctx) continue LOOP case <-ctx.Done(): close(addresses) diff --git a/internal/p2p/router_test.go b/internal/p2p/router_test.go index 8a58146fd..92f56f768 100644 --- a/internal/p2p/router_test.go +++ b/internal/p2p/router_test.go @@ -715,7 +715,6 @@ func TestRouter_DialPeers_Parallel(t *testing.T) { mockTransport, nil, p2p.RouterOptions{ - DialSleep: func(_ context.Context) {}, NumConcurrentDials: func() int { ncpu := runtime.NumCPU() if ncpu <= 3 { diff --git a/internal/p2p/transport_mconn.go b/internal/p2p/transport_mconn.go index 13a65b973..523fbf89f 100644 --- a/internal/p2p/transport_mconn.go +++ b/internal/p2p/transport_mconn.go @@ -315,7 +315,7 @@ func (c *mConnConnection) Handshake( select { case <-handshakeCtx.Done(): _ = c.Close() - return types.NodeInfo{}, nil, ctx.Err() + return types.NodeInfo{}, nil, handshakeCtx.Err() case err := <-errCh: if err != nil { diff --git a/internal/rpc/core/env.go b/internal/rpc/core/env.go index 124525f26..f1ba58ee3 100644 --- a/internal/rpc/core/env.go +++ b/internal/rpc/core/env.go @@ -18,6 +18,7 @@ import ( "github.com/tendermint/tendermint/internal/consensus" "github.com/tendermint/tendermint/internal/eventbus" "github.com/tendermint/tendermint/internal/eventlog" + "github.com/tendermint/tendermint/internal/libs/strings" "github.com/tendermint/tendermint/internal/mempool" "github.com/tendermint/tendermint/internal/p2p" tmpubsub "github.com/tendermint/tendermint/internal/pubsub" @@ -26,7 +27,6 @@ import ( "github.com/tendermint/tendermint/internal/state/indexer" "github.com/tendermint/tendermint/internal/statesync" "github.com/tendermint/tendermint/libs/log" - "github.com/tendermint/tendermint/libs/strings" "github.com/tendermint/tendermint/rpc/coretypes" rpcserver "github.com/tendermint/tendermint/rpc/jsonrpc/server" "github.com/tendermint/tendermint/types" diff --git a/libs/log/default.go b/libs/log/default.go index 706977659..557ba6551 100644 --- a/libs/log/default.go +++ b/libs/log/default.go @@ -61,20 +61,20 @@ func NewDefaultLogger(format, level string) (Logger, error) { } func (l defaultLogger) Info(msg string, keyVals ...interface{}) { - l.Logger.Info().Fields(getLogFields(keyVals...)).Msg(msg) + l.Logger.Info().Fields(keyVals).Msg(msg) } func (l defaultLogger) Error(msg string, keyVals ...interface{}) { - l.Logger.Error().Fields(getLogFields(keyVals...)).Msg(msg) + l.Logger.Error().Fields(keyVals).Msg(msg) } func (l defaultLogger) Debug(msg string, keyVals ...interface{}) { - l.Logger.Debug().Fields(getLogFields(keyVals...)).Msg(msg) + l.Logger.Debug().Fields(keyVals).Msg(msg) } func (l defaultLogger) With(keyVals ...interface{}) Logger { return &defaultLogger{ - Logger: l.Logger.With().Fields(getLogFields(keyVals...)).Logger(), + Logger: l.Logger.With().Fields(keyVals).Logger(), } } @@ -99,16 +99,3 @@ func OverrideWithNewLogger(logger Logger, format, level string) error { ol.Logger = nl.Logger return nil } - -func getLogFields(keyVals ...interface{}) map[string]interface{} { - if len(keyVals)%2 != 0 { - return nil - } - - fields := make(map[string]interface{}, len(keyVals)) - for i := 0; i < len(keyVals); i += 2 { - fields[fmt.Sprint(keyVals[i])] = keyVals[i+1] - } - - return fields -} diff --git a/light/client.go b/light/client.go index dfb4fccf7..f38e9d59d 100644 --- a/light/client.go +++ b/light/client.go @@ -1034,7 +1034,12 @@ func (c *Client) findNewPrimary(ctx context.Context, height int64, remove bool) // process all the responses as they come in for i := 0; i < cap(witnessResponsesC); i++ { - response := <-witnessResponsesC + var response witnessResponse + select { + case response = <-witnessResponsesC: + case <-ctx.Done(): + return nil, ctx.Err() + } switch response.err { // success! We have found a new primary case nil: @@ -1063,10 +1068,6 @@ func (c *Client) findNewPrimary(ctx context.Context, height int64, remove bool) // return the light block that new primary responded with return response.lb, nil - // catch canceled contexts or deadlines - case context.Canceled, context.DeadlineExceeded: - return nil, response.err - // process benign errors by logging them only case provider.ErrNoResponse, provider.ErrLightBlockNotFound, provider.ErrHeightTooHigh: lastError = response.err diff --git a/node/setup.go b/node/setup.go index 345489f8d..38ad58518 100644 --- a/node/setup.go +++ b/node/setup.go @@ -17,6 +17,7 @@ import ( "github.com/tendermint/tendermint/internal/consensus" "github.com/tendermint/tendermint/internal/eventbus" "github.com/tendermint/tendermint/internal/evidence" + tmstrings "github.com/tendermint/tendermint/internal/libs/strings" "github.com/tendermint/tendermint/internal/mempool" "github.com/tendermint/tendermint/internal/p2p" "github.com/tendermint/tendermint/internal/p2p/conn" @@ -28,7 +29,6 @@ import ( "github.com/tendermint/tendermint/libs/log" tmnet "github.com/tendermint/tendermint/libs/net" "github.com/tendermint/tendermint/libs/service" - tmstrings "github.com/tendermint/tendermint/libs/strings" "github.com/tendermint/tendermint/privval" tmgrpc "github.com/tendermint/tendermint/privval/grpc" "github.com/tendermint/tendermint/types" @@ -235,17 +235,18 @@ func createPeerManager( maxUpgradeConns := uint16(4) options := p2p.PeerManagerOptions{ - SelfAddress: selfAddr, - MaxConnected: maxConns, - MaxOutgoingConnections: maxOutgoingConns, - MaxConnectedUpgrade: maxUpgradeConns, - MaxPeers: maxUpgradeConns + 4*maxConns, - MinRetryTime: 250 * time.Millisecond, - MaxRetryTime: 30 * time.Minute, - MaxRetryTimePersistent: 5 * time.Minute, - RetryTimeJitter: 5 * time.Second, - PrivatePeers: privatePeerIDs, - Metrics: metrics, + SelfAddress: selfAddr, + MaxConnected: maxConns, + MaxOutgoingConnections: maxOutgoingConns, + MaxConnectedUpgrade: maxUpgradeConns, + DisconnectCooldownPeriod: 2 * time.Second, + MaxPeers: maxUpgradeConns + 4*maxConns, + MinRetryTime: 250 * time.Millisecond, + MaxRetryTime: 30 * time.Minute, + MaxRetryTimePersistent: 5 * time.Minute, + RetryTimeJitter: 5 * time.Second, + PrivatePeers: privatePeerIDs, + Metrics: metrics, } peers := []p2p.NodeAddress{} diff --git a/test/e2e/generator/main.go b/test/e2e/generator/main.go index bec78d89c..da32b2831 100644 --- a/test/e2e/generator/main.go +++ b/test/e2e/generator/main.go @@ -1,4 +1,3 @@ -//nolint: gosec package main import ( @@ -77,6 +76,8 @@ func (cli *CLI) generate() error { return err } + // nolint: gosec + // G404: Use of weak random number generator (math/rand instead of crypto/rand) manifests, err := Generate(rand.New(rand.NewSource(randomSeed)), cli.opts) if err != nil { return err diff --git a/test/e2e/pkg/exec/exec.go b/test/e2e/pkg/exec/exec.go new file mode 100644 index 000000000..9dcd79384 --- /dev/null +++ b/test/e2e/pkg/exec/exec.go @@ -0,0 +1,34 @@ +package exec + +import ( + "context" + "fmt" + "os" + osexec "os/exec" +) + +// Command executes a shell command. +func Command(ctx context.Context, args ...string) error { + // nolint: gosec + // G204: Subprocess launched with a potential tainted input or cmd arguments + cmd := osexec.CommandContext(ctx, args[0], args[1:]...) + out, err := cmd.CombinedOutput() + switch err := err.(type) { + case nil: + return nil + case *osexec.ExitError: + return fmt.Errorf("failed to run %q:\n%v", args, string(out)) + default: + return err + } +} + +// CommandVerbose executes a shell command while displaying its output. +func CommandVerbose(ctx context.Context, args ...string) error { + // nolint: gosec + // G204: Subprocess launched with a potential tainted input or cmd arguments + cmd := osexec.CommandContext(ctx, args[0], args[1:]...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} diff --git a/test/e2e/pkg/infra/docker/compose.go b/test/e2e/pkg/infra/docker/compose.go new file mode 100644 index 000000000..3ea5845ee --- /dev/null +++ b/test/e2e/pkg/infra/docker/compose.go @@ -0,0 +1,69 @@ +package docker + +import ( + "bytes" + "text/template" + + e2e "github.com/tendermint/tendermint/test/e2e/pkg" +) + +// makeDockerCompose generates a Docker Compose config for a testnet. +func makeDockerCompose(testnet *e2e.Testnet) ([]byte, error) { + // Must use version 2 Docker Compose format, to support IPv6. + tmpl, err := template.New("docker-compose").Funcs(template.FuncMap{ + "addUint32": func(x, y uint32) uint32 { + return x + y + }, + "isBuiltin": func(protocol e2e.Protocol, mode e2e.Mode) bool { + return mode == e2e.ModeLight || protocol == e2e.ProtocolBuiltin + }, + }).Parse(`version: '2.4' + +networks: + {{ .Name }}: + labels: + e2e: true + driver: bridge +{{- if .IPv6 }} + enable_ipv6: true +{{- end }} + ipam: + driver: default + config: + - subnet: {{ .IP }} + +services: +{{- range .Nodes }} + {{ .Name }}: + labels: + e2e: true + container_name: {{ .Name }} + image: tendermint/e2e-node +{{- if isBuiltin $.ABCIProtocol .Mode }} + entrypoint: /usr/bin/entrypoint-builtin +{{- else if .LogLevel }} + command: start --log-level {{ .LogLevel }} +{{- end }} + init: true + ports: + - 26656 + - {{ if .ProxyPort }}{{ addUint32 .ProxyPort 1000 }}:{{ end }}26660 + - {{ if .ProxyPort }}{{ .ProxyPort }}:{{ end }}26657 + - 6060 + volumes: + - ./{{ .Name }}:/tendermint + networks: + {{ $.Name }}: + ipv{{ if $.IPv6 }}6{{ else }}4{{ end}}_address: {{ .IP }} + +{{end}}`) + if err != nil { + return nil, err + } + var buf bytes.Buffer + err = tmpl.Execute(&buf, testnet) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} diff --git a/test/e2e/pkg/infra/docker/exec.go b/test/e2e/pkg/infra/docker/exec.go new file mode 100644 index 000000000..de0033e32 --- /dev/null +++ b/test/e2e/pkg/infra/docker/exec.go @@ -0,0 +1,27 @@ +package docker + +import ( + "context" + "path/filepath" + + "github.com/tendermint/tendermint/test/e2e/pkg/exec" +) + +// execCompose runs a Docker Compose command for a testnet. +func execCompose(ctx context.Context, dir string, args ...string) error { + return exec.Command(ctx, append( + []string{"docker-compose", "--ansi=never", "-f", filepath.Join(dir, "docker-compose.yml")}, + args...)...) +} + +// execComposeVerbose runs a Docker Compose command for a testnet and displays its output. +func execComposeVerbose(ctx context.Context, dir string, args ...string) error { + return exec.CommandVerbose(ctx, append( + []string{"docker-compose", "--ansi=never", "-f", filepath.Join(dir, "docker-compose.yml")}, + args...)...) +} + +// execDocker runs a Docker command. +func execDocker(ctx context.Context, args ...string) error { + return exec.Command(ctx, append([]string{"docker"}, args...)...) +} diff --git a/test/e2e/pkg/infra/docker/infra.go b/test/e2e/pkg/infra/docker/infra.go new file mode 100644 index 000000000..9827be241 --- /dev/null +++ b/test/e2e/pkg/infra/docker/infra.go @@ -0,0 +1,140 @@ +package docker + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "github.com/tendermint/tendermint/libs/log" + e2e "github.com/tendermint/tendermint/test/e2e/pkg" + "github.com/tendermint/tendermint/test/e2e/pkg/exec" + "github.com/tendermint/tendermint/test/e2e/pkg/infra" +) + +// testnetInfra provides an API for provisioning and manipulating +// infrastructure for a Docker-based testnet. +type testnetInfra struct { + logger log.Logger + testnet *e2e.Testnet +} + +var _ infra.TestnetInfra = &testnetInfra{} + +// NewTestnetInfra constructs an infrastructure provider that allows for Docker-based +// testnet infrastructure. +func NewTestnetInfra(logger log.Logger, testnet *e2e.Testnet) infra.TestnetInfra { + return &testnetInfra{ + logger: logger, + testnet: testnet, + } +} + +func (ti *testnetInfra) Setup(ctx context.Context) error { + compose, err := makeDockerCompose(ti.testnet) + if err != nil { + return err + } + // nolint: gosec + // G306: Expect WriteFile permissions to be 0600 or less + err = os.WriteFile(filepath.Join(ti.testnet.Dir, "docker-compose.yml"), compose, 0644) + if err != nil { + return err + } + return nil +} + +func (ti *testnetInfra) StartNode(ctx context.Context, node *e2e.Node) error { + return execCompose(ctx, ti.testnet.Dir, "up", "-d", node.Name) +} + +func (ti *testnetInfra) DisconnectNode(ctx context.Context, node *e2e.Node) error { + return execDocker(ctx, "network", "disconnect", ti.testnet.Name+"_"+ti.testnet.Name, node.Name) +} + +func (ti *testnetInfra) ConnectNode(ctx context.Context, node *e2e.Node) error { + return execDocker(ctx, "network", "connect", ti.testnet.Name+"_"+ti.testnet.Name, node.Name) +} + +func (ti *testnetInfra) KillNodeProcess(ctx context.Context, node *e2e.Node) error { + return execCompose(ctx, ti.testnet.Dir, "kill", "-s", "SIGKILL", node.Name) +} + +func (ti *testnetInfra) StartNodeProcess(ctx context.Context, node *e2e.Node) error { + return execCompose(ctx, ti.testnet.Dir, "start", node.Name) +} + +func (ti *testnetInfra) PauseNodeProcess(ctx context.Context, node *e2e.Node) error { + return execCompose(ctx, ti.testnet.Dir, "pause", node.Name) +} + +func (ti *testnetInfra) UnpauseNodeProcess(ctx context.Context, node *e2e.Node) error { + return execCompose(ctx, ti.testnet.Dir, "unpause", node.Name) +} + +func (ti *testnetInfra) TerminateNodeProcess(ctx context.Context, node *e2e.Node) error { + return execCompose(ctx, ti.testnet.Dir, "kill", "-s", "SIGTERM", node.Name) +} + +func (ti *testnetInfra) Stop(ctx context.Context) error { + return execCompose(ctx, ti.testnet.Dir, "down") +} + +func (ti *testnetInfra) Pause(ctx context.Context) error { + return execCompose(ctx, ti.testnet.Dir, "pause") +} + +func (ti *testnetInfra) Unpause(ctx context.Context) error { + return execCompose(ctx, ti.testnet.Dir, "unpause") +} + +func (ti *testnetInfra) ShowLogs(ctx context.Context) error { + return execComposeVerbose(ctx, ti.testnet.Dir, "logs", "--no-color") +} + +func (ti *testnetInfra) ShowNodeLogs(ctx context.Context, node *e2e.Node) error { + return execComposeVerbose(ctx, ti.testnet.Dir, "logs", "--no-color", node.Name) +} + +func (ti *testnetInfra) TailLogs(ctx context.Context) error { + return execComposeVerbose(ctx, ti.testnet.Dir, "logs", "--follow") +} + +func (ti *testnetInfra) TailNodeLogs(ctx context.Context, node *e2e.Node) error { + return execComposeVerbose(ctx, ti.testnet.Dir, "logs", "--follow", node.Name) +} + +func (ti *testnetInfra) Cleanup(ctx context.Context) error { + ti.logger.Info("Removing Docker containers and networks") + + // GNU xargs requires the -r flag to not run when input is empty, macOS + // does this by default. Ugly, but works. + xargsR := `$(if [[ $OSTYPE == "linux-gnu"* ]]; then echo -n "-r"; fi)` + + err := exec.Command(ctx, "bash", "-c", fmt.Sprintf( + "docker container ls -qa --filter label=e2e | xargs %v docker container rm -f", xargsR)) + if err != nil { + return err + } + + err = exec.Command(ctx, "bash", "-c", fmt.Sprintf( + "docker network ls -q --filter label=e2e | xargs %v docker network rm", xargsR)) + if err != nil { + return err + } + + // On Linux, some local files in the volume will be owned by root since Tendermint + // runs as root inside the container, so we need to clean them up from within a + // container running as root too. + absDir, err := filepath.Abs(ti.testnet.Dir) + if err != nil { + return err + } + err = execDocker(ctx, "run", "--rm", "--entrypoint", "", "-v", fmt.Sprintf("%v:/network", absDir), + "tendermint/e2e-node", "sh", "-c", "rm -rf /network/*/") + if err != nil { + return err + } + + return nil +} diff --git a/test/e2e/pkg/infra/infra.go b/test/e2e/pkg/infra/infra.go new file mode 100644 index 000000000..2fa7c5ad9 --- /dev/null +++ b/test/e2e/pkg/infra/infra.go @@ -0,0 +1,84 @@ +package infra + +import ( + "context" + + e2e "github.com/tendermint/tendermint/test/e2e/pkg" +) + +// TestnetInfra provides an API for manipulating the infrastructure of a +// specific testnet. +type TestnetInfra interface { + // + // Overarching testnet infrastructure management. + // + + // Setup generates any necessary configuration for the infrastructure + // provider during testnet setup. + Setup(ctx context.Context) error + + // Stop will stop all running processes throughout the testnet without + // destroying any infrastructure. + Stop(ctx context.Context) error + + // Pause will pause all processes in the testnet. + Pause(ctx context.Context) error + + // Unpause will resume a paused testnet. + Unpause(ctx context.Context) error + + // ShowLogs prints all logs for the whole testnet to stdout. + ShowLogs(ctx context.Context) error + + // TailLogs tails the logs for all nodes in the testnet, if this is + // supported by the infrastructure provider. + TailLogs(ctx context.Context) error + + // Cleanup stops and destroys all running testnet infrastructure and + // deletes any generated files. + Cleanup(ctx context.Context) error + + // + // Node management, including node infrastructure. + // + + // StartNode provisions infrastructure for the given node and starts it. + StartNode(ctx context.Context, node *e2e.Node) error + + // DisconnectNode modifies the specified node's network configuration such + // that it becomes bidirectionally disconnected from the network (it cannot + // see other nodes, and other nodes cannot see it). + DisconnectNode(ctx context.Context, node *e2e.Node) error + + // ConnectNode modifies the specified node's network configuration such + // that it can become bidirectionally connected. + ConnectNode(ctx context.Context, node *e2e.Node) error + + // ShowNodeLogs prints all logs for the node with the give ID to stdout. + ShowNodeLogs(ctx context.Context, node *e2e.Node) error + + // TailNodeLogs tails the logs for a single node, if this is supported by + // the infrastructure provider. + TailNodeLogs(ctx context.Context, node *e2e.Node) error + + // + // Node process management. + // + + // KillNodeProcess sends SIGKILL to a node's process. + KillNodeProcess(ctx context.Context, node *e2e.Node) error + + // StartNodeProcess will start a stopped node's process. Assumes that the + // node's infrastructure has previously been provisioned using + // ProvisionNode. + StartNodeProcess(ctx context.Context, node *e2e.Node) error + + // PauseNodeProcess sends a signal to the node's process to pause it. + PauseNodeProcess(ctx context.Context, node *e2e.Node) error + + // UnpauseNodeProcess resumes a paused node's process. + UnpauseNodeProcess(ctx context.Context, node *e2e.Node) error + + // TerminateNodeProcess sends SIGTERM to a node's process. + TerminateNodeProcess(ctx context.Context, node *e2e.Node) error +} diff --git a/test/e2e/pkg/testnet.go b/test/e2e/pkg/testnet.go index a5c8dffd6..2041796c4 100644 --- a/test/e2e/pkg/testnet.go +++ b/test/e2e/pkg/testnet.go @@ -1,4 +1,3 @@ -//nolint: gosec package e2e import ( @@ -467,7 +466,7 @@ func (n Node) AddressRPC() string { // Client returns an RPC client for a node. func (n Node) Client() (*rpchttp.HTTP, error) { - return rpchttp.New(fmt.Sprintf("http://127.0.0.1:%v", n.ProxyPort)) + return rpchttp.New(fmt.Sprintf("http://%s", n.AddressRPC())) } // Stateless returns true if the node is either a seed node or a light node @@ -481,6 +480,8 @@ type keyGenerator struct { } func newKeyGenerator(seed int64) *keyGenerator { + // nolint: gosec + // G404: Use of weak random number generator (math/rand instead of crypto/rand) return &keyGenerator{ random: rand.New(rand.NewSource(seed)), } diff --git a/test/e2e/runner/cleanup.go b/test/e2e/runner/cleanup.go index b08e39f6d..25a1008e6 100644 --- a/test/e2e/runner/cleanup.go +++ b/test/e2e/runner/cleanup.go @@ -1,70 +1,32 @@ package main import ( + "context" "errors" "fmt" "os" - "path/filepath" "github.com/tendermint/tendermint/libs/log" - e2e "github.com/tendermint/tendermint/test/e2e/pkg" + "github.com/tendermint/tendermint/test/e2e/pkg/infra" ) -// Cleanup removes the Docker Compose containers and testnet directory. -func Cleanup(logger log.Logger, testnet *e2e.Testnet) error { - err := cleanupDocker(logger) - if err != nil { - return err +// Cleanup destroys all infrastructure and removes all generated testnet files. +func Cleanup(ctx context.Context, logger log.Logger, testnetDir string, ti infra.TestnetInfra) error { + if testnetDir == "" { + return errors.New("no testnet directory set") } - return cleanupDir(logger, testnet.Dir) -} -// cleanupDocker removes all E2E resources (with label e2e=True), regardless -// of testnet. -func cleanupDocker(logger log.Logger) error { - logger.Info("Removing Docker containers and networks") - - // GNU xargs requires the -r flag to not run when input is empty, macOS - // does this by default. Ugly, but works. - xargsR := `$(if [[ $OSTYPE == "linux-gnu"* ]]; then echo -n "-r"; fi)` - - err := exec("bash", "-c", fmt.Sprintf( - "docker container ls -qa --filter label=e2e | xargs %v docker container rm -f", xargsR)) - if err != nil { + if err := ti.Cleanup(ctx); err != nil { return err } - return exec("bash", "-c", fmt.Sprintf( - "docker network ls -q --filter label=e2e | xargs %v docker network rm", xargsR)) -} - -// cleanupDir cleans up a testnet directory -func cleanupDir(logger log.Logger, dir string) error { - if dir == "" { - return errors.New("no directory set") - } - - _, err := os.Stat(dir) + _, err := os.Stat(testnetDir) if os.IsNotExist(err) { return nil } else if err != nil { return err } - logger.Info(fmt.Sprintf("Removing testnet directory %q", dir)) - - // On Linux, some local files in the volume will be owned by root since Tendermint - // runs as root inside the container, so we need to clean them up from within a - // container running as root too. - absDir, err := filepath.Abs(dir) - if err != nil { - return err - } - err = execDocker("run", "--rm", "--entrypoint", "", "-v", fmt.Sprintf("%v:/network", absDir), - "tendermint/e2e-node", "sh", "-c", "rm -rf /network/*/") - if err != nil { - return err - } - - return os.RemoveAll(dir) + logger.Info(fmt.Sprintf("Removing testnet directory %q", testnetDir)) + return os.RemoveAll(testnetDir) } diff --git a/test/e2e/runner/exec.go b/test/e2e/runner/exec.go deleted file mode 100644 index f2bc5163c..000000000 --- a/test/e2e/runner/exec.go +++ /dev/null @@ -1,50 +0,0 @@ -//nolint: gosec -package main - -import ( - "fmt" - "os" - osexec "os/exec" - "path/filepath" -) - -// execute executes a shell command. -func exec(args ...string) error { - cmd := osexec.Command(args[0], args[1:]...) - out, err := cmd.CombinedOutput() - switch err := err.(type) { - case nil: - return nil - case *osexec.ExitError: - return fmt.Errorf("failed to run %q:\n%v", args, string(out)) - default: - return err - } -} - -// execVerbose executes a shell command while displaying its output. -func execVerbose(args ...string) error { - cmd := osexec.Command(args[0], args[1:]...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - return cmd.Run() -} - -// execCompose runs a Docker Compose command for a testnet. -func execCompose(dir string, args ...string) error { - return exec(append( - []string{"docker-compose", "--ansi=never", "-f", filepath.Join(dir, "docker-compose.yml")}, - args...)...) -} - -// execComposeVerbose runs a Docker Compose command for a testnet and displays its output. -func execComposeVerbose(dir string, args ...string) error { - return execVerbose(append( - []string{"docker-compose", "--ansi=never", "-f", filepath.Join(dir, "docker-compose.yml")}, - args...)...) -} - -// execDocker runs a Docker command. -func execDocker(args ...string) error { - return exec(append([]string{"docker"}, args...)...) -} diff --git a/test/e2e/runner/main.go b/test/e2e/runner/main.go index c4a73d33f..9a24f1141 100644 --- a/test/e2e/runner/main.go +++ b/test/e2e/runner/main.go @@ -13,6 +13,8 @@ import ( "github.com/tendermint/tendermint/libs/log" e2e "github.com/tendermint/tendermint/test/e2e/pkg" + "github.com/tendermint/tendermint/test/e2e/pkg/infra" + "github.com/tendermint/tendermint/test/e2e/pkg/infra/docker" ) const randomSeed = 2308084734268 @@ -33,6 +35,7 @@ func main() { type CLI struct { root *cobra.Command testnet *e2e.Testnet + infra infra.TestnetInfra preserve bool } @@ -53,12 +56,23 @@ func NewCLI(logger log.Logger) *CLI { if err != nil { return err } + providerID, err := cmd.Flags().GetString("provider") + if err != nil { + return err + } + switch providerID { + case "docker": + cli.infra = docker.NewTestnetInfra(logger, testnet) + logger.Info("Using Docker-based infrastructure provider") + default: + return fmt.Errorf("unrecognized infrastructure provider ID: %s", providerID) + } cli.testnet = testnet return nil }, RunE: func(cmd *cobra.Command, args []string) (err error) { - if err = Cleanup(logger, cli.testnet); err != nil { + if err = Cleanup(cmd.Context(), logger, cli.testnet.Dir, cli.infra); err != nil { return err } defer func() { @@ -67,11 +81,11 @@ func NewCLI(logger log.Logger) *CLI { } else if err != nil { logger.Info("Preserving testnet that encountered error", "err", err) - } else if err := Cleanup(logger, cli.testnet); err != nil { + } else if err := Cleanup(cmd.Context(), logger, cli.testnet.Dir, cli.infra); err != nil { logger.Error("error cleaning up testnet contents", "err", err) } }() - if err = Setup(logger, cli.testnet); err != nil { + if err = Setup(cmd.Context(), logger, cli.testnet, cli.infra); err != nil { return err } @@ -87,7 +101,7 @@ func NewCLI(logger log.Logger) *CLI { chLoadResult <- Load(lctx, logger, r, cli.testnet) }() startAt := time.Now() - if err = Start(ctx, logger, cli.testnet); err != nil { + if err = Start(ctx, logger, cli.testnet, cli.infra); err != nil { return err } @@ -96,7 +110,7 @@ func NewCLI(logger log.Logger) *CLI { } if cli.testnet.HasPerturbations() { - if err = Perturb(ctx, logger, cli.testnet); err != nil { + if err = Perturb(ctx, logger, cli.testnet, cli.infra); err != nil { return err } if err = Wait(ctx, logger, cli.testnet, 5); err != nil { // allow some txs to go through @@ -134,7 +148,7 @@ func NewCLI(logger log.Logger) *CLI { if err = Wait(ctx, logger, cli.testnet, 5); err != nil { // wait for network to settle before tests return err } - if err := Test(cli.testnet); err != nil { + if err := Test(ctx, cli.testnet); err != nil { return err } return nil @@ -144,6 +158,8 @@ func NewCLI(logger log.Logger) *CLI { cli.root.PersistentFlags().StringP("file", "f", "", "Testnet TOML manifest") _ = cli.root.MarkPersistentFlagRequired("file") + cli.root.PersistentFlags().String("provider", "docker", "Which infrastructure provider to use") + cli.root.Flags().BoolVarP(&cli.preserve, "preserve", "p", false, "Preserves the running of the test net after tests are completed") @@ -156,7 +172,7 @@ func NewCLI(logger log.Logger) *CLI { Use: "setup", Short: "Generates the testnet directory and configuration", RunE: func(cmd *cobra.Command, args []string) error { - return Setup(logger, cli.testnet) + return Setup(cmd.Context(), logger, cli.testnet, cli.infra) }, }) @@ -166,12 +182,12 @@ func NewCLI(logger log.Logger) *CLI { RunE: func(cmd *cobra.Command, args []string) error { _, err := os.Stat(cli.testnet.Dir) if os.IsNotExist(err) { - err = Setup(logger, cli.testnet) + err = Setup(cmd.Context(), logger, cli.testnet, cli.infra) } if err != nil { return err } - return Start(cmd.Context(), logger, cli.testnet) + return Start(cmd.Context(), logger, cli.testnet, cli.infra) }, }) @@ -179,7 +195,7 @@ func NewCLI(logger log.Logger) *CLI { Use: "perturb", Short: "Perturbs the Docker testnet, e.g. by restarting or disconnecting nodes", RunE: func(cmd *cobra.Command, args []string) error { - return Perturb(cmd.Context(), logger, cli.testnet) + return Perturb(cmd.Context(), logger, cli.testnet, cli.infra) }, }) @@ -196,7 +212,7 @@ func NewCLI(logger log.Logger) *CLI { Short: "Stops the Docker testnet", RunE: func(cmd *cobra.Command, args []string) error { logger.Info("Stopping testnet") - return execCompose(cli.testnet.Dir, "down") + return cli.infra.Stop(cmd.Context()) }, }) @@ -205,7 +221,7 @@ func NewCLI(logger log.Logger) *CLI { Short: "Pauses the Docker testnet", RunE: func(cmd *cobra.Command, args []string) error { logger.Info("Pausing testnet") - return execCompose(cli.testnet.Dir, "pause") + return cli.infra.Pause(cmd.Context()) }, }) @@ -214,7 +230,7 @@ func NewCLI(logger log.Logger) *CLI { Short: "Resumes the Docker testnet", RunE: func(cmd *cobra.Command, args []string) error { logger.Info("Resuming testnet") - return execCompose(cli.testnet.Dir, "unpause") + return cli.infra.Unpause(cmd.Context()) }, }) @@ -259,7 +275,7 @@ func NewCLI(logger log.Logger) *CLI { Use: "test", Short: "Runs test cases against a running testnet", RunE: func(cmd *cobra.Command, args []string) error { - return Test(cli.testnet) + return Test(cmd.Context(), cli.testnet) }, }) @@ -267,17 +283,24 @@ func NewCLI(logger log.Logger) *CLI { Use: "cleanup", Short: "Removes the testnet directory", RunE: func(cmd *cobra.Command, args []string) error { - return Cleanup(logger, cli.testnet) + return Cleanup(cmd.Context(), logger, cli.testnet.Dir, cli.infra) }, }) cli.root.AddCommand(&cobra.Command{ Use: "logs [node]", - Short: "Shows the testnet or a specefic node's logs", + Short: "Shows the testnet or a specific node's logs", Example: "runner logs validator03", Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - return execComposeVerbose(cli.testnet.Dir, append([]string{"logs", "--no-color"}, args...)...) + if len(args) > 0 { + node := cli.testnet.LookupNode(args[0]) + if node == nil { + return fmt.Errorf("no such node: %s", args[0]) + } + return cli.infra.ShowNodeLogs(cmd.Context(), node) + } + return cli.infra.ShowLogs(cmd.Context()) }, }) @@ -287,9 +310,13 @@ func NewCLI(logger log.Logger) *CLI { Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { if len(args) == 1 { - return execComposeVerbose(cli.testnet.Dir, "logs", "--follow", args[0]) + node := cli.testnet.LookupNode(args[0]) + if node == nil { + return fmt.Errorf("no such node: %s", args[0]) + } + return cli.infra.TailNodeLogs(cmd.Context(), node) } - return execComposeVerbose(cli.testnet.Dir, "logs", "--follow") + return cli.infra.TailLogs(cmd.Context()) }, }) @@ -302,20 +329,20 @@ func NewCLI(logger log.Logger) *CLI { Min Block Interval Max Block Interval over a 100 block sampling period. - + Does not run any perbutations. `, RunE: func(cmd *cobra.Command, args []string) error { - if err := Cleanup(logger, cli.testnet); err != nil { + if err := Cleanup(cmd.Context(), logger, cli.testnet.Dir, cli.infra); err != nil { return err } defer func() { - if err := Cleanup(logger, cli.testnet); err != nil { + if err := Cleanup(cmd.Context(), logger, cli.testnet.Dir, cli.infra); err != nil { logger.Error("error cleaning up testnet contents", "err", err) } }() - if err := Setup(logger, cli.testnet); err != nil { + if err := Setup(cmd.Context(), logger, cli.testnet, cli.infra); err != nil { return err } @@ -331,7 +358,7 @@ Does not run any perbutations. chLoadResult <- Load(lctx, logger, r, cli.testnet) }() - if err := Start(ctx, logger, cli.testnet); err != nil { + if err := Start(ctx, logger, cli.testnet, cli.infra); err != nil { return err } diff --git a/test/e2e/runner/perturb.go b/test/e2e/runner/perturb.go index acabf7f34..76a209ea2 100644 --- a/test/e2e/runner/perturb.go +++ b/test/e2e/runner/perturb.go @@ -8,10 +8,11 @@ import ( "github.com/tendermint/tendermint/libs/log" rpctypes "github.com/tendermint/tendermint/rpc/coretypes" e2e "github.com/tendermint/tendermint/test/e2e/pkg" + "github.com/tendermint/tendermint/test/e2e/pkg/infra" ) // Perturbs a running testnet. -func Perturb(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error { +func Perturb(ctx context.Context, logger log.Logger, testnet *e2e.Testnet, ti infra.TestnetInfra) error { timer := time.NewTimer(0) // first tick fires immediately; reset below defer timer.Stop() @@ -21,7 +22,7 @@ func Perturb(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error case <-ctx.Done(): return ctx.Err() case <-timer.C: - _, err := PerturbNode(ctx, logger, node, perturbation) + _, err := PerturbNode(ctx, logger, node, perturbation, ti) if err != nil { return err } @@ -36,46 +37,45 @@ func Perturb(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error // PerturbNode perturbs a node with a given perturbation, returning its status // after recovering. -func PerturbNode(ctx context.Context, logger log.Logger, node *e2e.Node, perturbation e2e.Perturbation) (*rpctypes.ResultStatus, error) { - testnet := node.Testnet +func PerturbNode(ctx context.Context, logger log.Logger, node *e2e.Node, perturbation e2e.Perturbation, ti infra.TestnetInfra) (*rpctypes.ResultStatus, error) { switch perturbation { case e2e.PerturbationDisconnect: logger.Info(fmt.Sprintf("Disconnecting node %v...", node.Name)) - if err := execDocker("network", "disconnect", testnet.Name+"_"+testnet.Name, node.Name); err != nil { + if err := ti.DisconnectNode(ctx, node); err != nil { return nil, err } time.Sleep(10 * time.Second) - if err := execDocker("network", "connect", testnet.Name+"_"+testnet.Name, node.Name); err != nil { + if err := ti.ConnectNode(ctx, node); err != nil { return nil, err } case e2e.PerturbationKill: logger.Info(fmt.Sprintf("Killing node %v...", node.Name)) - if err := execCompose(testnet.Dir, "kill", "-s", "SIGKILL", node.Name); err != nil { + if err := ti.KillNodeProcess(ctx, node); err != nil { return nil, err } time.Sleep(10 * time.Second) - if err := execCompose(testnet.Dir, "start", node.Name); err != nil { + if err := ti.StartNodeProcess(ctx, node); err != nil { return nil, err } case e2e.PerturbationPause: logger.Info(fmt.Sprintf("Pausing node %v...", node.Name)) - if err := execCompose(testnet.Dir, "pause", node.Name); err != nil { + if err := ti.PauseNodeProcess(ctx, node); err != nil { return nil, err } time.Sleep(10 * time.Second) - if err := execCompose(testnet.Dir, "unpause", node.Name); err != nil { + if err := ti.UnpauseNodeProcess(ctx, node); err != nil { return nil, err } case e2e.PerturbationRestart: logger.Info(fmt.Sprintf("Restarting node %v...", node.Name)) - if err := execCompose(testnet.Dir, "kill", "-s", "SIGTERM", node.Name); err != nil { + if err := ti.TerminateNodeProcess(ctx, node); err != nil { return nil, err } time.Sleep(10 * time.Second) - if err := execCompose(testnet.Dir, "start", node.Name); err != nil { + if err := ti.StartNodeProcess(ctx, node); err != nil { return nil, err } diff --git a/test/e2e/runner/setup.go b/test/e2e/runner/setup.go index 9e6e5cc18..24477c9e0 100644 --- a/test/e2e/runner/setup.go +++ b/test/e2e/runner/setup.go @@ -1,8 +1,8 @@ -// nolint: gosec package main import ( "bytes" + "context" "encoding/base64" "encoding/json" "errors" @@ -12,7 +12,6 @@ import ( "regexp" "sort" "strings" - "text/template" "time" "github.com/BurntSushi/toml" @@ -22,6 +21,7 @@ import ( "github.com/tendermint/tendermint/libs/log" "github.com/tendermint/tendermint/privval" e2e "github.com/tendermint/tendermint/test/e2e/pkg" + "github.com/tendermint/tendermint/test/e2e/pkg/infra" "github.com/tendermint/tendermint/types" ) @@ -39,7 +39,7 @@ const ( ) // Setup sets up the testnet configuration. -func Setup(logger log.Logger, testnet *e2e.Testnet) error { +func Setup(ctx context.Context, logger log.Logger, testnet *e2e.Testnet, ti infra.TestnetInfra) error { logger.Info(fmt.Sprintf("Generating testnet files in %q", testnet.Dir)) err := os.MkdirAll(testnet.Dir, os.ModePerm) @@ -47,15 +47,6 @@ func Setup(logger log.Logger, testnet *e2e.Testnet) error { return err } - compose, err := MakeDockerCompose(testnet) - if err != nil { - return err - } - err = os.WriteFile(filepath.Join(testnet.Dir, "docker-compose.yml"), compose, 0644) - if err != nil { - return err - } - genesis, err := MakeGenesis(testnet) if err != nil { return err @@ -92,6 +83,8 @@ func Setup(logger log.Logger, testnet *e2e.Testnet) error { if err != nil { return err } + // nolint: gosec + // G306: Expect WriteFile permissions to be 0600 or less err = os.WriteFile(filepath.Join(nodeDir, "config", "app.toml"), appCfg, 0644) if err != nil { return err @@ -131,70 +124,13 @@ func Setup(logger log.Logger, testnet *e2e.Testnet) error { } } + if err := ti.Setup(ctx); err != nil { + return err + } + return nil } -// MakeDockerCompose generates a Docker Compose config for a testnet. -func MakeDockerCompose(testnet *e2e.Testnet) ([]byte, error) { - // Must use version 2 Docker Compose format, to support IPv6. - tmpl, err := template.New("docker-compose").Funcs(template.FuncMap{ - "addUint32": func(x, y uint32) uint32 { - return x + y - }, - "isBuiltin": func(protocol e2e.Protocol, mode e2e.Mode) bool { - return mode == e2e.ModeLight || protocol == e2e.ProtocolBuiltin - }, - }).Parse(`version: '2.4' - -networks: - {{ .Name }}: - labels: - e2e: true - driver: bridge -{{- if .IPv6 }} - enable_ipv6: true -{{- end }} - ipam: - driver: default - config: - - subnet: {{ .IP }} - -services: -{{- range .Nodes }} - {{ .Name }}: - labels: - e2e: true - container_name: {{ .Name }} - image: tendermint/e2e-node -{{- if isBuiltin $.ABCIProtocol .Mode }} - entrypoint: /usr/bin/entrypoint-builtin -{{- else if .LogLevel }} - command: start --log-level {{ .LogLevel }} -{{- end }} - init: true - ports: - - 26656 - - {{ if .ProxyPort }}{{ addUint32 .ProxyPort 1000 }}:{{ end }}26660 - - {{ if .ProxyPort }}{{ .ProxyPort }}:{{ end }}26657 - - 6060 - volumes: - - ./{{ .Name }}:/tendermint - networks: - {{ $.Name }}: - ipv{{ if $.IPv6 }}6{{ else }}4{{ end}}_address: {{ .IP }} - -{{end}}`) - if err != nil { - return nil, err - } - var buf bytes.Buffer - err = tmpl.Execute(&buf, testnet) - if err != nil { - return nil, err - } - return buf.Bytes(), nil -} - // MakeGenesis generates a genesis document. func MakeGenesis(testnet *e2e.Testnet) (types.GenesisDoc, error) { genesis := types.GenesisDoc{ @@ -421,5 +357,7 @@ func UpdateConfigStateSync(node *e2e.Node, height int64, hash []byte) error { } bz = regexp.MustCompile(`(?m)^trust-height =.*`).ReplaceAll(bz, []byte(fmt.Sprintf(`trust-height = %v`, height))) bz = regexp.MustCompile(`(?m)^trust-hash =.*`).ReplaceAll(bz, []byte(fmt.Sprintf(`trust-hash = "%X"`, hash))) + // nolint: gosec + // G306: Expect WriteFile permissions to be 0600 or less return os.WriteFile(cfgPath, bz, 0644) } diff --git a/test/e2e/runner/start.go b/test/e2e/runner/start.go index be9661df3..5d5c2e7a9 100644 --- a/test/e2e/runner/start.go +++ b/test/e2e/runner/start.go @@ -8,9 +8,10 @@ import ( "github.com/tendermint/tendermint/libs/log" e2e "github.com/tendermint/tendermint/test/e2e/pkg" + "github.com/tendermint/tendermint/test/e2e/pkg/infra" ) -func Start(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error { +func Start(ctx context.Context, logger log.Logger, testnet *e2e.Testnet, ti infra.TestnetInfra) error { if len(testnet.Nodes) == 0 { return fmt.Errorf("no nodes in testnet") } @@ -44,7 +45,7 @@ func Start(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error { for len(nodeQueue) > 0 && nodeQueue[0].StartAt == 0 { node := nodeQueue[0] nodeQueue = nodeQueue[1:] - if err := execCompose(testnet.Dir, "up", "-d", node.Name); err != nil { + if err := ti.StartNode(ctx, node); err != nil { return err } @@ -58,7 +59,7 @@ func Start(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error { return err } node.HasStarted = true - logger.Info(fmt.Sprintf("Node %v up on http://127.0.0.1:%v", node.Name, node.ProxyPort)) + logger.Info(fmt.Sprintf("Node %v up on http://%v:%v", node.IP, node.Name, node.ProxyPort)) } networkHeight := testnet.InitialHeight @@ -106,7 +107,7 @@ func Start(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error { } } - if err := execCompose(testnet.Dir, "up", "-d", node.Name); err != nil { + if err := ti.StartNode(ctx, node); err != nil { return err } @@ -128,8 +129,8 @@ func Start(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error { } else { lastNodeHeight = status.SyncInfo.LatestBlockHeight } - logger.Info(fmt.Sprintf("Node %v up on http://127.0.0.1:%v at height %v", - node.Name, node.ProxyPort, lastNodeHeight)) + logger.Info(fmt.Sprintf("Node %v up on http://%v:%v at height %v", + node.IP, node.Name, node.ProxyPort, lastNodeHeight)) } return nil diff --git a/test/e2e/runner/test.go b/test/e2e/runner/test.go index da7a4a50f..7766d6c8d 100644 --- a/test/e2e/runner/test.go +++ b/test/e2e/runner/test.go @@ -1,17 +1,19 @@ package main import ( + "context" "os" e2e "github.com/tendermint/tendermint/test/e2e/pkg" + "github.com/tendermint/tendermint/test/e2e/pkg/exec" ) // Test runs test cases under tests/ -func Test(testnet *e2e.Testnet) error { +func Test(ctx context.Context, testnet *e2e.Testnet) error { err := os.Setenv("E2E_MANIFEST", testnet.File) if err != nil { return err } - return execVerbose("./build/tests", "-test.count=1") + return exec.CommandVerbose(ctx, "./build/tests", "-test.count=1") } diff --git a/test/e2e/tests/net_test.go b/test/e2e/tests/net_test.go index 939c136c7..43cab77e8 100644 --- a/test/e2e/tests/net_test.go +++ b/test/e2e/tests/net_test.go @@ -20,7 +20,7 @@ func TestNet_Peers(t *testing.T) { // FIXME: https://github.com/tendermint/tendermint/issues/8848 // We should be able to assert that we can discover all peers in a network - expectedPeers := len(node.Testnet.Nodes) - 1 // includes extra tolerance + expectedPeers := len(node.Testnet.Nodes) peers := make(map[string]*e2e.Node, 0) seen := map[string]bool{} for _, n := range node.Testnet.Nodes { @@ -33,7 +33,7 @@ func TestNet_Peers(t *testing.T) { seen[n.Name] = false } - require.GreaterOrEqual(t, netInfo.NPeers, expectedPeers, + require.GreaterOrEqual(t, netInfo.NPeers, expectedPeers-1, "node is not fully meshed with peers") for _, peerInfo := range netInfo.Peers { @@ -45,8 +45,10 @@ func TestNet_Peers(t *testing.T) { seen[peer.Name] = true } - for name := range seen { - require.True(t, seen[name], "node %v not peered with %v", node.Name, name) - } + // FIXME: https://github.com/tendermint/tendermint/issues/8848 + // We should be able to assert that we can discover all peers in a network + // for name := range seen { + // require.True(t, seen[name], "node %v not peered with %v", node.Name, name) + // } }) } diff --git a/types/node_info.go b/types/node_info.go index fd47816e2..bc614f55e 100644 --- a/types/node_info.go +++ b/types/node_info.go @@ -7,8 +7,8 @@ import ( "strconv" "strings" + tmstrings "github.com/tendermint/tendermint/internal/libs/strings" "github.com/tendermint/tendermint/libs/bytes" - tmstrings "github.com/tendermint/tendermint/libs/strings" tmp2p "github.com/tendermint/tendermint/proto/tendermint/p2p" ) diff --git a/types/params.go b/types/params.go index 28c969e46..754b50322 100644 --- a/types/params.go +++ b/types/params.go @@ -9,7 +9,7 @@ import ( "github.com/tendermint/tendermint/crypto/ed25519" "github.com/tendermint/tendermint/crypto/secp256k1" "github.com/tendermint/tendermint/crypto/sr25519" - tmstrings "github.com/tendermint/tendermint/libs/strings" + tmstrings "github.com/tendermint/tendermint/internal/libs/strings" tmproto "github.com/tendermint/tendermint/proto/tendermint/types" )