diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 5fb219e24..524df1ef8 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -39,7 +39,7 @@ jobs: platforms: all - name: Set up Docker Build - uses: docker/setup-buildx-action@v1.6.0 + uses: docker/setup-buildx-action@v1.7.0 - name: Login to DockerHub if: ${{ github.event_name != 'pull_request' }} diff --git a/UPGRADING.md b/UPGRADING.md index 28e44e58c..93cd6c20f 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -212,22 +212,25 @@ and one function have moved to the Tendermint `crypto` package: The format of all tendermint on-disk database keys changes in 0.35. Upgrading nodes must either re-sync all data or run a migration -script provided in this release. The script located in -`github.com/tendermint/tendermint/scripts/keymigrate/migrate.go` -provides the function `Migrate(context.Context, db.DB)` which you can -operationalize as makes sense for your deployment. +script provided in this release. + +The script located in +`github.com/tendermint/tendermint/scripts/keymigrate/migrate.go` provides the +function `Migrate(context.Context, db.DB)` which you can operationalize as +makes sense for your deployment. For ease of use the `tendermint` command includes a CLI version of the migration script, which you can invoke, as in: tendermint key-migrate -This reads the configuration file as normal and allows the -`--db-backend` and `--db-dir` flags to change database operations as -needed. +This reads the configuration file as normal and allows the `--db-backend` and +`--db-dir` flags to override the database location as needed. -The migration operation is idempotent and can be run more than once, -if needed. +The migration operation is intended to be idempotent, and should be safe to +rerun on the same database multiple times. As a safety measure, however, we +recommend that operators test out the migration on a copy of the database +first, if it is practical to do so, before applying it to the production data. ### CLI Changes diff --git a/docs/tendermint-core/consensus/proposer-based-timestamps.md b/docs/tendermint-core/consensus/proposer-based-timestamps.md index 7f98f10d6..17036a9f2 100644 --- a/docs/tendermint-core/consensus/proposer-based-timestamps.md +++ b/docs/tendermint-core/consensus/proposer-based-timestamps.md @@ -13,14 +13,15 @@ order: 3 The PBTS algorithm defines a way for a Tendermint blockchain to create block timestamps that are within a reasonable bound of the clocks of the validators on the network. This replaces the original BFTTime algorithm for timestamp -assignment that relied on the timestamps included in precommit messages. +assignment that computed a timestamp using the timestamps included in precommit +messages. ## Algorithm Parameters The functionality of the PBTS algorithm is governed by two parameters within Tendermint. These two parameters are [consensus parameters](https://github.com/tendermint/tendermint/blob/master/spec/abci/apps.md#L291), -meaning they are configured by the ABCI application and are expected to be the +meaning they are configured by the ABCI application and are therefore the same same across all nodes on the network. ### `Precision` @@ -51,7 +52,7 @@ useful for the protocols and applications built on top of Tendermint. The following protocols and application features require a reliable source of time: * Tendermint Light Clients [rely on correspondence between their known time](https://github.com/tendermint/tendermint/blob/master/spec/light-client/verification/README.md#definitions-1) and the block time for block verification. -* Tendermint Evidence validity is determined [either in terms of heights or in terms of time](https://github.com/tendermint/tendermint/blob/master/spec/consensus/evidence.md#verification). +* Tendermint Evidence expiration is determined [either in terms of heights or in terms of time](https://github.com/tendermint/tendermint/blob/master/spec/consensus/evidence.md#verification). * Unbonding of staked assets in the Cosmos Hub [occurs after a period of 21 days](https://github.com/cosmos/governance/blob/master/params-change/Staking.md#unbondingtime). * IBC packets can use either a [timestamp or a height to timeout packet diff --git a/go.mod b/go.mod index ff5b4e11f..09ae2dad9 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ require ( ) require ( - github.com/creachadair/atomicfile v0.2.5 + github.com/creachadair/atomicfile v0.2.6 github.com/creachadair/taskgroup v0.3.2 github.com/golangci/golangci-lint v1.45.2 github.com/google/go-cmp v0.5.8 diff --git a/go.sum b/go.sum index a7b2d7bd8..9997d4c22 100644 --- a/go.sum +++ b/go.sum @@ -227,8 +227,8 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creachadair/atomicfile v0.2.5 h1:wkOlpsjyJOvJ3Hd8juHKdirJnCSIPacvtY21/3nYjAo= -github.com/creachadair/atomicfile v0.2.5/go.mod h1:BRq8Une6ckFneYXZQ+kO7p1ZZP3I2fzVzf28JxrIkBc= +github.com/creachadair/atomicfile v0.2.6 h1:FgYxYvGcqREApTY8Nxg8msM6P/KVKK3ob5h9FaRUTNg= +github.com/creachadair/atomicfile v0.2.6/go.mod h1:BRq8Une6ckFneYXZQ+kO7p1ZZP3I2fzVzf28JxrIkBc= github.com/creachadair/taskgroup v0.3.2 h1:zlfutDS+5XG40AOxcHDSThxKzns8Tnr9jnr6VqkYlkM= github.com/creachadair/taskgroup v0.3.2/go.mod h1:wieWwecHVzsidg2CsUnFinW1faVN4+kq+TDlRJQ0Wbk= github.com/creachadair/tomledit v0.0.19 h1:zbpfUtYFYFdpRjwJY9HJlto1iZ4M5YwYB6qqc37F6UM= diff --git a/internal/p2p/channel.go b/internal/p2p/channel.go index 8e6774612..d3d7d104f 100644 --- a/internal/p2p/channel.go +++ b/internal/p2p/channel.go @@ -59,25 +59,17 @@ type Channel struct { outCh chan<- Envelope // outbound messages (reactors to peers) errCh chan<- PeerError // peer error reporting - messageType proto.Message // the channel's message type, used for unmarshaling - name string + name string } // NewChannel creates a new channel. It is primarily for internal and test // use, reactors should use Router.OpenChannel(). -func NewChannel( - id ChannelID, - messageType proto.Message, - inCh <-chan Envelope, - outCh chan<- Envelope, - errCh chan<- PeerError, -) *Channel { +func NewChannel(id ChannelID, inCh <-chan Envelope, outCh chan<- Envelope, errCh chan<- PeerError) *Channel { return &Channel{ - ID: id, - messageType: messageType, - inCh: inCh, - outCh: outCh, - errCh: errCh, + ID: id, + inCh: inCh, + outCh: outCh, + errCh: errCh, } } diff --git a/internal/p2p/pex/reactor_test.go b/internal/p2p/pex/reactor_test.go index f2132fbba..ec2f03d83 100644 --- a/internal/p2p/pex/reactor_test.go +++ b/internal/p2p/pex/reactor_test.go @@ -289,7 +289,6 @@ func setupSingle(ctx context.Context, t *testing.T) *singleTestReactor { pexErrCh := make(chan p2p.PeerError, chBuf) pexCh := p2p.NewChannel( p2p.ChannelID(pex.PexChannel), - new(p2pproto.PexMessage), pexInCh, pexOutCh, pexErrCh, diff --git a/internal/p2p/router.go b/internal/p2p/router.go index df096dbb6..459be7975 100644 --- a/internal/p2p/router.go +++ b/internal/p2p/router.go @@ -262,7 +262,7 @@ func (r *Router) OpenChannel(ctx context.Context, chDesc *ChannelDescriptor) (*C queue := r.queueFactory(chDesc.RecvBufferCapacity) outCh := make(chan Envelope, chDesc.RecvBufferCapacity) errCh := make(chan PeerError, chDesc.RecvBufferCapacity) - channel := NewChannel(id, messageType, queue.dequeue(), outCh, errCh) + channel := NewChannel(id, queue.dequeue(), outCh, errCh) channel.name = chDesc.Name var wrapper Wrapper diff --git a/internal/statesync/dispatcher_test.go b/internal/statesync/dispatcher_test.go index 65c517be4..8ec074bd1 100644 --- a/internal/statesync/dispatcher_test.go +++ b/internal/statesync/dispatcher_test.go @@ -30,7 +30,7 @@ func testChannel(size int) (*channelInternal, *p2p.Channel) { Out: make(chan p2p.Envelope, size), Error: make(chan p2p.PeerError, size), } - return in, p2p.NewChannel(0, nil, in.In, in.Out, in.Error) + return in, p2p.NewChannel(0, in.In, in.Out, in.Error) } func TestDispatcherBasic(t *testing.T) { diff --git a/internal/statesync/reactor_test.go b/internal/statesync/reactor_test.go index c6b2c2d2b..55a9fcf8c 100644 --- a/internal/statesync/reactor_test.go +++ b/internal/statesync/reactor_test.go @@ -102,7 +102,6 @@ func setup( rts.snapshotChannel = p2p.NewChannel( SnapshotChannel, - new(ssproto.Message), rts.snapshotInCh, rts.snapshotOutCh, rts.snapshotPeerErrCh, @@ -110,7 +109,6 @@ func setup( rts.chunkChannel = p2p.NewChannel( ChunkChannel, - new(ssproto.Message), rts.chunkInCh, rts.chunkOutCh, rts.chunkPeerErrCh, @@ -118,7 +116,6 @@ func setup( rts.blockChannel = p2p.NewChannel( LightBlockChannel, - new(ssproto.Message), rts.blockInCh, rts.blockOutCh, rts.blockPeerErrCh, @@ -126,7 +123,6 @@ func setup( rts.paramsChannel = p2p.NewChannel( ParamsChannel, - new(ssproto.Message), rts.paramsInCh, rts.paramsOutCh, rts.paramsPeerErrCh, diff --git a/scripts/keymigrate/migrate.go b/scripts/keymigrate/migrate.go index ca2c528e2..a0b43aef6 100644 --- a/scripts/keymigrate/migrate.go +++ b/scripts/keymigrate/migrate.go @@ -86,27 +86,30 @@ const ( var prefixes = []struct { prefix []byte ktype keyType + check func(keyID) bool }{ - {[]byte("consensusParamsKey:"), consensusParamsKey}, - {[]byte("abciResponsesKey:"), abciResponsesKey}, - {[]byte("validatorsKey:"), validatorsKey}, - {[]byte("stateKey"), stateStoreKey}, - {[]byte("H:"), blockMetaKey}, - {[]byte("P:"), blockPartKey}, - {[]byte("C:"), commitKey}, - {[]byte("SC:"), seenCommitKey}, - {[]byte("BH:"), blockHashKey}, - {[]byte("size"), lightSizeKey}, - {[]byte("lb/"), lightBlockKey}, - {[]byte("\x00"), evidenceCommittedKey}, - {[]byte("\x01"), evidencePendingKey}, + {[]byte("consensusParamsKey:"), consensusParamsKey, nil}, + {[]byte("abciResponsesKey:"), abciResponsesKey, nil}, + {[]byte("validatorsKey:"), validatorsKey, nil}, + {[]byte("stateKey"), stateStoreKey, nil}, + {[]byte("H:"), blockMetaKey, nil}, + {[]byte("P:"), blockPartKey, nil}, + {[]byte("C:"), commitKey, nil}, + {[]byte("SC:"), seenCommitKey, nil}, + {[]byte("BH:"), blockHashKey, nil}, + {[]byte("size"), lightSizeKey, nil}, + {[]byte("lb/"), lightBlockKey, nil}, + {[]byte("\x00"), evidenceCommittedKey, checkEvidenceKey}, + {[]byte("\x01"), evidencePendingKey, checkEvidenceKey}, } // checkKeyType classifies a candidate key based on its structure. func checkKeyType(key keyID) keyType { for _, p := range prefixes { if bytes.HasPrefix(key, p.prefix) { - return p.ktype + if p.check == nil || p.check(key) { + return p.ktype + } } } @@ -342,6 +345,35 @@ func convertEvidence(key keyID, newPrefix int64) ([]byte, error) { return orderedcode.Append(nil, newPrefix, binary.BigEndian.Uint64(hb), string(evidenceHash)) } +// checkEvidenceKey reports whether a candidate key with one of the legacy +// evidence prefixes has the correct structure for a legacy evidence key. +// +// This check is needed because transaction hashes are stored without a prefix, +// so checking the one-byte prefix alone is not enough to distinguish them. +// Legacy evidence keys are suffixed with a string of the format: +// +// "%0.16X/%X" +// +// where the first element is the height and the second is the hash. Thus, we +// check +func checkEvidenceKey(key keyID) bool { + parts := bytes.SplitN(key[1:], []byte("/"), 2) + if len(parts) != 2 || len(parts[0]) != 16 || !isHex(parts[0]) || !isHex(parts[1]) { + return false + } + return true +} + +func isHex(data []byte) bool { + for _, b := range data { + if ('0' <= b && b <= '9') || ('a' <= b && b <= 'f') || ('A' <= b && b <= 'F') { + continue + } + return false + } + return len(data) != 0 +} + func replaceKey(db dbm.DB, key keyID, gooseFn migrateFunc) error { exists, err := db.Has(key) if err != nil { diff --git a/scripts/keymigrate/migrate_test.go b/scripts/keymigrate/migrate_test.go index b2727a5df..f7322b352 100644 --- a/scripts/keymigrate/migrate_test.go +++ b/scripts/keymigrate/migrate_test.go @@ -1,11 +1,11 @@ package keymigrate import ( - "bytes" "context" "errors" "fmt" "math" + "strings" "testing" "github.com/google/orderedcode" @@ -21,6 +21,7 @@ func makeKey(t *testing.T, elems ...interface{}) []byte { } func getLegacyPrefixKeys(val int) map[string][]byte { + vstr := fmt.Sprintf("%02x", byte(val)) return map[string][]byte{ "Height": []byte(fmt.Sprintf("H:%d", val)), "BlockPart": []byte(fmt.Sprintf("P:%d:%d", val, val)), @@ -40,14 +41,19 @@ func getLegacyPrefixKeys(val int) map[string][]byte { "UserKey1": []byte(fmt.Sprintf("foo/bar/baz/%d/%d", val, val)), "TxHeight": []byte(fmt.Sprintf("tx.height/%s/%d/%d", fmt.Sprint(val), val, val)), "TxHash": append( - bytes.Repeat([]byte{fmt.Sprint(val)[0]}, 16), - bytes.Repeat([]byte{fmt.Sprint(val)[len([]byte(fmt.Sprint(val)))-1]}, 16)..., + []byte(strings.Repeat(vstr[:1], 16)), + []byte(strings.Repeat(vstr[1:], 16))..., ), + + // Transaction hashes that could be mistaken for evidence keys. + "TxHashMimic0": append([]byte{0}, []byte(strings.Repeat(vstr, 16)[:31])...), + "TxHashMimic1": append([]byte{1}, []byte(strings.Repeat(vstr, 16)[:31])...), } } func getNewPrefixKeys(t *testing.T, val int) map[string][]byte { t.Helper() + vstr := fmt.Sprintf("%02x", byte(val)) return map[string][]byte{ "Height": makeKey(t, int64(0), int64(val)), "BlockPart": makeKey(t, int64(1), int64(val), int64(val)), @@ -66,7 +72,9 @@ func getNewPrefixKeys(t *testing.T, val int) map[string][]byte { "UserKey0": makeKey(t, "foo", "bar", int64(val), int64(val)), "UserKey1": makeKey(t, "foo", "bar/baz", int64(val), int64(val)), "TxHeight": makeKey(t, "tx.height", fmt.Sprint(val), int64(val), int64(val+2), int64(val+val)), - "TxHash": makeKey(t, "tx.hash", string(bytes.Repeat([]byte{[]byte(fmt.Sprint(val))[0]}, 32))), + "TxHash": makeKey(t, "tx.hash", strings.Repeat(vstr, 16)), + "TxHashMimic0": makeKey(t, "tx.hash", "\x00"+strings.Repeat(vstr, 16)[:31]), + "TxHashMimic1": makeKey(t, "tx.hash", "\x01"+strings.Repeat(vstr, 16)[:31]), } }