Merge branch 'master' into marko/bringbackdocs

.github/workflows/docker.yml

@@ -39,7 +39,7 @@ jobs:
           platforms: all
 
       - name: Set up Docker Build
-        uses: docker/setup-buildx-action@v1.6.0
+        uses: docker/setup-buildx-action@v1.7.0
 
       - name: Login to DockerHub
         if: ${{ github.event_name != 'pull_request' }}

UPGRADING.md

@@ -212,22 +212,25 @@ and one function have moved to the Tendermint `crypto` package:
 
 The format of all tendermint on-disk database keys changes in
 0.35. Upgrading nodes must either re-sync all data or run a migration
-script provided in this release. The script located in
-`github.com/tendermint/tendermint/scripts/keymigrate/migrate.go`
-provides the function `Migrate(context.Context, db.DB)` which you can
-operationalize as makes sense for your deployment.
+script provided in this release.
+
+The script located in
+`github.com/tendermint/tendermint/scripts/keymigrate/migrate.go` provides the
+function `Migrate(context.Context, db.DB)` which you can operationalize as
+makes sense for your deployment.
 
 For ease of use the `tendermint` command includes a CLI version of the
 migration script, which you can invoke, as in:
 
 	tendermint key-migrate
 
-This reads the configuration file as normal and allows the
-`--db-backend` and `--db-dir` flags to change database operations as
-needed.
+This reads the configuration file as normal and allows the `--db-backend` and
+`--db-dir` flags to override the database location as needed.
 
-The migration operation is idempotent and can be run more than once,
-if needed.
+The migration operation is intended to be idempotent, and should be safe to
+rerun on the same database multiple times.  As a safety measure, however, we
+recommend that operators test out the migration on a copy of the database
+first, if it is practical to do so, before applying it to the production data.
 
 ### CLI Changes
 

docs/tendermint-core/consensus/proposer-based-timestamps.md

@@ -13,14 +13,15 @@ order: 3
 The PBTS algorithm defines a way for a Tendermint blockchain to create block
 timestamps that are within a reasonable bound of the clocks of the validators on
 the network. This replaces the original BFTTime algorithm for timestamp
-assignment that relied on the timestamps included in precommit messages.
+assignment that computed a timestamp using the timestamps included in precommit
+messages.
 
 ## Algorithm Parameters
 
 The functionality of the PBTS algorithm is governed by two parameters within
 Tendermint. These two parameters are [consensus
 parameters](https://github.com/tendermint/tendermint/blob/master/spec/abci/apps.md#L291),
-meaning they are configured by the ABCI application and are expected to be the
+meaning they are configured by the ABCI application and are therefore the same
 same across all nodes on the network.
 
 ### `Precision`
@@ -51,7 +52,7 @@ useful for the protocols and applications built on top of Tendermint.
 The following protocols and application features require a reliable source of time:
 
 * Tendermint Light Clients [rely on correspondence between their known time](https://github.com/tendermint/tendermint/blob/master/spec/light-client/verification/README.md#definitions-1) and the block time for block verification.
-* Tendermint Evidence validity is determined [either in terms of heights or in terms of time](https://github.com/tendermint/tendermint/blob/master/spec/consensus/evidence.md#verification).
+* Tendermint Evidence expiration is determined [either in terms of heights or in terms of time](https://github.com/tendermint/tendermint/blob/master/spec/consensus/evidence.md#verification).
 * Unbonding of staked assets in the Cosmos Hub [occurs after a period of 21
  days](https://github.com/cosmos/governance/blob/master/params-change/Staking.md#unbondingtime).
 * IBC packets can use either a [timestamp or a height to timeout packet

go.mod

@@ -38,7 +38,7 @@ require (
 )
 
 require (
-	github.com/creachadair/atomicfile v0.2.5
+	github.com/creachadair/atomicfile v0.2.6
 	github.com/creachadair/taskgroup v0.3.2
 	github.com/golangci/golangci-lint v1.45.2
 	github.com/google/go-cmp v0.5.8

go.sum

@@ -227,8 +227,8 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
-github.com/creachadair/atomicfile v0.2.5 h1:wkOlpsjyJOvJ3Hd8juHKdirJnCSIPacvtY21/3nYjAo=
-github.com/creachadair/atomicfile v0.2.5/go.mod h1:BRq8Une6ckFneYXZQ+kO7p1ZZP3I2fzVzf28JxrIkBc=
+github.com/creachadair/atomicfile v0.2.6 h1:FgYxYvGcqREApTY8Nxg8msM6P/KVKK3ob5h9FaRUTNg=
+github.com/creachadair/atomicfile v0.2.6/go.mod h1:BRq8Une6ckFneYXZQ+kO7p1ZZP3I2fzVzf28JxrIkBc=
 github.com/creachadair/taskgroup v0.3.2 h1:zlfutDS+5XG40AOxcHDSThxKzns8Tnr9jnr6VqkYlkM=
 github.com/creachadair/taskgroup v0.3.2/go.mod h1:wieWwecHVzsidg2CsUnFinW1faVN4+kq+TDlRJQ0Wbk=
 github.com/creachadair/tomledit v0.0.19 h1:zbpfUtYFYFdpRjwJY9HJlto1iZ4M5YwYB6qqc37F6UM=

internal/p2p/channel.go

@@ -59,25 +59,17 @@ type Channel struct {
 	outCh chan<- Envelope  // outbound messages (reactors to peers)
 	errCh chan<- PeerError // peer error reporting
 
-	messageType proto.Message // the channel's message type, used for unmarshaling
-	name        string
+	name string
 }
 
 // NewChannel creates a new channel. It is primarily for internal and test
 // use, reactors should use Router.OpenChannel().
-func NewChannel(
-	id ChannelID,
-	messageType proto.Message,
-	inCh <-chan Envelope,
-	outCh chan<- Envelope,
-	errCh chan<- PeerError,
-) *Channel {
+func NewChannel(id ChannelID, inCh <-chan Envelope, outCh chan<- Envelope, errCh chan<- PeerError) *Channel {
 	return &Channel{
-		ID:          id,
-		messageType: messageType,
-		inCh:        inCh,
-		outCh:       outCh,
-		errCh:       errCh,
+		ID:    id,
+		inCh:  inCh,
+		outCh: outCh,
+		errCh: errCh,
 	}
 }
 

internal/p2p/pex/reactor_test.go

@@ -289,7 +289,6 @@ func setupSingle(ctx context.Context, t *testing.T) *singleTestReactor {
 	pexErrCh := make(chan p2p.PeerError, chBuf)
 	pexCh := p2p.NewChannel(
 		p2p.ChannelID(pex.PexChannel),
-		new(p2pproto.PexMessage),
 		pexInCh,
 		pexOutCh,
 		pexErrCh,

internal/p2p/router.go

@@ -262,7 +262,7 @@ func (r *Router) OpenChannel(ctx context.Context, chDesc *ChannelDescriptor) (*C
 	queue := r.queueFactory(chDesc.RecvBufferCapacity)
 	outCh := make(chan Envelope, chDesc.RecvBufferCapacity)
 	errCh := make(chan PeerError, chDesc.RecvBufferCapacity)
-	channel := NewChannel(id, messageType, queue.dequeue(), outCh, errCh)
+	channel := NewChannel(id, queue.dequeue(), outCh, errCh)
 	channel.name = chDesc.Name
 
 	var wrapper Wrapper

internal/statesync/dispatcher_test.go

@@ -30,7 +30,7 @@ func testChannel(size int) (*channelInternal, *p2p.Channel) {
 		Out:   make(chan p2p.Envelope, size),
 		Error: make(chan p2p.PeerError, size),
 	}
-	return in, p2p.NewChannel(0, nil, in.In, in.Out, in.Error)
+	return in, p2p.NewChannel(0, in.In, in.Out, in.Error)
 }
 
 func TestDispatcherBasic(t *testing.T) {

internal/statesync/reactor_test.go

@@ -102,7 +102,6 @@ func setup(
 
 	rts.snapshotChannel = p2p.NewChannel(
 		SnapshotChannel,
-		new(ssproto.Message),
 		rts.snapshotInCh,
 		rts.snapshotOutCh,
 		rts.snapshotPeerErrCh,
@@ -110,7 +109,6 @@ func setup(
 
 	rts.chunkChannel = p2p.NewChannel(
 		ChunkChannel,
-		new(ssproto.Message),
 		rts.chunkInCh,
 		rts.chunkOutCh,
 		rts.chunkPeerErrCh,
@@ -118,7 +116,6 @@ func setup(
 
 	rts.blockChannel = p2p.NewChannel(
 		LightBlockChannel,
-		new(ssproto.Message),
 		rts.blockInCh,
 		rts.blockOutCh,
 		rts.blockPeerErrCh,
@@ -126,7 +123,6 @@ func setup(
 
 	rts.paramsChannel = p2p.NewChannel(
 		ParamsChannel,
-		new(ssproto.Message),
 		rts.paramsInCh,
 		rts.paramsOutCh,
 		rts.paramsPeerErrCh,

scripts/keymigrate/migrate.go

@@ -86,27 +86,30 @@ const (
 var prefixes = []struct {
 	prefix []byte
 	ktype  keyType
+	check  func(keyID) bool
 }{
-	{[]byte("consensusParamsKey:"), consensusParamsKey},
-	{[]byte("abciResponsesKey:"), abciResponsesKey},
-	{[]byte("validatorsKey:"), validatorsKey},
-	{[]byte("stateKey"), stateStoreKey},
-	{[]byte("H:"), blockMetaKey},
-	{[]byte("P:"), blockPartKey},
-	{[]byte("C:"), commitKey},
-	{[]byte("SC:"), seenCommitKey},
-	{[]byte("BH:"), blockHashKey},
-	{[]byte("size"), lightSizeKey},
-	{[]byte("lb/"), lightBlockKey},
-	{[]byte("\x00"), evidenceCommittedKey},
-	{[]byte("\x01"), evidencePendingKey},
+	{[]byte("consensusParamsKey:"), consensusParamsKey, nil},
+	{[]byte("abciResponsesKey:"), abciResponsesKey, nil},
+	{[]byte("validatorsKey:"), validatorsKey, nil},
+	{[]byte("stateKey"), stateStoreKey, nil},
+	{[]byte("H:"), blockMetaKey, nil},
+	{[]byte("P:"), blockPartKey, nil},
+	{[]byte("C:"), commitKey, nil},
+	{[]byte("SC:"), seenCommitKey, nil},
+	{[]byte("BH:"), blockHashKey, nil},
+	{[]byte("size"), lightSizeKey, nil},
+	{[]byte("lb/"), lightBlockKey, nil},
+	{[]byte("\x00"), evidenceCommittedKey, checkEvidenceKey},
+	{[]byte("\x01"), evidencePendingKey, checkEvidenceKey},
 }
 
 // checkKeyType classifies a candidate key based on its structure.
 func checkKeyType(key keyID) keyType {
 	for _, p := range prefixes {
 		if bytes.HasPrefix(key, p.prefix) {
-			return p.ktype
+			if p.check == nil || p.check(key) {
+				return p.ktype
+			}
 		}
 	}
 
@@ -342,6 +345,35 @@ func convertEvidence(key keyID, newPrefix int64) ([]byte, error) {
 	return orderedcode.Append(nil, newPrefix, binary.BigEndian.Uint64(hb), string(evidenceHash))
 }
 
+// checkEvidenceKey reports whether a candidate key with one of the legacy
+// evidence prefixes has the correct structure for a legacy evidence key.
+//
+// This check is needed because transaction hashes are stored without a prefix,
+// so checking the one-byte prefix alone is not enough to distinguish them.
+// Legacy evidence keys are suffixed with a string of the format:
+//
+//     "%0.16X/%X"
+//
+// where the first element is the height and the second is the hash.  Thus, we
+// check
+func checkEvidenceKey(key keyID) bool {
+	parts := bytes.SplitN(key[1:], []byte("/"), 2)
+	if len(parts) != 2 || len(parts[0]) != 16 || !isHex(parts[0]) || !isHex(parts[1]) {
+		return false
+	}
+	return true
+}
+
+func isHex(data []byte) bool {
+	for _, b := range data {
+		if ('0' <= b && b <= '9') || ('a' <= b && b <= 'f') || ('A' <= b && b <= 'F') {
+			continue
+		}
+		return false
+	}
+	return len(data) != 0
+}
+
 func replaceKey(db dbm.DB, key keyID, gooseFn migrateFunc) error {
 	exists, err := db.Has(key)
 	if err != nil {

scripts/keymigrate/migrate_test.go

@@ -1,11 +1,11 @@
 package keymigrate
 
 import (
-	"bytes"
 	"context"
 	"errors"
 	"fmt"
 	"math"
+	"strings"
 	"testing"
 
 	"github.com/google/orderedcode"
@@ -21,6 +21,7 @@ func makeKey(t *testing.T, elems ...interface{}) []byte {
 }
 
 func getLegacyPrefixKeys(val int) map[string][]byte {
+	vstr := fmt.Sprintf("%02x", byte(val))
 	return map[string][]byte{
 		"Height":            []byte(fmt.Sprintf("H:%d", val)),
 		"BlockPart":         []byte(fmt.Sprintf("P:%d:%d", val, val)),
@@ -40,14 +41,19 @@ func getLegacyPrefixKeys(val int) map[string][]byte {
 		"UserKey1":          []byte(fmt.Sprintf("foo/bar/baz/%d/%d", val, val)),
 		"TxHeight":          []byte(fmt.Sprintf("tx.height/%s/%d/%d", fmt.Sprint(val), val, val)),
 		"TxHash": append(
-			bytes.Repeat([]byte{fmt.Sprint(val)[0]}, 16),
-			bytes.Repeat([]byte{fmt.Sprint(val)[len([]byte(fmt.Sprint(val)))-1]}, 16)...,
+			[]byte(strings.Repeat(vstr[:1], 16)),
+			[]byte(strings.Repeat(vstr[1:], 16))...,
 		),
+
+		// Transaction hashes that could be mistaken for evidence keys.
+		"TxHashMimic0": append([]byte{0}, []byte(strings.Repeat(vstr, 16)[:31])...),
+		"TxHashMimic1": append([]byte{1}, []byte(strings.Repeat(vstr, 16)[:31])...),
 	}
 }
 
 func getNewPrefixKeys(t *testing.T, val int) map[string][]byte {
 	t.Helper()
+	vstr := fmt.Sprintf("%02x", byte(val))
 	return map[string][]byte{
 		"Height":            makeKey(t, int64(0), int64(val)),
 		"BlockPart":         makeKey(t, int64(1), int64(val), int64(val)),
@@ -66,7 +72,9 @@ func getNewPrefixKeys(t *testing.T, val int) map[string][]byte {
 		"UserKey0":          makeKey(t, "foo", "bar", int64(val), int64(val)),
 		"UserKey1":          makeKey(t, "foo", "bar/baz", int64(val), int64(val)),
 		"TxHeight":          makeKey(t, "tx.height", fmt.Sprint(val), int64(val), int64(val+2), int64(val+val)),
-		"TxHash":            makeKey(t, "tx.hash", string(bytes.Repeat([]byte{[]byte(fmt.Sprint(val))[0]}, 32))),
+		"TxHash":            makeKey(t, "tx.hash", strings.Repeat(vstr, 16)),
+		"TxHashMimic0":      makeKey(t, "tx.hash", "\x00"+strings.Repeat(vstr, 16)[:31]),
+		"TxHashMimic1":      makeKey(t, "tx.hash", "\x01"+strings.Repeat(vstr, 16)[:31]),
 	}
 }