From baa20a4b9c219bf173461a3cb2e7440336c63aa8 Mon Sep 17 00:00:00 2001 From: Marko Date: Wed, 11 Nov 2020 09:32:57 +0100 Subject: [PATCH 01/27] fix docker deployment (#5647) --- .github/workflows/docker.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 35d132b79..0a1dab464 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -2,6 +2,7 @@ name: Build & Push # Build & Push rebuilds the tendermint docker image on every push to master and creation of tags # and pushes the image to https://hub.docker.com/r/interchainio/simapp/tags on: + pull_request: push: branches: - master @@ -13,6 +14,9 @@ jobs: build: runs-on: ubuntu-latest steps: + - uses: actions/setup-go@v2 + with: + go-version: "^1.15.4" - uses: actions/checkout@master - name: Prepare id: prep From a399fae100d07b506c6065a5e3f82c14746fb317 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2020 11:05:17 +0000 Subject: [PATCH 02/27] build(deps): Bump github.com/tendermint/tm-db from 0.6.2 to 0.6.3 Bumps [github.com/tendermint/tm-db](https://github.com/tendermint/tm-db) from 0.6.2 to 0.6.3. - [Release notes](https://github.com/tendermint/tm-db/releases) - [Changelog](https://github.com/tendermint/tm-db/blob/master/CHANGELOG.md) - [Commits](https://github.com/tendermint/tm-db/compare/v0.6.2...v0.6.3) Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 9460cff30..c545fa783 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/spf13/cobra v1.1.1 github.com/spf13/viper v1.7.1 github.com/stretchr/testify v1.6.1 - github.com/tendermint/tm-db v0.6.2 + github.com/tendermint/tm-db v0.6.3 golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc google.golang.org/grpc v1.33.2 diff --git a/go.sum b/go.sum index 6ae0cde04..71ecc0639 100644 --- a/go.sum +++ b/go.sum @@ -108,6 +108,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/dgraph-io/badger/v2 v2.2007.1 h1:t36VcBCpo4SsmAD5M8wVv1ieVzcALyGfaJ92z4ccULM= github.com/dgraph-io/badger/v2 v2.2007.1/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= +github.com/dgraph-io/badger/v2 v2.2007.2 h1:EjjK0KqwaFMlPin1ajhP943VPENHJdEz1KLIegjaI3k= +github.com/dgraph-io/badger/v2 v2.2007.2/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de h1:t0UHb5vdojIDUqktM6+xJAfScFBsVpXZmqC9dsgJmeA= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= @@ -517,6 +519,8 @@ github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c/go.mod h1:ahpPrc7 github.com/tendermint/tendermint v0.34.0-rc4/go.mod h1:yotsojf2C1QBOw4dZrTcxbyxmPUrT4hNuOQWX9XUwB4= github.com/tendermint/tm-db v0.6.2 h1:DOn8jwCdjJblrCFJbtonEIPD1IuJWpbRUUdR8GWE4RM= github.com/tendermint/tm-db v0.6.2/go.mod h1:GYtQ67SUvATOcoY8/+x6ylk8Qo02BQyLrAs+yAcLvGI= +github.com/tendermint/tm-db v0.6.3 h1:ZkhQcKnB8/2jr5EaZwGndN4owkPsGezW2fSisS9zGbg= +github.com/tendermint/tm-db v0.6.3/go.mod h1:lfA1dL9/Y/Y8wwyPp2NMLyn5P5Ptr/gvDFNWtrCWSf8= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= From eb0d35376719b8f80f4b0c5d294d974b630acdac Mon Sep 17 00:00:00 2001 From: Alessio Treglia Date: Thu, 12 Nov 2020 06:19:05 +0000 Subject: [PATCH 03/27] libs/os: add test case for TrapSignal (#5646) --- libs/os/os.go | 16 ++++++------- libs/os/os_test.go | 56 ++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 62 insertions(+), 10 deletions(-) diff --git a/libs/os/os.go b/libs/os/os.go index ea24a42f6..c2f37992a 100644 --- a/libs/os/os.go +++ b/libs/os/os.go @@ -13,19 +13,19 @@ type logger interface { Info(msg string, keyvals ...interface{}) } -// TrapSignal catches the SIGTERM/SIGINT and executes cb function. After that it exits -// with code 0. +// TrapSignal catches SIGTERM and SIGINT, executes the cleanup function, +// and exits with code 0. func TrapSignal(logger logger, cb func()) { c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt, syscall.SIGTERM) + go func() { - for sig := range c { - logger.Info(fmt.Sprintf("captured %v, exiting...", sig)) - if cb != nil { - cb() - } - os.Exit(0) + sig := <-c + logger.Info(fmt.Sprintf("captured %v, exiting...", sig)) + if cb != nil { + cb() } + os.Exit(0) }() } diff --git a/libs/os/os_test.go b/libs/os/os_test.go index 9c80f1f5a..70086afad 100644 --- a/libs/os/os_test.go +++ b/libs/os/os_test.go @@ -1,11 +1,15 @@ -package os +package os_test import ( "bytes" "fmt" "io/ioutil" "os" + "os/exec" "testing" + "time" + + tmos "github.com/tendermint/tendermint/libs/os" ) func TestCopyFile(t *testing.T) { @@ -20,7 +24,7 @@ func TestCopyFile(t *testing.T) { } copyfile := fmt.Sprintf("%s.copy", tmpfile.Name()) - if err := CopyFile(tmpfile.Name(), copyfile); err != nil { + if err := tmos.CopyFile(tmpfile.Name(), copyfile); err != nil { t.Fatal(err) } if _, err := os.Stat(copyfile); os.IsNotExist(err) { @@ -35,3 +39,51 @@ func TestCopyFile(t *testing.T) { } os.Remove(copyfile) } + +func TestTrapSignal(t *testing.T) { + if os.Getenv("TM_TRAP_SIGNAL_TEST") == "1" { + t.Log("inside test process") + killer() + return + } + + cmd := exec.Command(os.Args[0], "-test.run="+t.Name()) + mockStderr := bytes.NewBufferString("") + cmd.Env = append(os.Environ(), "TM_TRAP_SIGNAL_TEST=1") + cmd.Stderr = mockStderr + + err := cmd.Run() + if err == nil { + wantStderr := "exiting" + if mockStderr.String() != wantStderr { + t.Fatalf("stderr: want %q, got %q", wantStderr, mockStderr.String()) + } + + return + } + + if e, ok := err.(*exec.ExitError); ok && !e.Success() { + t.Fatalf("wrong exit code, want 0, got %d", e.ExitCode()) + } + + t.Fatal("this error should not be triggered") + +} + +type mockLogger struct{} + +func (ml mockLogger) Info(msg string, keyvals ...interface{}) {} + +func killer() { + logger := mockLogger{} + + tmos.TrapSignal(logger, func() { _, _ = fmt.Fprintf(os.Stderr, "exiting") }) + time.Sleep(1 * time.Second) + + // use Kill() to test SIGTERM + if err := tmos.Kill(); err != nil { + panic(err) + } + + time.Sleep(1 * time.Second) +} From 95cff1efb4131efa654aeec971bd124ebe57d14a Mon Sep 17 00:00:00 2001 From: Marko Date: Thu, 12 Nov 2020 12:15:18 +0100 Subject: [PATCH 04/27] proto: buf for everything (#5650) ## Description - remove installation of protoc - use buf protoc to generate proto stubs prior to approving could someone test locally. I restarted my docker instance and its been stuck for 20+ minutes Closes: #XXX --- CONTRIBUTING.md | 4 +-- Makefile | 4 +-- proto/tendermint/types/evidence.proto | 18 ++++++------ proto/tendermint/types/types.proto | 8 ++--- scripts/protocgen.sh | 2 +- tests.mk => test/Makefile | 0 tools.mk => tools/Makefile | 42 +++++++++------------------ 7 files changed, 32 insertions(+), 46 deletions(-) rename tests.mk => test/Makefile (100%) rename tools.mk => tools/Makefile (70%) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d4cbf86ae..d94ecb137 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -106,12 +106,12 @@ specify exactly the dependency you want to update, eg. We use [Protocol Buffers](https://developers.google.com/protocol-buffers) along with [gogoproto](https://github.com/gogo/protobuf) to generate code for use across Tendermint Core. -For linting and checking breaking changes, we use [buf](https://buf.build/). If you would like to run linting and check if the changes you have made are breaking then you will need to have docker running locally. Then the linting cmd will be `make proto-lint` and the breaking changes check will be `make proto-check-breaking`. +For linting, checking breaking changes and generating proto stubs, we use [buf](https://buf.build/). If you would like to run linting and check if the changes you have made are breaking then you will need to have docker running locally. Then the linting cmd will be `make proto-lint` and the breaking changes check will be `make proto-check-breaking`. There are two ways to generate your proto stubs. 1. Use Docker, pull an image that will generate your proto stubs with no need to install anything. `make proto-gen-docker` -2. Run `make proto-gen` after installing `protoc` and gogoproto, you can do this by running `make protobuf`. +2. Run `make proto-gen` after installing `buf` and `gogoproto`, you can do this by running `make protobuf`. ### Installation Instructions diff --git a/Makefile b/Makefile index 8789dc3b0..b73a51467 100644 --- a/Makefile +++ b/Makefile @@ -52,8 +52,8 @@ all: check build test install .PHONY: all # The below include contains the tools. -include tools.mk -include tests.mk +include tools/Makefile +include test/Makefile ############################################################################### ### Build Tendermint ### diff --git a/proto/tendermint/types/evidence.proto b/proto/tendermint/types/evidence.proto index 3b234571b..451b8dca3 100644 --- a/proto/tendermint/types/evidence.proto +++ b/proto/tendermint/types/evidence.proto @@ -17,20 +17,20 @@ message Evidence { // DuplicateVoteEvidence contains evidence of a validator signed two conflicting votes. message DuplicateVoteEvidence { - tendermint.types.Vote vote_a = 1; - tendermint.types.Vote vote_b = 2; - int64 total_voting_power = 3; - int64 validator_power = 4; - google.protobuf.Timestamp timestamp = 5 [(gogoproto.nullable) = false, (gogoproto.stdtime) = true]; + tendermint.types.Vote vote_a = 1; + tendermint.types.Vote vote_b = 2; + int64 total_voting_power = 3; + int64 validator_power = 4; + google.protobuf.Timestamp timestamp = 5 [(gogoproto.nullable) = false, (gogoproto.stdtime) = true]; } // LightClientAttackEvidence contains evidence of a set of validators attempting to mislead a light client. message LightClientAttackEvidence { - tendermint.types.LightBlock conflicting_block = 1; - int64 common_height = 2; + tendermint.types.LightBlock conflicting_block = 1; + int64 common_height = 2; repeated tendermint.types.Validator byzantine_validators = 3; - int64 total_voting_power = 4; - google.protobuf.Timestamp timestamp = 5 [(gogoproto.nullable) = false, (gogoproto.stdtime) = true]; + int64 total_voting_power = 4; + google.protobuf.Timestamp timestamp = 5 [(gogoproto.nullable) = false, (gogoproto.stdtime) = true]; } message EvidenceList { diff --git a/proto/tendermint/types/types.proto b/proto/tendermint/types/types.proto index 7f7ea74ca..8d4f00972 100644 --- a/proto/tendermint/types/types.proto +++ b/proto/tendermint/types/types.proto @@ -106,10 +106,10 @@ message Vote { // Commit contains the evidence that a block was committed by a set of validators. message Commit { - int64 height = 1; - int32 round = 2; - BlockID block_id = 3 [(gogoproto.nullable) = false, (gogoproto.customname) = "BlockID"]; - repeated CommitSig signatures = 4 [(gogoproto.nullable) = false]; + int64 height = 1; + int32 round = 2; + BlockID block_id = 3 [(gogoproto.nullable) = false, (gogoproto.customname) = "BlockID"]; + repeated CommitSig signatures = 4 [(gogoproto.nullable) = false]; } // CommitSig is a part of the Vote included in a Commit. diff --git a/scripts/protocgen.sh b/scripts/protocgen.sh index 91dd47056..16f0ec733 100755 --- a/scripts/protocgen.sh +++ b/scripts/protocgen.sh @@ -4,7 +4,7 @@ set -eo pipefail proto_dirs=$(find ./proto -path -prune -o -name '*.proto' -print0 | xargs -0 -n1 dirname | sort | uniq) for dir in $proto_dirs; do - protoc \ + buf protoc \ -I "proto" \ -I "third_party/proto" \ --gogofaster_out=\ diff --git a/tests.mk b/test/Makefile similarity index 100% rename from tests.mk rename to test/Makefile diff --git a/tools.mk b/tools/Makefile similarity index 70% rename from tools.mk rename to tools/Makefile index 6e1a61ca0..fa9081ce2 100644 --- a/tools.mk +++ b/tools/Makefile @@ -43,6 +43,11 @@ TOOLS_DESTDIR ?= $(GOPATH)/bin CERTSTRAP = $(TOOLS_DESTDIR)/certstrap PROTOBUF = $(TOOLS_DESTDIR)/protoc GOODMAN = $(TOOLS_DESTDIR)/goodman +BUF_VERSION = "0.30.0" +BINARY_NAME = "buf" +BIN = "/usr/local/bin" +OS = $(shell uname -s) +ARCH = $(shell uname -m) all: tools .PHONY: all @@ -71,6 +76,14 @@ $(PROTOBUF): @go get github.com/gogo/protobuf/protoc-gen-gogofaster@v1.3.1 .PHONY: protobuf +buf: + @echo "Install Buf" + curl -sSL \ + "https://github.com/bufbuild/buf/releases/download/v$(BUF_VERSION)/$(BINARY_NAME)-$(OS)-$(ARCH)" \ + -o "${BIN}/${BINARY_NAME}" && \ + chmod +x "${BIN}/${BINARY_NAME}" +.PHONY: buf + goodman: $(GOODMAN) $(GOODMAN): @echo "Get Goodman" @@ -80,32 +93,5 @@ $(GOODMAN): tools-clean: rm -f $(CERTSTRAP) $(PROTOBUF) $(GOX) $(GOODMAN) rm -f tools-stamp - rm -rf /usr/local/include/google/protobuf - rm -f /usr/local/bin/protoc + rm -f "${BIN}/${BINARY_NAME}" .PHONY: tooks-clean - -### -# Non Go tools -### - -# Choose protobuf binary based on OS (only works for 64bit Linux and Mac). -# NOTE: On Mac, installation via brew (brew install protoc) might be favorable. -PROTOC_ZIP="" -ifneq ($(OS),Windows_NT) - UNAME_S := $(shell uname -s) - ifeq ($(UNAME_S),Linux) - PROTOC_ZIP="protoc-3.10.1-linux-x86_64.zip" - endif - ifeq ($(UNAME_S),Darwin) - PROTOC_ZIP="protoc-3.10.1-osx-x86_64.zip" - endif -endif - -protoc: - @echo "Get Protobuf" - @echo "In case of any errors, please install directly from https://github.com/protocolbuffers/protobuf/releases" - @curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v3.10.1/$(PROTOC_ZIP) - @unzip -o $(PROTOC_ZIP) -d /usr/local bin/protoc - @unzip -o $(PROTOC_ZIP) -d /usr/local 'include/*' - @rm -f $(PROTOC_ZIP) -.PHONY: protoc From 8bd3d5105f2cb56e37d9ee3bc08bc1fce37e50e2 Mon Sep 17 00:00:00 2001 From: Alessio Treglia Date: Fri, 13 Nov 2020 10:59:45 +0000 Subject: [PATCH 05/27] libs/os: remove unused aliases, add test cases (#5654) Remove unused ReadFile (unused) and WriteFile (almost unused, alias of ioutil.WriteFile). Add testcases for Must{Read,Write}File. --- CHANGELOG_PENDING.md | 2 ++ config/toml.go | 15 ++++++++--- libs/autofile/autofile_test.go | 43 ++++++++++++++---------------- libs/fail/fail.go | 11 ++------ libs/os/os.go | 34 ----------------------- libs/os/os_test.go | 27 ++++++++++++++----- p2p/conn/secret_connection_test.go | 5 ++-- proxy/multi_app_conn.go | 15 ++++++++--- types/genesis.go | 4 +-- 9 files changed, 71 insertions(+), 85 deletions(-) diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 48f296b82..56e4ba1b4 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -18,6 +18,8 @@ Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermi - Go API +- [libs/os] Kill() and {Must,}{Read,Write}File() functions have been removed. (@alessio) + - Blockchain Protocol ### FEATURES diff --git a/config/toml.go b/config/toml.go index 0dce3e6dc..19973a3fb 100644 --- a/config/toml.go +++ b/config/toml.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" "io/ioutil" + "os" "path/filepath" "strings" "text/template" @@ -63,7 +64,7 @@ func WriteConfigFile(configFilePath string, config *Config) { panic(err) } - tmos.MustWriteFile(configFilePath, buffer.Bytes(), 0644) + mustWriteFile(configFilePath, buffer.Bytes(), 0644) } // Note: any changes to the comments/variables/mapstructure @@ -492,16 +493,22 @@ func ResetTestRootWithChainID(testName string, chainID string) *Config { chainID = "tendermint_test" } testGenesis := fmt.Sprintf(testGenesisFmt, chainID) - tmos.MustWriteFile(genesisFilePath, []byte(testGenesis), 0644) + mustWriteFile(genesisFilePath, []byte(testGenesis), 0644) } // we always overwrite the priv val - tmos.MustWriteFile(privKeyFilePath, []byte(testPrivValidatorKey), 0644) - tmos.MustWriteFile(privStateFilePath, []byte(testPrivValidatorState), 0644) + mustWriteFile(privKeyFilePath, []byte(testPrivValidatorKey), 0644) + mustWriteFile(privStateFilePath, []byte(testPrivValidatorState), 0644) config := TestConfig().SetRoot(rootDir) return config } +func mustWriteFile(filePath string, contents []byte, mode os.FileMode) { + if err := ioutil.WriteFile(filePath, contents, mode); err != nil { + tmos.Exit(fmt.Sprintf("failed to write file: %v", err)) + } +} + var testGenesisFmt = `{ "genesis_time": "2018-10-10T08:20:13.695936996Z", "chain_id": "%s", diff --git a/libs/autofile/autofile_test.go b/libs/autofile/autofile_test.go index 3e3814d5e..c2442a56f 100644 --- a/libs/autofile/autofile_test.go +++ b/libs/autofile/autofile_test.go @@ -10,8 +10,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - - tmos "github.com/tendermint/tendermint/libs/os" ) func TestSIGHUP(t *testing.T) { @@ -27,10 +25,9 @@ func TestSIGHUP(t *testing.T) { dir, err := ioutil.TempDir("", "sighup_test") require.NoError(t, err) t.Cleanup(func() { - os.RemoveAll(dir) + _ = os.RemoveAll(dir) }) - err = os.Chdir(dir) - require.NoError(t, err) + require.NoError(t, os.Chdir(dir)) // Create an AutoFile in the temporary directory name := "sighup_test" @@ -45,19 +42,16 @@ func TestSIGHUP(t *testing.T) { require.NoError(t, err) // Move the file over - err = os.Rename(name, name+"_old") - require.NoError(t, err) + require.NoError(t, os.Rename(name, name+"_old")) // Move into a different temporary directory otherDir, err := ioutil.TempDir("", "sighup_test_other") require.NoError(t, err) - defer os.RemoveAll(otherDir) - err = os.Chdir(otherDir) - require.NoError(t, err) + t.Cleanup(func() { os.RemoveAll(otherDir) }) + require.NoError(t, os.Chdir(otherDir)) // Send SIGHUP to self. - err = syscall.Kill(syscall.Getpid(), syscall.SIGHUP) - require.NoError(t, err) + require.NoError(t, syscall.Kill(syscall.Getpid(), syscall.SIGHUP)) // Wait a bit... signals are not handled synchronously. time.Sleep(time.Millisecond * 10) @@ -67,14 +61,13 @@ func TestSIGHUP(t *testing.T) { require.NoError(t, err) _, err = af.Write([]byte("Line 4\n")) require.NoError(t, err) - err = af.Close() - require.NoError(t, err) + require.NoError(t, af.Close()) // Both files should exist - if body := tmos.MustReadFile(filepath.Join(dir, name+"_old")); string(body) != "Line 1\nLine 2\n" { + if body := mustReadFile(t, filepath.Join(dir, name+"_old")); string(body) != "Line 1\nLine 2\n" { t.Errorf("unexpected body %s", body) } - if body := tmos.MustReadFile(filepath.Join(dir, name)); string(body) != "Line 3\nLine 4\n" { + if body := mustReadFile(t, filepath.Join(dir, name)); string(body) != "Line 3\nLine 4\n" { t.Errorf("unexpected body %s", body) } @@ -115,8 +108,7 @@ func TestAutoFileSize(t *testing.T) { // First, create an AutoFile writing to a tempfile dir f, err := ioutil.TempFile("", "sighup_test") require.NoError(t, err) - err = f.Close() - require.NoError(t, err) + require.NoError(t, f.Close()) // Here is the actual AutoFile. af, err := OpenAutoFile(f.Name()) @@ -136,14 +128,19 @@ func TestAutoFileSize(t *testing.T) { require.NoError(t, err) // 3. Not existing file - err = af.Close() - require.NoError(t, err) - err = os.Remove(f.Name()) - require.NoError(t, err) + require.NoError(t, af.Close()) + require.NoError(t, os.Remove(f.Name())) size, err = af.Size() require.EqualValues(t, 0, size, "Expected a new file to be empty") require.NoError(t, err) // Cleanup - _ = os.Remove(f.Name()) + t.Cleanup(func() { os.Remove(f.Name()) }) +} + +func mustReadFile(t *testing.T, filePath string) []byte { + fileBytes, err := ioutil.ReadFile(filePath) + require.NoError(t, err) + + return fileBytes } diff --git a/libs/fail/fail.go b/libs/fail/fail.go index 38cec9a29..03a2ca668 100644 --- a/libs/fail/fail.go +++ b/libs/fail/fail.go @@ -32,16 +32,9 @@ func Fail() { } if callIndex == callIndexToFail { - Exit() + fmt.Printf("*** fail-test %d ***\n", callIndex) + os.Exit(1) } callIndex++ } - -func Exit() { - fmt.Printf("*** fail-test %d ***\n", callIndex) - os.Exit(1) - // proc, _ := os.FindProcess(os.Getpid()) - // proc.Signal(os.Interrupt) - // panic(fmt.Sprintf("*** fail-test %d ***", callIndex)) -} diff --git a/libs/os/os.go b/libs/os/os.go index c2f37992a..733f7e942 100644 --- a/libs/os/os.go +++ b/libs/os/os.go @@ -3,7 +3,6 @@ package os import ( "fmt" "io" - "io/ioutil" "os" "os/signal" "syscall" @@ -29,15 +28,6 @@ func TrapSignal(logger logger, cb func()) { }() } -// Kill the running process by sending itself SIGTERM. -func Kill() error { - p, err := os.FindProcess(os.Getpid()) - if err != nil { - return err - } - return p.Signal(syscall.SIGTERM) -} - func Exit(s string) { fmt.Printf(s + "\n") os.Exit(1) @@ -58,30 +48,6 @@ func FileExists(filePath string) bool { return !os.IsNotExist(err) } -func ReadFile(filePath string) ([]byte, error) { - return ioutil.ReadFile(filePath) -} - -func MustReadFile(filePath string) []byte { - fileBytes, err := ioutil.ReadFile(filePath) - if err != nil { - Exit(fmt.Sprintf("MustReadFile failed: %v", err)) - return nil - } - return fileBytes -} - -func WriteFile(filePath string, contents []byte, mode os.FileMode) error { - return ioutil.WriteFile(filePath, contents, mode) -} - -func MustWriteFile(filePath string, contents []byte, mode os.FileMode) { - err := WriteFile(filePath, contents, mode) - if err != nil { - Exit(fmt.Sprintf("MustWriteFile failed: %v", err)) - } -} - // CopyFile copies a file. It truncates the destination file if it exists. func CopyFile(src, dst string) error { info, err := os.Stat(src) diff --git a/libs/os/os_test.go b/libs/os/os_test.go index 70086afad..2b44dc8e7 100644 --- a/libs/os/os_test.go +++ b/libs/os/os_test.go @@ -6,6 +6,7 @@ import ( "io/ioutil" "os" "os/exec" + "syscall" "testing" "time" @@ -47,10 +48,7 @@ func TestTrapSignal(t *testing.T) { return } - cmd := exec.Command(os.Args[0], "-test.run="+t.Name()) - mockStderr := bytes.NewBufferString("") - cmd.Env = append(os.Environ(), "TM_TRAP_SIGNAL_TEST=1") - cmd.Stderr = mockStderr + cmd, _, mockStderr := newTestProgram(t, "TM_TRAP_SIGNAL_TEST") err := cmd.Run() if err == nil { @@ -67,7 +65,6 @@ func TestTrapSignal(t *testing.T) { } t.Fatal("this error should not be triggered") - } type mockLogger struct{} @@ -80,10 +77,26 @@ func killer() { tmos.TrapSignal(logger, func() { _, _ = fmt.Fprintf(os.Stderr, "exiting") }) time.Sleep(1 * time.Second) - // use Kill() to test SIGTERM - if err := tmos.Kill(); err != nil { + p, err := os.FindProcess(os.Getpid()) + if err != nil { + panic(err) + } + + if err := p.Signal(syscall.SIGTERM); err != nil { panic(err) } time.Sleep(1 * time.Second) } + +func newTestProgram(t *testing.T, environVar string) (cmd *exec.Cmd, stdout *bytes.Buffer, stderr *bytes.Buffer) { + t.Helper() + + cmd = exec.Command(os.Args[0], "-test.run="+t.Name()) + stdout, stderr = bytes.NewBufferString(""), bytes.NewBufferString("") + cmd.Env = append(os.Environ(), fmt.Sprintf("%s=1", environVar)) + cmd.Stdout = stdout + cmd.Stderr = stderr + + return +} diff --git a/p2p/conn/secret_connection_test.go b/p2p/conn/secret_connection_test.go index d4997d81c..e787e1348 100644 --- a/p2p/conn/secret_connection_test.go +++ b/p2p/conn/secret_connection_test.go @@ -6,6 +6,7 @@ import ( "flag" "fmt" "io" + "io/ioutil" "log" "os" "path/filepath" @@ -21,7 +22,6 @@ import ( "github.com/tendermint/tendermint/crypto/ed25519" "github.com/tendermint/tendermint/crypto/sr25519" "github.com/tendermint/tendermint/libs/async" - tmos "github.com/tendermint/tendermint/libs/os" tmrand "github.com/tendermint/tendermint/libs/rand" ) @@ -228,8 +228,7 @@ func TestDeriveSecretsAndChallengeGolden(t *testing.T) { if *update { t.Logf("Updating golden test vector file %s", goldenFilepath) data := createGoldenTestVectors(t) - err := tmos.WriteFile(goldenFilepath, []byte(data), 0644) - require.NoError(t, err) + require.NoError(t, ioutil.WriteFile(goldenFilepath, []byte(data), 0644)) } f, err := os.Open(goldenFilepath) if err != nil { diff --git a/proxy/multi_app_conn.go b/proxy/multi_app_conn.go index a7a6f7014..369b685ba 100644 --- a/proxy/multi_app_conn.go +++ b/proxy/multi_app_conn.go @@ -2,10 +2,11 @@ package proxy import ( "fmt" + "os" + "syscall" abcicli "github.com/tendermint/tendermint/abci/client" tmlog "github.com/tendermint/tendermint/libs/log" - tmos "github.com/tendermint/tendermint/libs/os" "github.com/tendermint/tendermint/libs/service" ) @@ -129,8 +130,7 @@ func (app *multiAppConn) killTMOnClientError() { logger.Error( fmt.Sprintf("%s connection terminated. Did the application crash? Please restart tendermint", conn), "err", err) - killErr := tmos.Kill() - if killErr != nil { + if killErr := kill(); killErr != nil { logger.Error("Failed to kill this process - please do so manually", "err", killErr) } } @@ -189,3 +189,12 @@ func (app *multiAppConn) abciClientFor(conn string) (abcicli.Client, error) { } return c, nil } + +func kill() error { + p, err := os.FindProcess(os.Getpid()) + if err != nil { + return err + } + + return p.Signal(syscall.SIGTERM) +} diff --git a/types/genesis.go b/types/genesis.go index 20fc79721..964df923e 100644 --- a/types/genesis.go +++ b/types/genesis.go @@ -11,7 +11,6 @@ import ( "github.com/tendermint/tendermint/crypto" tmbytes "github.com/tendermint/tendermint/libs/bytes" tmjson "github.com/tendermint/tendermint/libs/json" - tmos "github.com/tendermint/tendermint/libs/os" tmproto "github.com/tendermint/tendermint/proto/tendermint/types" tmtime "github.com/tendermint/tendermint/types/time" ) @@ -52,7 +51,8 @@ func (genDoc *GenesisDoc) SaveAs(file string) error { if err != nil { return err } - return tmos.WriteFile(file, genDocBytes, 0644) + + return ioutil.WriteFile(file, genDocBytes, 0644) // nolint:gosec } // ValidatorHash returns the hash of the validator set contained in the GenesisDoc From fdecfa177d0c253a63149bf038c887a90742e9aa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 13 Nov 2020 11:02:01 +0000 Subject: [PATCH 06/27] build(deps): Bump rtCamp/action-slack-notify from e9db0ef to 2.1.1 Bumps [rtCamp/action-slack-notify](https://github.com/rtCamp/action-slack-notify) from e9db0ef to 2.1.1. This release includes the previously tagged commit. - [Release notes](https://github.com/rtCamp/action-slack-notify/releases) - [Commits](https://github.com/rtCamp/action-slack-notify/compare/e9db0ef...ecc1353ce30ef086ce3fc3d1ea9ac2e32e150402) Signed-off-by: dependabot[bot] --- .github/workflows/e2e-nightly.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e-nightly.yml b/.github/workflows/e2e-nightly.yml index 99b56af9d..b21a482c0 100644 --- a/.github/workflows/e2e-nightly.yml +++ b/.github/workflows/e2e-nightly.yml @@ -42,7 +42,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Notify Slack on failure - uses: rtCamp/action-slack-notify@e9db0ef + uses: rtCamp/action-slack-notify@ecc1353ce30ef086ce3fc3d1ea9ac2e32e150402 env: SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} SLACK_CHANNEL: tendermint-internal From e9294de94698382b0bc9a00aabab265b3c2cbd5d Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Fri, 13 Nov 2020 12:30:34 +0100 Subject: [PATCH 07/27] go.mod: upgrade iavl and deps (#5657) Bumps IAVL, which pulled in some other upgrades as well. I think they should be fine though. --- go.mod | 8 ++++---- go.sum | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index c545fa783..30037fb88 100644 --- a/go.mod +++ b/go.mod @@ -8,8 +8,8 @@ require ( github.com/Workiva/go-datastructures v1.0.52 github.com/btcsuite/btcd v0.21.0-beta github.com/btcsuite/btcutil v1.0.2 - github.com/confio/ics23/go v0.0.0-20200817220745-f173e6211efb - github.com/cosmos/iavl v0.15.0-rc3.0.20201009144442-230e9bdf52cd + github.com/confio/ics23/go v0.6.3 + github.com/cosmos/iavl v0.15.0-rc5 github.com/fortytw2/leaktest v1.3.0 github.com/go-kit/kit v0.10.0 github.com/go-logfmt/logfmt v0.5.0 @@ -30,7 +30,7 @@ require ( github.com/spf13/viper v1.7.1 github.com/stretchr/testify v1.6.1 github.com/tendermint/tm-db v0.6.3 - golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a - golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc + golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897 + golang.org/x/net v0.0.0-20200822124328-c89045814202 google.golang.org/grpc v1.33.2 ) diff --git a/go.sum b/go.sum index 71ecc0639..1358bfb40 100644 --- a/go.sum +++ b/go.sum @@ -82,6 +82,8 @@ github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:z github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/confio/ics23/go v0.0.0-20200817220745-f173e6211efb h1:+7FsS1gZ1Km5LRjGV2hztpier/5i6ngNjvNpxbWP5I0= github.com/confio/ics23/go v0.0.0-20200817220745-f173e6211efb/go.mod h1:E45NqnlpxGnpfTWL/xauN7MRwEE28T4Dd4uraToOaKg= +github.com/confio/ics23/go v0.6.3 h1:PuGK2V1NJWZ8sSkNDq91jgT/cahFEW9RGp4Y5jxulf0= +github.com/confio/ics23/go v0.6.3/go.mod h1:E45NqnlpxGnpfTWL/xauN7MRwEE28T4Dd4uraToOaKg= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible h1:8F3hqu9fGYLBifCmRCJsicFqDx/D68Rt3q1JMazcgBQ= @@ -97,6 +99,8 @@ github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d h1:49RLWk1j44Xu4fj github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d/go.mod h1:tSxLoYXyBmiFeKpvmq4dzayMdCjCnu8uqmCysIGBT2Y= github.com/cosmos/iavl v0.15.0-rc3.0.20201009144442-230e9bdf52cd h1:K3bmPkMDnd2KVQ7xoGmgp+pxoXcBW58vMWaMl9ZWx3c= github.com/cosmos/iavl v0.15.0-rc3.0.20201009144442-230e9bdf52cd/go.mod h1:3xOIaNNX19p0QrX0VqWa6voPRoJRGGYtny+DH8NEPvE= +github.com/cosmos/iavl v0.15.0-rc5 h1:AMKgaAjXwGANWv56NL4q4hV+a0puSkLYD6cCQAv3i44= +github.com/cosmos/iavl v0.15.0-rc5/go.mod h1:WqoPL9yPTQ85QBMT45OOUzPxG/U/JcJoN7uMjgxke/I= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -231,12 +235,15 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.2.1/go.mod h1:EaizFBKfUKtMIF5iaDEhniwNedqGo9FuLFzppDr3uwI= +github.com/grpc-ecosystem/go-grpc-middleware v1.2.2/go.mod h1:EaizFBKfUKtMIF5iaDEhniwNedqGo9FuLFzppDr3uwI= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.14.7 h1:Nk5kuHrnWUTf/0GL1a/vchH/om9Ap2/HnVna+jYZgTY= github.com/grpc-ecosystem/grpc-gateway v1.14.7/go.mod h1:oYZKL012gGh6LMyg/xA7Q2yq6j8bu0wa+9w14EEthWU= +github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/gtank/merlin v0.1.1-0.20191105220539-8318aed1a79f h1:8N8XWLZelZNibkhM1FuF+3Ad3YIbgirjdMiVA0eUkaM= github.com/gtank/merlin v0.1.1-0.20191105220539-8318aed1a79f/go.mod h1:T86dnYJhcGOh5BjZFCJWTDeTK7XW8uE+E21Cy/bIQ+s= github.com/gtank/merlin v0.1.1 h1:eQ90iG7K9pOhtereWsmyRJ6RAwcP4tHTDBHXNg+u5is= @@ -517,6 +524,7 @@ github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKk github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c h1:g+WoO5jjkqGAzHWCjJB1zZfXPIAaDpzXIEJ0eS6B5Ok= github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c/go.mod h1:ahpPrc7HpcfEWDQRZEmnXMzHY03mLDYMCxeDzy46i+8= github.com/tendermint/tendermint v0.34.0-rc4/go.mod h1:yotsojf2C1QBOw4dZrTcxbyxmPUrT4hNuOQWX9XUwB4= +github.com/tendermint/tendermint v0.34.0-rc6/go.mod h1:ugzyZO5foutZImv0Iyx/gOFCX6mjJTgbLHTwi17VDVg= github.com/tendermint/tm-db v0.6.2 h1:DOn8jwCdjJblrCFJbtonEIPD1IuJWpbRUUdR8GWE4RM= github.com/tendermint/tm-db v0.6.2/go.mod h1:GYtQ67SUvATOcoY8/+x6ylk8Qo02BQyLrAs+yAcLvGI= github.com/tendermint/tm-db v0.6.3 h1:ZkhQcKnB8/2jr5EaZwGndN4owkPsGezW2fSisS9zGbg= @@ -567,6 +575,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnk golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897 h1:pLI5jrR7OSLijeIDcmRxNmw2api+jEfxLoykJVice/E= +golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -616,6 +626,8 @@ golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc h1:zK/HqS5bZxDptfPJNq8v7vJfXtkU7r9TLIoSr1bXaP4= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -737,6 +749,8 @@ google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987 h1:PDIOdWxZ8eRizhKa1AAvY53xsvLB1cWorMjslvY3VA8= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201111145450-ac7456db90a6 h1:iRN4+t0lvZX/l9gH14ARF9i58tsVa5a97k6aH95rC3Y= +google.golang.org/genproto v0.0.0-20201111145450-ac7456db90a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.19.1/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -757,6 +771,7 @@ google.golang.org/grpc v1.31.1 h1:SfXqXS5hkufcdZ/mHtYCh53P2b+92WQq/DZcKLgsFRs= google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.32.0 h1:zWTV+LMdc3kaiJMSTOFz2UgSBgx8RNQoTGiZu3fR9S0= google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2 h1:EQyQC3sa8M+p6Ulc8yy9SWSS2GVwyRc83gAbG8lrl4o= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -782,6 +797,8 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogR gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= From 2f5e4548922ce9dd634802318b9cb4e7de2fb400 Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Fri, 13 Nov 2020 13:09:38 +0100 Subject: [PATCH 08/27] remove unused version struct (#5656) --- abci/types/types.pb.go | 3 +- proto/tendermint/abci/types.proto | 1 + proto/tendermint/version/types.pb.go | 235 ++------------------------- proto/tendermint/version/types.proto | 8 - 4 files changed, 14 insertions(+), 233 deletions(-) diff --git a/abci/types/types.pb.go b/abci/types/types.pb.go index 8801f17d6..67599af15 100644 --- a/abci/types/types.pb.go +++ b/abci/types/types.pb.go @@ -1529,7 +1529,8 @@ func (m *ResponseFlush) XXX_DiscardUnknown() { var xxx_messageInfo_ResponseFlush proto.InternalMessageInfo type ResponseInfo struct { - Data string `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` + Data string `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` + // this is the software version of the application. TODO: remove? Version string `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` AppVersion uint64 `protobuf:"varint,3,opt,name=app_version,json=appVersion,proto3" json:"app_version,omitempty"` LastBlockHeight int64 `protobuf:"varint,4,opt,name=last_block_height,json=lastBlockHeight,proto3" json:"last_block_height,omitempty"` diff --git a/proto/tendermint/abci/types.proto b/proto/tendermint/abci/types.proto index b95a8ed83..e4a1dd1bc 100644 --- a/proto/tendermint/abci/types.proto +++ b/proto/tendermint/abci/types.proto @@ -155,6 +155,7 @@ message ResponseFlush {} message ResponseInfo { string data = 1; + // this is the software version of the application. TODO: remove? string version = 2; uint64 app_version = 3; diff --git a/proto/tendermint/version/types.pb.go b/proto/tendermint/version/types.pb.go index 676872668..9aeb3ae1a 100644 --- a/proto/tendermint/version/types.pb.go +++ b/proto/tendermint/version/types.pb.go @@ -23,61 +23,6 @@ var _ = math.Inf // proto package needs to be updated. const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package -// App includes the protocol and software version for the application. -// This information is included in ResponseInfo. The App.Protocol can be -// updated in ResponseEndBlock. -type App struct { - Protocol uint64 `protobuf:"varint,1,opt,name=protocol,proto3" json:"protocol,omitempty"` - Software string `protobuf:"bytes,2,opt,name=software,proto3" json:"software,omitempty"` -} - -func (m *App) Reset() { *m = App{} } -func (m *App) String() string { return proto.CompactTextString(m) } -func (*App) ProtoMessage() {} -func (*App) Descriptor() ([]byte, []int) { - return fileDescriptor_f9b42966edc5edad, []int{0} -} -func (m *App) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *App) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_App.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *App) XXX_Merge(src proto.Message) { - xxx_messageInfo_App.Merge(m, src) -} -func (m *App) XXX_Size() int { - return m.Size() -} -func (m *App) XXX_DiscardUnknown() { - xxx_messageInfo_App.DiscardUnknown(m) -} - -var xxx_messageInfo_App proto.InternalMessageInfo - -func (m *App) GetProtocol() uint64 { - if m != nil { - return m.Protocol - } - return 0 -} - -func (m *App) GetSoftware() string { - if m != nil { - return m.Software - } - return "" -} - // Consensus captures the consensus rules for processing a block in the blockchain, // including all blockchain data structures and the rules of the application's // state transition machine. @@ -90,7 +35,7 @@ func (m *Consensus) Reset() { *m = Consensus{} } func (m *Consensus) String() string { return proto.CompactTextString(m) } func (*Consensus) ProtoMessage() {} func (*Consensus) Descriptor() ([]byte, []int) { - return fileDescriptor_f9b42966edc5edad, []int{1} + return fileDescriptor_f9b42966edc5edad, []int{0} } func (m *Consensus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -134,28 +79,25 @@ func (m *Consensus) GetApp() uint64 { } func init() { - proto.RegisterType((*App)(nil), "tendermint.version.App") proto.RegisterType((*Consensus)(nil), "tendermint.version.Consensus") } func init() { proto.RegisterFile("tendermint/version/types.proto", fileDescriptor_f9b42966edc5edad) } var fileDescriptor_f9b42966edc5edad = []byte{ - // 218 bytes of a gzipped FileDescriptorProto + // 179 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x2b, 0x49, 0xcd, 0x4b, 0x49, 0x2d, 0xca, 0xcd, 0xcc, 0x2b, 0xd1, 0x2f, 0x4b, 0x2d, 0x2a, 0xce, 0xcc, 0xcf, 0xd3, 0x2f, 0xa9, 0x2c, 0x48, 0x2d, 0xd6, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x12, 0x42, 0xc8, 0xeb, 0x41, - 0xe5, 0xa5, 0x44, 0xd2, 0xf3, 0xd3, 0xf3, 0xc1, 0xd2, 0xfa, 0x20, 0x16, 0x44, 0xa5, 0x92, 0x2d, - 0x17, 0xb3, 0x63, 0x41, 0x81, 0x90, 0x14, 0x17, 0x07, 0x98, 0x9f, 0x9c, 0x9f, 0x23, 0xc1, 0xa8, - 0xc0, 0xa8, 0xc1, 0x12, 0x04, 0xe7, 0x83, 0xe4, 0x8a, 0xf3, 0xd3, 0x4a, 0xca, 0x13, 0x8b, 0x52, - 0x25, 0x98, 0x14, 0x18, 0x35, 0x38, 0x83, 0xe0, 0x7c, 0x25, 0x4b, 0x2e, 0x4e, 0xe7, 0xfc, 0xbc, - 0xe2, 0xd4, 0xbc, 0xe2, 0xd2, 0x62, 0x21, 0x11, 0x2e, 0xd6, 0xa4, 0x9c, 0xfc, 0xe4, 0x6c, 0xa8, - 0x09, 0x10, 0x8e, 0x90, 0x00, 0x17, 0x73, 0x62, 0x41, 0x01, 0x58, 0x27, 0x4b, 0x10, 0x88, 0x69, - 0xc5, 0xf2, 0x62, 0x81, 0x3c, 0xa3, 0x53, 0xf0, 0x89, 0x47, 0x72, 0x8c, 0x17, 0x1e, 0xc9, 0x31, - 0x3e, 0x78, 0x24, 0xc7, 0x38, 0xe1, 0xb1, 0x1c, 0xc3, 0x85, 0xc7, 0x72, 0x0c, 0x37, 0x1e, 0xcb, - 0x31, 0x44, 0x59, 0xa6, 0x67, 0x96, 0x64, 0x94, 0x26, 0xe9, 0x25, 0xe7, 0xe7, 0xea, 0x23, 0x79, - 0x14, 0x89, 0x09, 0xf1, 0x06, 0x66, 0x20, 0x24, 0xb1, 0x81, 0x65, 0x8c, 0x01, 0x01, 0x00, 0x00, - 0xff, 0xff, 0x42, 0x43, 0x65, 0xc7, 0x21, 0x01, 0x00, 0x00, + 0xe5, 0xa5, 0x44, 0xd2, 0xf3, 0xd3, 0xf3, 0xc1, 0xd2, 0xfa, 0x20, 0x16, 0x44, 0xa5, 0x92, 0x25, + 0x17, 0xa7, 0x73, 0x7e, 0x5e, 0x71, 0x6a, 0x5e, 0x71, 0x69, 0xb1, 0x90, 0x08, 0x17, 0x6b, 0x52, + 0x4e, 0x7e, 0x72, 0xb6, 0x04, 0xa3, 0x02, 0xa3, 0x06, 0x4b, 0x10, 0x84, 0x23, 0x24, 0xc0, 0xc5, + 0x9c, 0x58, 0x50, 0x20, 0xc1, 0x04, 0x16, 0x03, 0x31, 0xad, 0x58, 0x5e, 0x2c, 0x90, 0x67, 0x74, + 0x0a, 0x3e, 0xf1, 0x48, 0x8e, 0xf1, 0xc2, 0x23, 0x39, 0xc6, 0x07, 0x8f, 0xe4, 0x18, 0x27, 0x3c, + 0x96, 0x63, 0xb8, 0xf0, 0x58, 0x8e, 0xe1, 0xc6, 0x63, 0x39, 0x86, 0x28, 0xcb, 0xf4, 0xcc, 0x92, + 0x8c, 0xd2, 0x24, 0xbd, 0xe4, 0xfc, 0x5c, 0x7d, 0x24, 0x97, 0x22, 0x31, 0x21, 0xee, 0xc0, 0xf4, + 0x45, 0x12, 0x1b, 0x58, 0xc6, 0x18, 0x10, 0x00, 0x00, 0xff, 0xff, 0xf8, 0xf0, 0x65, 0xd2, 0xe2, + 0x00, 0x00, 0x00, } func (this *Consensus) Equal(that interface{}) bool { @@ -185,41 +127,6 @@ func (this *Consensus) Equal(that interface{}) bool { } return true } -func (m *App) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *App) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *App) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if len(m.Software) > 0 { - i -= len(m.Software) - copy(dAtA[i:], m.Software) - i = encodeVarintTypes(dAtA, i, uint64(len(m.Software))) - i-- - dAtA[i] = 0x12 - } - if m.Protocol != 0 { - i = encodeVarintTypes(dAtA, i, uint64(m.Protocol)) - i-- - dAtA[i] = 0x8 - } - return len(dAtA) - i, nil -} - func (m *Consensus) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -264,22 +171,6 @@ func encodeVarintTypes(dAtA []byte, offset int, v uint64) int { dAtA[offset] = uint8(v) return base } -func (m *App) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if m.Protocol != 0 { - n += 1 + sovTypes(uint64(m.Protocol)) - } - l = len(m.Software) - if l > 0 { - n += 1 + l + sovTypes(uint64(l)) - } - return n -} - func (m *Consensus) Size() (n int) { if m == nil { return 0 @@ -301,110 +192,6 @@ func sovTypes(x uint64) (n int) { func sozTypes(x uint64) (n int) { return sovTypes(uint64((x << 1) ^ uint64((int64(x) >> 63)))) } -func (m *App) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowTypes - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: App: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: App: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Protocol", wireType) - } - m.Protocol = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowTypes - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Protocol |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Software", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowTypes - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthTypes - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthTypes - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Software = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipTypes(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthTypes - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthTypes - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} func (m *Consensus) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 diff --git a/proto/tendermint/version/types.proto b/proto/tendermint/version/types.proto index 6061868bd..3c4e4cc53 100644 --- a/proto/tendermint/version/types.proto +++ b/proto/tendermint/version/types.proto @@ -5,14 +5,6 @@ option go_package = "github.com/tendermint/tendermint/proto/tendermint/version"; import "gogoproto/gogo.proto"; -// App includes the protocol and software version for the application. -// This information is included in ResponseInfo. The App.Protocol can be -// updated in ResponseEndBlock. -message App { - uint64 protocol = 1; - string software = 2; -} - // Consensus captures the consensus rules for processing a block in the blockchain, // including all blockchain data structures and the rules of the application's // state transition machine. From 8aa47c7da5b854a69f367f316a31c03d2fc1e942 Mon Sep 17 00:00:00 2001 From: Aleksandr Bezobchuk Date: Fri, 13 Nov 2020 14:07:10 -0500 Subject: [PATCH 09/27] rpc: fix content-type header (#5661) --- CHANGELOG_PENDING.md | 4 ++-- rpc/jsonrpc/client/http_json_client.go | 23 +++++++++++++++++------ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 56e4ba1b4..9b790f286 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -9,8 +9,7 @@ Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermi ### BREAKING CHANGES - CLI/RPC/Config - - - [config] \#5598 The `test_fuzz` and `test_fuzz_config` P2P settings have been removed. (@erikgrinaker) + - [config] \#5598 The `test_fuzz` and `test_fuzz_config` P2P settings have been removed. (@erikgrinaker) - Apps @@ -31,5 +30,6 @@ Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermi ### BUG FIXES +- [rpc] \#5660 Set `application/json` as the `Content-Type` header in RPC responses. - [types] \#5523 Change json naming of `PartSetHeader` within `BlockID` from `parts` to `part_set_header` (@marbar3778) - [privval] \#5638 Increase read/write timeout to 5s and calculate ping interval based on it (@JoeKash) diff --git a/rpc/jsonrpc/client/http_json_client.go b/rpc/jsonrpc/client/http_json_client.go index c6e1af035..59727390a 100644 --- a/rpc/jsonrpc/client/http_json_client.go +++ b/rpc/jsonrpc/client/http_json_client.go @@ -151,10 +151,13 @@ func NewWithHTTPClient(remote string, client *http.Client) (*Client, error) { } // Call issues a POST HTTP request. Requests are JSON encoded. Content-Type: -// text/json. -func (c *Client) Call(ctx context.Context, method string, - params map[string]interface{}, result interface{}) (interface{}, error) { - +// application/json. +func (c *Client) Call( + ctx context.Context, + method string, + params map[string]interface{}, + result interface{}, +) (interface{}, error) { id := c.nextRequestID() request, err := types.MapToRequest(id, method, params) @@ -172,14 +175,18 @@ func (c *Client) Call(ctx context.Context, method string, if err != nil { return nil, fmt.Errorf("request failed: %w", err) } - httpRequest.Header.Set("Content-Type", "text/json") + + httpRequest.Header.Set("Content-Type", "application/json") + if c.username != "" || c.password != "" { httpRequest.SetBasicAuth(c.username, c.password) } + httpResponse, err := c.client.Do(httpRequest) if err != nil { return nil, fmt.Errorf("post failed: %w", err) } + defer httpResponse.Body.Close() responseBytes, err := ioutil.ReadAll(httpResponse.Body) @@ -216,14 +223,18 @@ func (c *Client) sendBatch(ctx context.Context, requests []*jsonRPCBufferedReque if err != nil { return nil, fmt.Errorf("new request: %w", err) } - httpRequest.Header.Set("Content-Type", "text/json") + + httpRequest.Header.Set("Content-Type", "application/json") + if c.username != "" || c.password != "" { httpRequest.SetBasicAuth(c.username, c.password) } + httpResponse, err := c.client.Do(httpRequest) if err != nil { return nil, fmt.Errorf("post: %w", err) } + defer httpResponse.Body.Close() responseBytes, err := ioutil.ReadAll(httpResponse.Body) From e0950515fffa94d835dfe7f72fe863407bf4683c Mon Sep 17 00:00:00 2001 From: Marko Date: Mon, 16 Nov 2020 12:31:32 +0100 Subject: [PATCH 10/27] test/e2e: fix secp failures (#5649) --- abci/example/kvstore/helpers.go | 2 +- abci/example/kvstore/persistent_kvstore.go | 2 +- abci/tests/server/client.go | 2 +- abci/types/pubkey.go | 28 ++++++++++++++++++---- crypto/encoding/codec.go | 2 +- test/e2e/app/app.go | 4 +++- test/e2e/app/config.go | 1 + test/e2e/networks/single.toml | 1 - test/e2e/pkg/testnet.go | 8 ++++--- test/e2e/runner/setup.go | 11 +++++---- test/e2e/tests/evidence_test.go | 2 +- test/e2e/tests/validator_test.go | 6 ++--- 12 files changed, 47 insertions(+), 22 deletions(-) diff --git a/abci/example/kvstore/helpers.go b/abci/example/kvstore/helpers.go index d1334b312..e59fee279 100644 --- a/abci/example/kvstore/helpers.go +++ b/abci/example/kvstore/helpers.go @@ -10,7 +10,7 @@ import ( func RandVal(i int) types.ValidatorUpdate { pubkey := tmrand.Bytes(32) power := tmrand.Uint16() + 1 - v := types.Ed25519ValidatorUpdate(pubkey, int64(power)) + v := types.UpdateValidator(pubkey, int64(power), "") return v } diff --git a/abci/example/kvstore/persistent_kvstore.go b/abci/example/kvstore/persistent_kvstore.go index b584f497d..d40983f85 100644 --- a/abci/example/kvstore/persistent_kvstore.go +++ b/abci/example/kvstore/persistent_kvstore.go @@ -234,7 +234,7 @@ func (app *PersistentKVStoreApplication) execValidatorTx(tx []byte) types.Respon } // update - return app.updateValidator(types.Ed25519ValidatorUpdate(pubkey, power)) + return app.updateValidator(types.UpdateValidator(pubkey, power, "")) } // add, update, or remove a validator diff --git a/abci/tests/server/client.go b/abci/tests/server/client.go index c9d15a98a..1a11a9380 100644 --- a/abci/tests/server/client.go +++ b/abci/tests/server/client.go @@ -16,7 +16,7 @@ func InitChain(client abcicli.Client) error { for i := 0; i < total; i++ { pubkey := tmrand.Bytes(33) power := tmrand.Int() - vals[i] = types.Ed25519ValidatorUpdate(pubkey, int64(power)) + vals[i] = types.UpdateValidator(pubkey, int64(power), "") } _, err := client.InitChainSync(types.RequestInitChain{ Validators: vals, diff --git a/abci/types/pubkey.go b/abci/types/pubkey.go index aaff6debb..8530d9538 100644 --- a/abci/types/pubkey.go +++ b/abci/types/pubkey.go @@ -1,16 +1,16 @@ package types import ( + fmt "fmt" + "github.com/tendermint/tendermint/crypto/ed25519" cryptoenc "github.com/tendermint/tendermint/crypto/encoding" -) - -const ( - PubKeyEd25519 = "ed25519" + "github.com/tendermint/tendermint/crypto/secp256k1" ) func Ed25519ValidatorUpdate(pk []byte, power int64) ValidatorUpdate { pke := ed25519.PubKey(pk) + pkp, err := cryptoenc.PubKeyToProto(pke) if err != nil { panic(err) @@ -22,3 +22,23 @@ func Ed25519ValidatorUpdate(pk []byte, power int64) ValidatorUpdate { Power: power, } } + +func UpdateValidator(pk []byte, power int64, keyType string) ValidatorUpdate { + switch keyType { + case "", ed25519.KeyType: + return Ed25519ValidatorUpdate(pk, power) + case secp256k1.KeyType: + pke := secp256k1.PubKey(pk) + pkp, err := cryptoenc.PubKeyToProto(pke) + if err != nil { + panic(err) + } + return ValidatorUpdate{ + // Address: + PubKey: pkp, + Power: power, + } + default: + panic(fmt.Sprintf("key type %s not supported", keyType)) + } +} diff --git a/crypto/encoding/codec.go b/crypto/encoding/codec.go index f9c721131..3c552ed23 100644 --- a/crypto/encoding/codec.go +++ b/crypto/encoding/codec.go @@ -51,7 +51,7 @@ func PubKeyFromProto(k pc.PublicKey) (crypto.PubKey, error) { return pk, nil case *pc.PublicKey_Secp256K1: if len(k.Secp256K1) != secp256k1.PubKeySize { - return nil, fmt.Errorf("invalid size for PubKeyEd25519. Got %d, expected %d", + return nil, fmt.Errorf("invalid size for PubKeySecp256k1. Got %d, expected %d", len(k.Secp256K1), secp256k1.PubKeySize) } pk := make(secp256k1.PubKey, secp256k1.PubKeySize) diff --git a/test/e2e/app/app.go b/test/e2e/app/app.go index 415fc7ad7..13002a708 100644 --- a/test/e2e/app/app.go +++ b/test/e2e/app/app.go @@ -193,13 +193,15 @@ func (app *Application) validatorUpdates(height uint64) (abci.ValidatorUpdates, if len(updates) == 0 { return nil, nil } + valUpdates := abci.ValidatorUpdates{} for keyString, power := range updates { + keyBytes, err := base64.StdEncoding.DecodeString(keyString) if err != nil { return nil, fmt.Errorf("invalid base64 pubkey value %q: %w", keyString, err) } - valUpdates = append(valUpdates, abci.Ed25519ValidatorUpdate(keyBytes, int64(power))) + valUpdates = append(valUpdates, abci.UpdateValidator(keyBytes, int64(power), app.cfg.KeyType)) } return valUpdates, nil } diff --git a/test/e2e/app/config.go b/test/e2e/app/config.go index 281419160..38c967916 100644 --- a/test/e2e/app/config.go +++ b/test/e2e/app/config.go @@ -22,6 +22,7 @@ type Config struct { PrivValKey string `toml:"privval_key"` PrivValState string `toml:"privval_state"` Misbehaviors map[string]string `toml:"misbehaviors"` + KeyType string `toml:"key_type"` } // LoadConfig loads the configuration from disk. diff --git a/test/e2e/networks/single.toml b/test/e2e/networks/single.toml index b80e54027..54c40b19e 100644 --- a/test/e2e/networks/single.toml +++ b/test/e2e/networks/single.toml @@ -1,2 +1 @@ - [node.validator] diff --git a/test/e2e/pkg/testnet.go b/test/e2e/pkg/testnet.go index 940ddf1cb..df2be5699 100644 --- a/test/e2e/pkg/testnet.go +++ b/test/e2e/pkg/testnet.go @@ -67,7 +67,8 @@ type Node struct { Name string Testnet *Testnet Mode Mode - Key crypto.PrivKey + PrivvalKey crypto.PrivKey + NodeKey crypto.PrivKey IP net.IP ProxyPort uint32 StartAt int64 @@ -142,7 +143,8 @@ func LoadTestnet(file string) (*Testnet, error) { node := &Node{ Name: name, Testnet: testnet, - Key: keyGen.Generate(manifest.KeyType), + PrivvalKey: keyGen.Generate(manifest.KeyType), + NodeKey: keyGen.Generate("ed25519"), IP: ipGen.Next(), ProxyPort: proxyPortGen.Next(), Mode: ModeValidator, @@ -447,7 +449,7 @@ func (n Node) AddressP2P(withID bool) string { } addr := fmt.Sprintf("%v:26656", ip) if withID { - addr = fmt.Sprintf("%x@%v", n.Key.PubKey().Address().Bytes(), addr) + addr = fmt.Sprintf("%x@%v", n.NodeKey.PubKey().Address().Bytes(), addr) } return addr } diff --git a/test/e2e/runner/setup.go b/test/e2e/runner/setup.go index f5b2091d6..d9c3ab0cf 100644 --- a/test/e2e/runner/setup.go +++ b/test/e2e/runner/setup.go @@ -96,12 +96,12 @@ func Setup(testnet *e2e.Testnet) error { return err } - err = (&p2p.NodeKey{PrivKey: node.Key}).SaveAs(filepath.Join(nodeDir, "config", "node_key.json")) + err = (&p2p.NodeKey{PrivKey: node.NodeKey}).SaveAs(filepath.Join(nodeDir, "config", "node_key.json")) if err != nil { return err } - (privval.NewFilePV(node.Key, + (privval.NewFilePV(node.PrivvalKey, filepath.Join(nodeDir, PrivvalKeyFile), filepath.Join(nodeDir, PrivvalStateFile), )).Save() @@ -201,8 +201,8 @@ func MakeGenesis(testnet *e2e.Testnet) (types.GenesisDoc, error) { for validator, power := range testnet.Validators { genesis.Validators = append(genesis.Validators, types.GenesisValidator{ Name: validator.Name, - Address: validator.Key.PubKey().Address(), - PubKey: validator.Key.PubKey(), + Address: validator.PrivvalKey.PubKey().Address(), + PubKey: validator.PrivvalKey.PubKey(), Power: power, }) } @@ -324,6 +324,7 @@ func MakeAppConfig(node *e2e.Node) ([]byte, error) { "persist_interval": node.PersistInterval, "snapshot_interval": node.SnapshotInterval, "retain_blocks": node.RetainBlocks, + "key_type": node.PrivvalKey.Type(), } switch node.ABCIProtocol { case e2e.ProtocolUNIX: @@ -366,7 +367,7 @@ func MakeAppConfig(node *e2e.Node) ([]byte, error) { for height, validators := range node.Testnet.ValidatorUpdates { updateVals := map[string]int64{} for node, power := range validators { - updateVals[base64.StdEncoding.EncodeToString(node.Key.PubKey().Bytes())] = power + updateVals[base64.StdEncoding.EncodeToString(node.PrivvalKey.PubKey().Bytes())] = power } validatorUpdates[fmt.Sprintf("%v", height)] = updateVals } diff --git a/test/e2e/tests/evidence_test.go b/test/e2e/tests/evidence_test.go index 8abb361c8..ea24b51e5 100644 --- a/test/e2e/tests/evidence_test.go +++ b/test/e2e/tests/evidence_test.go @@ -22,7 +22,7 @@ func TestEvidence_Misbehavior(t *testing.T) { for _, evidence := range block.Evidence.Evidence { switch evidence := evidence.(type) { case *types.DuplicateVoteEvidence: - if bytes.Equal(evidence.VoteA.ValidatorAddress, node.Key.PubKey().Address()) { + if bytes.Equal(evidence.VoteA.ValidatorAddress, node.PrivvalKey.PubKey().Address()) { nodeEvidence = evidence } default: diff --git a/test/e2e/tests/validator_test.go b/test/e2e/tests/validator_test.go index 29f63bd92..8a36bb55d 100644 --- a/test/e2e/tests/validator_test.go +++ b/test/e2e/tests/validator_test.go @@ -60,7 +60,7 @@ func TestValidator_Propose(t *testing.T) { if node.Mode != e2e.ModeValidator { return } - address := node.Key.PubKey().Address() + address := node.PrivvalKey.PubKey().Address() valSchedule := newValidatorSchedule(*node.Testnet) expectCount := 0 @@ -90,7 +90,7 @@ func TestValidator_Sign(t *testing.T) { if node.Mode != e2e.ModeValidator { return } - address := node.Key.PubKey().Address() + address := node.PrivvalKey.PubKey().Address() valSchedule := newValidatorSchedule(*node.Testnet) expectCount := 0 @@ -160,7 +160,7 @@ func (s *validatorSchedule) Increment(heights int64) { func makeVals(valMap map[*e2e.Node]int64) []*types.Validator { vals := make([]*types.Validator, 0, len(valMap)) for node, power := range valMap { - vals = append(vals, types.NewValidator(node.Key.PubKey(), power)) + vals = append(vals, types.NewValidator(node.PrivvalKey.PubKey(), power)) } return vals } From a736530e012c168b60cf4a0f6c6a0bc7036a00f0 Mon Sep 17 00:00:00 2001 From: Tess Rinearson Date: Mon, 16 Nov 2020 13:54:52 +0100 Subject: [PATCH 11/27] docs/architecture: add missing ADRs to README, update status of ADR 034 (#5663) --- docs/architecture/README.md | 13 ++++++++----- .../adr-034-priv-validator-file-structure.md | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/architecture/README.md b/docs/architecture/README.md index a0b95d880..ee3f5fd1f 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -63,7 +63,7 @@ Note the context/background should be written in the present tense. - [ADR-034-Priv-Validator-File-Structure](./adr-034-priv-validator-file-structure.md) - [ADR-035-Documentation](./adr-035-documentation.md) - [ADR-037-Deliver-Block](./adr-037-deliver-block.md) -- [ADR-038-non-zero-start-height](./adr-038-non-zero-start-height.md) +- [ADR-038-Non-Zero-Start-Height](./adr-038-non-zero-start-height.md) - [ADR-039-Peer-Behaviour](./adr-039-peer-behaviour.md) - [ADR-041-Proposer-Selection-via-ABCI](./adr-041-proposer-selection-via-abci.md) - [ADR-043-Blockchain-RiRi-Org](./adr-043-blockchain-riri-org.md) @@ -74,8 +74,11 @@ Note the context/background should be written in the present tense. - [ADR-051-Double-Signing-Risk-Reduction](./adr-051-double-signing-risk-reduction.md) - [ADR-052-Tendermint-Mode](./adr-052-tendermint-mode.md) - [ADR-053-State-Sync-Prototype](./adr-053-state-sync-prototype.md) -- [ADR-054-crypto-encoding-2](./adr-054-crypto-encoding-2.md) -- [ADR-055-protobuf-design](./adr-055-protobuf-design.md) -- [ADR-056-light-client-amnesia-attacks](./adr-056-light-client-amnesia-attacks) +- [ADR-054-Crypto-Encoding-2](./adr-054-crypto-encoding-2.md) +- [ADR-055-Protobuf-Design](./adr-055-protobuf-design.md) +- [ADR-056-Light-Client-Amnesia-Attacks](./adr-056-light-client-amnesia-attacks) - [ADR-057-RPC](./adr-057-RPC.md) -- [ADR-058-event-hashing](./adr-058-event-hashing.md) +- [ADR-058-Event-Hashing](./adr-058-event-hashing.md) +- [ADR-059-Evidence-Composition-and-Lifecycle](./adr-059-evidence-composition-and-lifecycle.md) +- [ADR-060-Go-API-Stability](./adr-060-go-api-stability.md) +- [ADR-061-P2P-Refactor-Scope](./adr-061-p2p-refactor-scope.md) diff --git a/docs/architecture/adr-034-priv-validator-file-structure.md b/docs/architecture/adr-034-priv-validator-file-structure.md index 83160bfb8..8eb7464b4 100644 --- a/docs/architecture/adr-034-priv-validator-file-structure.md +++ b/docs/architecture/adr-034-priv-validator-file-structure.md @@ -57,7 +57,7 @@ What we need to do next is changing the methods of `FilePV`. ## Status -Draft. +Accepted and implemented in [#2870](https://github.com/tendermint/tendermint/pull/2870). ## Consequences From deb4f6061379b25927686b7123b67a3e43b03acb Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Mon, 16 Nov 2020 17:17:09 +0100 Subject: [PATCH 12/27] docs: add P2P architecture ADR (#5637) [Rendered](https://github.com/tendermint/tendermint/blob/erik/adr-p2p-architecture/docs/architecture/adr-062-p2p-architecture.md) ADR for the new P2P architecture and abstractions. Related to #2067. --- docs/architecture/README.md | 1 + docs/architecture/adr-062-p2p-architecture.md | 531 ++++++++++++++++++ .../architecture/img/adr-062-architecture.svg | 1 + 3 files changed, 533 insertions(+) create mode 100644 docs/architecture/adr-062-p2p-architecture.md create mode 100644 docs/architecture/img/adr-062-architecture.svg diff --git a/docs/architecture/README.md b/docs/architecture/README.md index ee3f5fd1f..68145d6e1 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -82,3 +82,4 @@ Note the context/background should be written in the present tense. - [ADR-059-Evidence-Composition-and-Lifecycle](./adr-059-evidence-composition-and-lifecycle.md) - [ADR-060-Go-API-Stability](./adr-060-go-api-stability.md) - [ADR-061-P2P-Refactor-Scope](./adr-061-p2p-refactor-scope.md) +- [ADR-062-P2P-Architecture](./adr-062-p2p-architecture.md) diff --git a/docs/architecture/adr-062-p2p-architecture.md b/docs/architecture/adr-062-p2p-architecture.md new file mode 100644 index 000000000..5fae7301d --- /dev/null +++ b/docs/architecture/adr-062-p2p-architecture.md @@ -0,0 +1,531 @@ +# ADR 062: P2P Architecture and Abstractions + +## Changelog + +- 2020-11-09: Initial version (@erikgrinaker) + +- 2020-11-13: Remove stream IDs, move peer errors onto channel, note on moving PEX into core (@erikgrinaker) + +- 2020-11-16: Notes on recommended reactor implementation patterns, approve ADR (@erikgrinaker) + +## Context + +In [ADR 061](adr-061-p2p-refactor-scope.md) we decided to refactor the peer-to-peer (P2P) networking stack. The first phase is to redesign and refactor the internal P2P architecture, while retaining protocol compatibility as far as possible. + +## Alternative Approaches + +Several variations of the proposed design were considered, including e.g. calling interface methods instead of passing messages (like the current architecture), merging channels with streams, exposing the internal peer data structure to reactors, being message format-agnostic via arbitrary codecs, and so on. This design was chosen because it has very loose coupling, is simpler to reason about and more convenient to use, avoids race conditions and lock contention for internal data structures, gives reactors better control of message ordering and processing semantics, and allows for QoS scheduling and backpressure in a very natural way. + +[multiaddr](https://github.com/multiformats/multiaddr) was considered as a transport-agnostic peer address format over regular URLs, but it does not appear to have very widespread adoption, and advanced features like protocol encapsulation and tunneling do not appear to be immediately useful to us. + +There were also proposals to use LibP2P instead of maintaining our own P2P stack, which were rejected (for now) in [ADR 061](adr-061-p2p-refactor-scope.md). + +## Decision + +The P2P stack will be redesigned as a message-oriented architecture, primarily relying on Go channels for communication and scheduling. It will use IO stream transports to exchange raw bytes with individual peers, bidirectional peer-addressable channels to send and receive Protobuf messages, and a router to route messages between reactors and peers. Message passing is asynchronous with at-most-once delivery. + +## Detailed Design + +This ADR is primarily concerned with the architecture and interfaces of the P2P stack, not implementation details. Separate ADRs may be submitted for individual components, since implementation may be non-trivial. The interfaces described here should therefore be considered a rough architecture outline, not a complete and final design. + +Primary design objectives have been: + +* Loose coupling between components, for a simpler, more robust, and test-friendly architecture. +* Pluggable transports (not necessarily networked). +* Better scheduling of messages, with improved prioritization, backpressure, and performance. +* Centralized peer lifecycle and connection management. +* Better peer address detection, advertisement, and exchange. +* Wire-level backwards compatibility with current P2P network protocols, except where it proves too obstructive. + +The main abstractions in the new stack are: + +* `peer`: A node in the network, uniquely identified by a `PeerID` and stored in a `peerStore`. +* `Transport`: An arbitrary mechanism to exchange bytes with a peer using IO `Stream`s across a `Connection`. +* `Channel`: A bidirectional channel to asynchronously exchange Protobuf messages with peers addressed with `PeerID`. +* `Router`: Maintains transport connections to relevant peers and routes channel messages. +* Reactor: A design pattern loosely defined as "something which listens on a channel and reacts to messages". + +These abstractions are illustrated in the following diagram (representing the internals of node A) and described in detail below. + +![P2P Architecture Diagram](img/adr-062-architecture.svg) + +### Transports + +Transports are arbitrary mechanisms for exchanging raw bytes with a peer. For example, a gRPC transport would connect to a peer over TCP/IP and send data using the gRPC protocol, while an in-memory transport might communicate with a peer running in another goroutine using internal byte buffers. Note that transports don't have a notion of a `peer` as such - instead, they communicate with an arbitrary endpoint address (e.g. IP address and port number), to decouple them from the rest of the P2P stack. + +Transports must satisfy the following requirements: + +* Be connection-oriented, and support both listening for inbound connections and making outbound connections using endpoint addresses. + +* Support multiple logical IO streams within a single connection, to take full advantage of protocols with native stream support. For example, QUIC supports multiple independent streams, while HTTP/2 and MConn multiplex logical streams onto a single TCP connection. + +* Provide the public key of the peer, and possibly encrypt or sign the traffic as appropriate. This should be compared with known data (e.g. the peer ID) to authenticate the peer and avoid man-in-the-middle attacks. + +The initial transport implementation will be a port of the current MConn protocol currently used by Tendermint, and should be backwards-compatible at the wire level as far as possible. This will be followed by an in-memory transport for testing, and a QUIC transport that may eventually replace MConn. + +The `Transport` interface is: + +```go +// Transport is an arbitrary mechanism for exchanging bytes with a peer. +type Transport interface { + // Accept waits for the next inbound connection on a listening endpoint. + Accept(context.Context) (Connection, error) + + // Dial creates an outbound connection to an endpoint. + Dial(context.Context, Endpoint) (Connection, error) + + // Endpoints lists endpoints the transport is listening on. Any endpoint IP + // addresses do not need to be normalized in any way (e.g. 0.0.0.0 is + // valid), as they should be preprocessed before being advertised. + Endpoints() []Endpoint +} +``` + +How the transport configures listening is transport-dependent, and not covered by the interface. This typically happens during transport construction, where a single instance of the transport is created and set to listen on an appropriate network interface before being passed to the router. + +#### Endpoints + +`Endpoint` represents a transport endpoint (e.g. an IP address and port). A connection always has two endpoints: one at the local node and one at the remote peer. Outbound connections to remote endpoints are made via `Dial()`, and inbound connections to listening endpoints are returned via `Accept()`. + +The `Endpoint` struct is: + +```go +// Endpoint represents a transport connection endpoint, either local or remote. +type Endpoint struct { + // Protocol specifies the transport protocol, used by the router to pick a + // transport for an endpoint. + Protocol Protocol + + // Path is an optional, arbitrary transport-specific path or identifier. + Path string + + // IP is an IP address (v4 or v6) to connect to. If set, this defines the + // endpoint as a networked endpoint. + IP net.IP + + // Port is a network port (either TCP or UDP). If not set, a default port + // may be used depending on the protocol. + Port uint16 +} + +// Protocol identifies a transport protocol. +type Protocol string +``` + +Endpoints are arbitrary transport-specific addresses, but if they are networked they must use IP addresses and thus rely on IP as a fundamental packet routing protocol. This enables policies for address discovery, advertisement, and exchange - for example, a private `192.168.0.0/24` IP address should only be advertised to peers on that IP network, while the public address `8.8.8.8` may be advertised to all peers. Similarly, any port numbers if given must represent TCP and/or UDP port numbers, in order to use [UPnP](https://en.wikipedia.org/wiki/Universal_Plug_and_Play) to autoconfigure e.g. NAT gateways. + +Non-networked endpoints (without an IP address) are considered local, and will only be advertised to other peers connecting via the same protocol. For example, an in-memory transport used for testing might have `Endpoint{Protocol: "memory", Path: "foo"}` as an address for the node "foo", and this should only be advertised to other nodes using `Protocol: "memory"`. + +#### Connections and Streams + +A connection represents an established transport connection between two endpoints (and thus two nodes), which can be used to exchange bytes via logically distinct IO streams. Connections are set up either via `Transport.Dial()` (outbound) or `Transport.Accept()` (inbound). The caller is responsible for verifying the remote peer's public key as returned by the connection, following the current MConn protocol behavior for now. + +Data is exchanged over IO streams created with `Connection.Stream()`. These implement the standard Go `io.Reader` and `io.Writer` interfaces to read and write bytes. Transports are free to choose how to implement such streams, e.g. by taking advantage of native stream support in the underlying protocol or through multiplexing. + +`Connection` and the related `Stream` interfaces are: + +```go +// Connection represents an established connection between two endpoints. +type Connection interface { + // Stream creates a new logically distinct IO stream within the connection. + Stream() (Stream, error) + + // LocalEndpoint returns the local endpoint for the connection. + LocalEndpoint() Endpoint + + // RemoteEndpoint returns the remote endpoint for the connection. + RemoteEndpoint() Endpoint + + // PubKey returns the public key of the remote peer. + PubKey() crypto.PubKey + + // Close closes the connection. + Close() error +} + +// Stream represents a single logical IO stream within a connection. +type Stream interface { + io.Reader // Read([]byte) (int, error) + io.Writer // Write([]byte) (int, error) + io.Closer // Close() error +} +``` + +### Peers + +Peers are other Tendermint network nodes. Each peer is identified by a unique `PeerID`, and has a set of `PeerAddress` addresses expressed as URLs that they can be reached at. Examples of peer addresses might be e.g.: + +* `mconn://b10c@host.domain.com:25567/path` +* `unix:///var/run/tendermint/peer.sock` +* `memory:testpeer` + +Addresses are resolved into one or more transport endpoints, e.g. by resolving DNS hostnames into IP addresses (which should be refreshed periodically). Peers should always be expressed as address URLs, and never as endpoints which are a lower-level construct. + +```go +// PeerID is a unique peer ID, generally expressed in hex form. +type PeerID []byte + +// PeerAddress is a peer address URL. The User field, if set, gives the +// hex-encoded remote PeerID, which should be verified with the remote peer's +// public key as returned by the connection. +type PeerAddress url.URL + +// Resolve resolves a PeerAddress into a set of Endpoints, typically by +// expanding out a DNS name in Host to its IP addresses. Field mapping: +// +// Scheme → Endpoint.Protocol +// Host → Endpoint.IP +// Port → Endpoint.Port +// Path+Query+Fragment,Opaque → Endpoint.Path +// +func (a PeerAddress) Resolve(ctx context.Context) []Endpoint { return nil } +``` + +The P2P stack needs to track a lot of internal information about peers, such as endpoints, status, priorities, and so on. This is done in an internal `peer` struct, which should not be exposed outside of the `p2p` package (e.g. to reactors) in order to avoid race conditions and lock contention - other packages should use `PeerID`. + +The `peer` struct might look like the following, but is intentionally underspecified and will depend on implementation requirements (for example, it will almost certainly have to track statistics about connection failures and retries): + +```go +// peer tracks internal status information about a peer. +type peer struct { + ID PeerID + Status PeerStatus + Priority PeerPriority + Endpoints map[PeerAddress][]Endpoint // Resolved endpoints by address. +} + +// PeerStatus specifies peer statuses. +type PeerStatus string + +const ( + PeerStatusNew = "new" // New peer which we haven't tried to contact yet. + PeerStatusUp = "up" // Peer which we have an active connection to. + PeerStatusDown = "down" // Peer which we're temporarily disconnected from. + PeerStatusRemoved = "removed" // Peer which has been removed. + PeerStatusBanned = "banned" // Peer which is banned for misbehavior. +) + +// PeerPriority specifies peer priorities. +type PeerPriority int + +const ( + PeerPriorityNormal PeerPriority = iota + 1 + PeerPriorityValidator + PeerPriorityPersistent +) +``` + +Peer information is stored in a `peerStore`, which may be persisted in an underlying database, and will replace the current address book either partially or in full. It is kept internal to avoid race conditions and tight coupling, and should at the very least contain basic CRUD functionality as outlined below, but will likely need additional functionality and is intentionally underspecified: + +```go +// peerStore contains information about peers, possibly persisted to disk. +type peerStore struct { + peers map[string]*peer // Entire set in memory, with PeerID.String() keys. + db dbm.DB // Database for persistence, if non-nil. +} + +func (p *peerStore) Delete(id PeerID) error { return nil } +func (p *peerStore) Get(id PeerID) (peer, bool) { return peer{}, false } +func (p *peerStore) List() []peer { return nil } +func (p *peerStore) Set(peer peer) error { return nil } +``` + +Peer address detection, advertisement and exchange (including detection of externally-reachable addresses via e.g. NAT gateways) is out of scope for this ADR, but may be covered in a separate ADR. The current PEX reactor should probably be absorbed into the core P2P stack and protocol instead of running as a separate reactor, since this needs to mutate the core peer data structures and will thus be tightly coupled with the router. + +### Channels + +While low-level data exchange happens via transport IO streams, the high-level API is based on a bidirectional `Channel` that can send and receive Protobuf messages addressed by `PeerID`. A channel is identified by an arbitrary `ChannelID` identifier, and can exchange Protobuf messages of one specific type (since the type to unmarshal into must be known). Message delivery is asynchronous and at-most-once. + +The channel can also be used to report peer errors, e.g. when receiving an invalid or malignant message. This may cause the peer to be disconnected or banned depending on the router's policy. + +A `Channel` has this interface: + +```go +// Channel is a bidirectional channel for Protobuf message exchange with peers. +type Channel struct { + // ID contains the channel ID. + ID ChannelID + + // messageType specifies the type of messages exchanged via the channel, and + // is used e.g. for automatic unmarshaling. + messageType proto.Message + + // In is a channel for receiving inbound messages. Envelope.From is always + // set. + In <-chan Envelope + + // Out is a channel for sending outbound messages. Envelope.To or Broadcast + // must be set, otherwise the message is discarded. + Out chan<- Envelope + + // Error is a channel for reporting peer errors to the router, typically used + // when peers send an invalid or malignant message. + Error chan<- PeerError +} + +// Close closes the channel, and is equivalent to close(Channel.Out). This will +// cause Channel.In to be closed when appropriate. The ID can then be reused. +func (c *Channel) Close() error { return nil } + +// ChannelID is an arbitrary channel ID. +type ChannelID uint16 + +// Envelope specifies the message receiver and sender. +type Envelope struct { + From PeerID // Message sender, or empty for outbound messages. + To PeerID // Message receiver, or empty for inbound messages. + Broadcast bool // Send message to all connected peers, ignoring To. + Message proto.Message // Payload. +} + +// PeerError is a peer error reported by a reactor via the Error channel. The +// severity may cause the peer to be disconnected or banned depending on policy. +type PeerError struct { + PeerID PeerID + Err error + Severity PeerErrorSeverity +} + +// PeerErrorSeverity determines the severity of a peer error. +type PeerErrorSeverity string + +const ( + PeerErrorSeverityLow PeerErrorSeverity = "low" // Mostly ignored. + PeerErrorSeverityHigh PeerErrorSeverity = "high" // May disconnect. + PeerErrorSeverityCritical PeerErrorSeverity = "critical" // Ban. +) +``` + +A channel can reach any connected peer, and is implemented using transport streams against each individual peer, with an initial handshake to exchange the channel ID and any other metadata. The channel will automatically (un)marshal Protobuf to byte slices and use length-prefixed framing (the de facto standard for Protobuf streams) when writing them to the stream. + +Message scheduling and queueing is left as an implementation detail, and can use any number of algorithms such as FIFO, round-robin, priority queues, etc. Since message delivery is not guaranteed, both inbound and outbound messages may be dropped, buffered, or blocked as appropriate. + +Since a channel can only exchange messages of a single type, it is often useful to use a wrapper message type with e.g. a Protobuf `oneof` field that specifies a set of inner message types that it can contain. The channel can automatically perform this (un)wrapping if the outer message type implements the `Wrapper` interface (see [Reactor Example](#reactor-example) for an example): + +```go +// Wrapper is a Protobuf message that can contain a variety of inner messages. +// If a Channel's message type implements Wrapper, the channel will +// automatically (un)wrap passed messages using the container type, such that +// the channel can transparently support multiple message types. +type Wrapper interface { + // Wrap will take a message and wrap it in this one. + Wrap(proto.Message) error + + // Unwrap will unwrap the inner message contained in this message. + Unwrap() (proto.Message, error) +} +``` + +### Routers + +The router manages all P2P networking for a node, and is responsible for keeping track of network peers, maintaining transport connections, and routing channel messages. As such, it must do e.g. connection retries and backoff, message QoS scheduling and backpressure, peer quality assessments, and endpoint detection and advertisement. In addition, the router provides a mechanism to subscribe to peer updates (e.g. peers connecting or disconnecting), and handles reported peer errors from reactors. + +The implementation of the router is likely to be non-trivial, and is intentionally unspecified here. A separate ADR will likely be submitted for this. It is unclear whether message routing/scheduling and peer lifecycle management can be split into two separate components, or if these need to be tightly coupled. + +The `Router` API is as follows: + +```go +// Router manages connections to peers and routes Protobuf messages between them +// and local reactors. It also provides peer status updates and error reporting. +type Router struct{} + +// NewRouter creates a new router, using the given peer store to track peers. +// Transports must be pre-initialized to listen on appropriate endpoints. +func NewRouter(peerStore *peerStore, transports map[Protocol]Transport) *Router { return nil } + +// Channel opens a new channel with the given ID. messageType should be an empty +// Protobuf message of the type that will be passed through the channel. The +// message can implement Wrapper for automatic message (un)wrapping. +func (r *Router) Channel(id ChannelID, messageType proto.Message) (*Channel, error) { return nil, nil } + +// PeerUpdates returns a channel with peer updates. The caller must cancel the +// context to end the subscription, and keep consuming messages in a timely +// fashion until the channel is closed to avoid blocking updates. +func (r *Router) PeerUpdates(ctx context.Context) PeerUpdates { return nil } + +// PeerUpdates is a channel for receiving peer updates. +type PeerUpdates <-chan PeerUpdate + +// PeerUpdate is a peer status update for reactors. +type PeerUpdate struct { + PeerID PeerID + Status PeerStatus +} +``` + +### Reactor Example + +While reactors are a first-class concept in the current P2P stack (i.e. there is an explicit `p2p.Reactor` interface), they will simply be a design pattern in the new stack, loosely defined as "something which listens on a channel and reacts to messages". + +Since reactors have very few formal constraints, they can be implemented in a variety of ways. There is currently no recommended pattern for implementing reactors, to avoid overspecification and scope creep in this ADR. However, prototyping and developing a reactor pattern should be done early during implementation, to make sure reactors built using the `Channel` interface can satisfy the needs for convenience, deterministic tests, and reliability. + +Below is a trivial example of a simple echo reactor implemented as a function. The reactor will exchange the following Protobuf messages: + +```protobuf +message EchoMessage { + oneof inner { + PingMessage ping = 1; + PongMessage pong = 2; + } +} + +message PingMessage { + string content = 1; +} + +message PongMessage { + string content = 1; +} +``` + +Implementing the `Wrapper` interface for `EchoMessage` allows transparently passing `PingMessage` and `PongMessage` through the channel, where it will automatically be (un)wrapped in an `EchoMessage`: + +```go +func (m *EchoMessage) Wrap(inner proto.Message) error { + switch inner := inner.(type) { + case *PingMessage: + m.Inner = &EchoMessage_PingMessage{Ping: inner} + case *PongMessage: + m.Inner = &EchoMessage_PongMessage{Pong: inner} + default: + return fmt.Errorf("unknown message %T", inner) + } + return nil +} + +func (m *EchoMessage) Unwrap() (proto.Message, error) { + switch inner := m.Inner.(type) { + case *EchoMessage_PingMessage: + return inner.Ping, nil + case *EchoMessage_PongMessage: + return inner.Pong, nil + default: + return nil, fmt.Errorf("unknown message %T", inner) + } +} +``` + +The reactor itself would be implemented e.g. like this: + +```go +// RunEchoReactor wires up an echo reactor to a router and runs it. +func RunEchoReactor(router *p2p.Router) error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + channel, err := router.Channel(1, &EchoMessage{}) + if err != nil { + return err + } + defer channel.Close() + + return EchoReactor(ctx, channel, router.PeerUpdates(ctx)) +} + +// EchoReactor provides an echo service, pinging all known peers until cancelled. +func EchoReactor(ctx context.Context, channel *p2p.Channel, peerUpdates p2p.PeerUpdates) error { + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + + for { + select { + // Send ping message to all known peers every 5 seconds. + case <-ticker.C: + channel.Out <- Envelope{ + Broadcast: true, + Message: &PingMessage{Content: "👋"}, + } + + // When we receive a message from a peer, either respond to ping, output + // pong, or report peer error on unknown message type. + case envelope := <-channel.In: + switch msg := envelope.Message.(type) { + case *PingMessage: + channel.Out <- Envelope{ + To: envelope.From, + Message: &PongMessage{Content: msg.Content}, + } + + case *PongMessage: + fmt.Printf("%q replied with %q\n", envelope.From, msg.Content) + + default: + channel.Error <- PeerError{ + PeerID: envelope.From, + Err: fmt.Errorf("unexpected message %T", msg), + Severity: PeerErrorSeverityLow, + } + } + + // Output info about any peer status changes. + case peerUpdate := <-peerUpdates: + fmt.Printf("Peer %q changed status to %q", peerUpdate.PeerID, peerUpdate.Status) + + // Exit when context is cancelled. + case <-ctx.Done(): + return nil + } + } +} +``` + +### Implementation Plan + +The existing P2P stack should be gradually migrated towards this design. The easiest path would likely be: + +1. Implement the `Channel` and `PeerUpdates` APIs as shims on top of the current `Switch` and `Peer` APIs, and rewrite all reactors to use them instead. + +2. Port the `privval` package to no longer use `SecretConnection` (e.g. by using gRPC instead), or temporarily duplicate its functionality. + +3. Rewrite the current MConn connection and transport code to use the new `Transport` API, and migrate existing code to use it instead. + +4. Implement the new `peer` and `peerStore` APIs, and either make the current address book a shim on top of these or replace it. + +5. Replace the existing `Switch` abstraction with the new `Router`. + +6. Move the PEX reactor and other address advertisement/exchange into the P2P core, possibly the `Router`. + +7. Consider rewriting and/or cleaning up reactors and other P2P-related code to make better use of the new abstractions. + +A note on backwards-compatibility: the current MConn protocol takes whole messages expressed as byte slices and splits them up into `PacketMsg` messages, where the final packet of a message has `PacketMsg.EOF` set. In order to maintain wire-compatibility with this protocol, the MConn transport needs to be aware of message boundaries, even though it does not care what the messages actually are. One way to handle this is to break abstraction boundaries and have the transport decode the input's length-prefixed message framing and use this to determine message boundaries, unless we accept breaking the protocol here. + +Similarly, implementing channel handshakes with the current MConn protocol would require doing an initial connection handshake as today and use that information to "fake" the local channel handshake without it hitting the wire. + +## Status + +Accepted + +## Consequences + +### Positive + +* Reduced coupling and simplified interfaces should lead to better understandability, increased reliability, and more testing. + +* Using message passing via Go channels gives better control of backpressure and quality-of-service scheduling. + +* Peer lifecycle and connection management is centralized in a single entity, making it easier to reason about. + +* Detection, advertisement, and exchange of node addresses will be improved. + +* Additional transports (e.g. QUIC) can be implemented and used in parallel with the existing MConn protocol. + +* The P2P protocol will not be broken in the initial version, if possible. + +### Negative + +* Fully implementing the new design as indended is likely to require breaking changes to the P2P protocol at some point, although the initial implementation shouldn't. + +* Gradually migrating the existing stack and maintaining backwards-compatibility will be more labor-intensive than simply replacing the entire stack. + +* A complete overhaul of P2P internals is likely to cause temporary performance regressions and bugs as the implementation matures. + +* Hiding peer management information inside the `p2p` package may prevent certain functionality or require additional deliberate interfaces for information exchange, as a tradeoff to simplify the design, reduce coupling, and avoid race conditions and lock contention. + +### Neutral + +* Implementation details around e.g. peer management, message scheduling, and peer and endpoint advertisement are not yet determined. + +## References + +* [ADR 061: P2P Refactor Scope](adr-061-p2p-refactor-scope.md) +* [#5670 p2p: internal refactor and architecture redesign](https://github.com/tendermint/tendermint/issues/5670) diff --git a/docs/architecture/img/adr-062-architecture.svg b/docs/architecture/img/adr-062-architecture.svg new file mode 100644 index 000000000..1ad18a3e0 --- /dev/null +++ b/docs/architecture/img/adr-062-architecture.svg @@ -0,0 +1 @@ + \ No newline at end of file From 2d0fcf498dca9c08a60214258c48b66abb2a49dc Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Mon, 16 Nov 2020 19:16:10 +0100 Subject: [PATCH 13/27] privval: duplicate SecretConnection from p2p package (#5672) This is so that the `privval` package will not be affected when we refactor `p2p` (#5670). We will be migrating to gRPC shortly (#4698). --- privval/secret_connection.go | 469 +++++++++++++++++++++++++++ privval/socket_dialers.go | 3 +- privval/socket_listeners.go | 3 +- proto/tendermint/privval/types.pb.go | 328 ++++++++++++++++--- proto/tendermint/privval/types.proto | 8 + 5 files changed, 759 insertions(+), 52 deletions(-) create mode 100644 privval/secret_connection.go diff --git a/privval/secret_connection.go b/privval/secret_connection.go new file mode 100644 index 000000000..10f7653d7 --- /dev/null +++ b/privval/secret_connection.go @@ -0,0 +1,469 @@ +package privval + +import ( + "bytes" + "crypto/cipher" + crand "crypto/rand" + "crypto/sha256" + "encoding/binary" + "errors" + "fmt" + "io" + "math" + "net" + "time" + + gogotypes "github.com/gogo/protobuf/types" + "github.com/gtank/merlin" + pool "github.com/libp2p/go-buffer-pool" + "golang.org/x/crypto/chacha20poly1305" + "golang.org/x/crypto/curve25519" + "golang.org/x/crypto/hkdf" + "golang.org/x/crypto/nacl/box" + + "github.com/tendermint/tendermint/crypto" + "github.com/tendermint/tendermint/crypto/ed25519" + cryptoenc "github.com/tendermint/tendermint/crypto/encoding" + "github.com/tendermint/tendermint/libs/async" + "github.com/tendermint/tendermint/libs/protoio" + tmsync "github.com/tendermint/tendermint/libs/sync" + tmprivval "github.com/tendermint/tendermint/proto/tendermint/privval" +) + +// This code has been duplicated from p2p/conn prior to the P2P refactor. +// It is left here temporarily until we migrate privval to gRPC. +// https://github.com/tendermint/tendermint/issues/4698 + +// 4 + 1024 == 1028 total frame size +const ( + dataLenSize = 4 + dataMaxSize = 1024 + totalFrameSize = dataMaxSize + dataLenSize + aeadSizeOverhead = 16 // overhead of poly 1305 authentication tag + aeadKeySize = chacha20poly1305.KeySize + aeadNonceSize = chacha20poly1305.NonceSize +) + +var ( + ErrSmallOrderRemotePubKey = errors.New("detected low order point from remote peer") + + labelEphemeralLowerPublicKey = []byte("EPHEMERAL_LOWER_PUBLIC_KEY") + labelEphemeralUpperPublicKey = []byte("EPHEMERAL_UPPER_PUBLIC_KEY") + labelDHSecret = []byte("DH_SECRET") + labelSecretConnectionMac = []byte("SECRET_CONNECTION_MAC") + + secretConnKeyAndChallengeGen = []byte("TENDERMINT_SECRET_CONNECTION_KEY_AND_CHALLENGE_GEN") +) + +// SecretConnection implements net.Conn. +// It is an implementation of the STS protocol. +// See https://github.com/tendermint/tendermint/blob/0.1/docs/sts-final.pdf for +// details on the protocol. +// +// Consumers of the SecretConnection are responsible for authenticating +// the remote peer's pubkey against known information, like a nodeID. +// Otherwise they are vulnerable to MITM. +// (TODO(ismail): see also https://github.com/tendermint/tendermint/issues/3010) +type SecretConnection struct { + + // immutable + recvAead cipher.AEAD + sendAead cipher.AEAD + + remPubKey crypto.PubKey + conn io.ReadWriteCloser + + // net.Conn must be thread safe: + // https://golang.org/pkg/net/#Conn. + // Since we have internal mutable state, + // we need mtxs. But recv and send states + // are independent, so we can use two mtxs. + // All .Read are covered by recvMtx, + // all .Write are covered by sendMtx. + recvMtx tmsync.Mutex + recvBuffer []byte + recvNonce *[aeadNonceSize]byte + + sendMtx tmsync.Mutex + sendNonce *[aeadNonceSize]byte +} + +// MakeSecretConnection performs handshake and returns a new authenticated +// SecretConnection. +// Returns nil if there is an error in handshake. +// Caller should call conn.Close() +// See docs/sts-final.pdf for more information. +func MakeSecretConnection(conn io.ReadWriteCloser, locPrivKey crypto.PrivKey) (*SecretConnection, error) { + var ( + locPubKey = locPrivKey.PubKey() + ) + + // Generate ephemeral keys for perfect forward secrecy. + locEphPub, locEphPriv := genEphKeys() + + // Write local ephemeral pubkey and receive one too. + // NOTE: every 32-byte string is accepted as a Curve25519 public key (see + // DJB's Curve25519 paper: http://cr.yp.to/ecdh/curve25519-20060209.pdf) + remEphPub, err := shareEphPubKey(conn, locEphPub) + if err != nil { + return nil, err + } + + // Sort by lexical order. + loEphPub, hiEphPub := sort32(locEphPub, remEphPub) + + transcript := merlin.NewTranscript("TENDERMINT_SECRET_CONNECTION_TRANSCRIPT_HASH") + + transcript.AppendMessage(labelEphemeralLowerPublicKey, loEphPub[:]) + transcript.AppendMessage(labelEphemeralUpperPublicKey, hiEphPub[:]) + + // Check if the local ephemeral public key was the least, lexicographically + // sorted. + locIsLeast := bytes.Equal(locEphPub[:], loEphPub[:]) + + // Compute common diffie hellman secret using X25519. + dhSecret, err := computeDHSecret(remEphPub, locEphPriv) + if err != nil { + return nil, err + } + + transcript.AppendMessage(labelDHSecret, dhSecret[:]) + + // Generate the secret used for receiving, sending, challenge via HKDF-SHA2 + // on the transcript state (which itself also uses HKDF-SHA2 to derive a key + // from the dhSecret). + recvSecret, sendSecret := deriveSecrets(dhSecret, locIsLeast) + + const challengeSize = 32 + var challenge [challengeSize]byte + challengeSlice := transcript.ExtractBytes(labelSecretConnectionMac, challengeSize) + + copy(challenge[:], challengeSlice[0:challengeSize]) + + sendAead, err := chacha20poly1305.New(sendSecret[:]) + if err != nil { + return nil, errors.New("invalid send SecretConnection Key") + } + recvAead, err := chacha20poly1305.New(recvSecret[:]) + if err != nil { + return nil, errors.New("invalid receive SecretConnection Key") + } + + sc := &SecretConnection{ + conn: conn, + recvBuffer: nil, + recvNonce: new([aeadNonceSize]byte), + sendNonce: new([aeadNonceSize]byte), + recvAead: recvAead, + sendAead: sendAead, + } + + // Sign the challenge bytes for authentication. + locSignature, err := signChallenge(&challenge, locPrivKey) + if err != nil { + return nil, err + } + + // Share (in secret) each other's pubkey & challenge signature + authSigMsg, err := shareAuthSignature(sc, locPubKey, locSignature) + if err != nil { + return nil, err + } + + remPubKey, remSignature := authSigMsg.Key, authSigMsg.Sig + if _, ok := remPubKey.(ed25519.PubKey); !ok { + return nil, fmt.Errorf("expected ed25519 pubkey, got %T", remPubKey) + } + if !remPubKey.VerifySignature(challenge[:], remSignature) { + return nil, errors.New("challenge verification failed") + } + + // We've authorized. + sc.remPubKey = remPubKey + return sc, nil +} + +// RemotePubKey returns authenticated remote pubkey +func (sc *SecretConnection) RemotePubKey() crypto.PubKey { + return sc.remPubKey +} + +// Writes encrypted frames of `totalFrameSize + aeadSizeOverhead`. +// CONTRACT: data smaller than dataMaxSize is written atomically. +func (sc *SecretConnection) Write(data []byte) (n int, err error) { + sc.sendMtx.Lock() + defer sc.sendMtx.Unlock() + + for 0 < len(data) { + if err := func() error { + var sealedFrame = pool.Get(aeadSizeOverhead + totalFrameSize) + var frame = pool.Get(totalFrameSize) + defer func() { + pool.Put(sealedFrame) + pool.Put(frame) + }() + var chunk []byte + if dataMaxSize < len(data) { + chunk = data[:dataMaxSize] + data = data[dataMaxSize:] + } else { + chunk = data + data = nil + } + chunkLength := len(chunk) + binary.LittleEndian.PutUint32(frame, uint32(chunkLength)) + copy(frame[dataLenSize:], chunk) + + // encrypt the frame + sc.sendAead.Seal(sealedFrame[:0], sc.sendNonce[:], frame, nil) + incrNonce(sc.sendNonce) + // end encryption + + _, err = sc.conn.Write(sealedFrame) + if err != nil { + return err + } + n += len(chunk) + return nil + }(); err != nil { + return n, err + } + } + return n, err +} + +// CONTRACT: data smaller than dataMaxSize is read atomically. +func (sc *SecretConnection) Read(data []byte) (n int, err error) { + sc.recvMtx.Lock() + defer sc.recvMtx.Unlock() + + // read off and update the recvBuffer, if non-empty + if 0 < len(sc.recvBuffer) { + n = copy(data, sc.recvBuffer) + sc.recvBuffer = sc.recvBuffer[n:] + return + } + + // read off the conn + var sealedFrame = pool.Get(aeadSizeOverhead + totalFrameSize) + defer pool.Put(sealedFrame) + _, err = io.ReadFull(sc.conn, sealedFrame) + if err != nil { + return + } + + // decrypt the frame. + // reads and updates the sc.recvNonce + var frame = pool.Get(totalFrameSize) + defer pool.Put(frame) + _, err = sc.recvAead.Open(frame[:0], sc.recvNonce[:], sealedFrame, nil) + if err != nil { + return n, fmt.Errorf("failed to decrypt SecretConnection: %w", err) + } + incrNonce(sc.recvNonce) + // end decryption + + // copy checkLength worth into data, + // set recvBuffer to the rest. + var chunkLength = binary.LittleEndian.Uint32(frame) // read the first four bytes + if chunkLength > dataMaxSize { + return 0, errors.New("chunkLength is greater than dataMaxSize") + } + var chunk = frame[dataLenSize : dataLenSize+chunkLength] + n = copy(data, chunk) + if n < len(chunk) { + sc.recvBuffer = make([]byte, len(chunk)-n) + copy(sc.recvBuffer, chunk[n:]) + } + return n, err +} + +// Implements net.Conn +// nolint +func (sc *SecretConnection) Close() error { return sc.conn.Close() } +func (sc *SecretConnection) LocalAddr() net.Addr { return sc.conn.(net.Conn).LocalAddr() } +func (sc *SecretConnection) RemoteAddr() net.Addr { return sc.conn.(net.Conn).RemoteAddr() } +func (sc *SecretConnection) SetDeadline(t time.Time) error { return sc.conn.(net.Conn).SetDeadline(t) } +func (sc *SecretConnection) SetReadDeadline(t time.Time) error { + return sc.conn.(net.Conn).SetReadDeadline(t) +} +func (sc *SecretConnection) SetWriteDeadline(t time.Time) error { + return sc.conn.(net.Conn).SetWriteDeadline(t) +} + +func genEphKeys() (ephPub, ephPriv *[32]byte) { + var err error + // TODO: Probably not a problem but ask Tony: different from the rust implementation (uses x25519-dalek), + // we do not "clamp" the private key scalar: + // see: https://github.com/dalek-cryptography/x25519-dalek/blob/34676d336049df2bba763cc076a75e47ae1f170f/src/x25519.rs#L56-L74 + ephPub, ephPriv, err = box.GenerateKey(crand.Reader) + if err != nil { + panic("Could not generate ephemeral key-pair") + } + return +} + +func shareEphPubKey(conn io.ReadWriter, locEphPub *[32]byte) (remEphPub *[32]byte, err error) { + + // Send our pubkey and receive theirs in tandem. + var trs, _ = async.Parallel( + func(_ int) (val interface{}, abort bool, err error) { + lc := *locEphPub + _, err = protoio.NewDelimitedWriter(conn).WriteMsg(&gogotypes.BytesValue{Value: lc[:]}) + if err != nil { + return nil, true, err // abort + } + return nil, false, nil + }, + func(_ int) (val interface{}, abort bool, err error) { + var bytes gogotypes.BytesValue + err = protoio.NewDelimitedReader(conn, 1024*1024).ReadMsg(&bytes) + if err != nil { + return nil, true, err // abort + } + + var _remEphPub [32]byte + copy(_remEphPub[:], bytes.Value) + return _remEphPub, false, nil + }, + ) + + // If error: + if trs.FirstError() != nil { + err = trs.FirstError() + return + } + + // Otherwise: + var _remEphPub = trs.FirstValue().([32]byte) + return &_remEphPub, nil +} + +func deriveSecrets( + dhSecret *[32]byte, + locIsLeast bool, +) (recvSecret, sendSecret *[aeadKeySize]byte) { + hash := sha256.New + hkdf := hkdf.New(hash, dhSecret[:], nil, secretConnKeyAndChallengeGen) + // get enough data for 2 aead keys, and a 32 byte challenge + res := new([2*aeadKeySize + 32]byte) + _, err := io.ReadFull(hkdf, res[:]) + if err != nil { + panic(err) + } + + recvSecret = new([aeadKeySize]byte) + sendSecret = new([aeadKeySize]byte) + + // bytes 0 through aeadKeySize - 1 are one aead key. + // bytes aeadKeySize through 2*aeadKeySize -1 are another aead key. + // which key corresponds to sending and receiving key depends on whether + // the local key is less than the remote key. + if locIsLeast { + copy(recvSecret[:], res[0:aeadKeySize]) + copy(sendSecret[:], res[aeadKeySize:aeadKeySize*2]) + } else { + copy(sendSecret[:], res[0:aeadKeySize]) + copy(recvSecret[:], res[aeadKeySize:aeadKeySize*2]) + } + + return +} + +// computeDHSecret computes a Diffie-Hellman shared secret key +// from our own local private key and the other's public key. +func computeDHSecret(remPubKey, locPrivKey *[32]byte) (*[32]byte, error) { + shrKey, err := curve25519.X25519(locPrivKey[:], remPubKey[:]) + if err != nil { + return nil, err + } + var shrKeyArray [32]byte + copy(shrKeyArray[:], shrKey) + return &shrKeyArray, nil +} + +func sort32(foo, bar *[32]byte) (lo, hi *[32]byte) { + if bytes.Compare(foo[:], bar[:]) < 0 { + lo = foo + hi = bar + } else { + lo = bar + hi = foo + } + return +} + +func signChallenge(challenge *[32]byte, locPrivKey crypto.PrivKey) ([]byte, error) { + signature, err := locPrivKey.Sign(challenge[:]) + if err != nil { + return nil, err + } + return signature, nil +} + +type authSigMessage struct { + Key crypto.PubKey + Sig []byte +} + +func shareAuthSignature(sc io.ReadWriter, pubKey crypto.PubKey, signature []byte) (recvMsg authSigMessage, err error) { + + // Send our info and receive theirs in tandem. + var trs, _ = async.Parallel( + func(_ int) (val interface{}, abort bool, err error) { + pbpk, err := cryptoenc.PubKeyToProto(pubKey) + if err != nil { + return nil, true, err + } + _, err = protoio.NewDelimitedWriter(sc).WriteMsg(&tmprivval.AuthSigMessage{PubKey: pbpk, Sig: signature}) + if err != nil { + return nil, true, err // abort + } + return nil, false, nil + }, + func(_ int) (val interface{}, abort bool, err error) { + var pba tmprivval.AuthSigMessage + err = protoio.NewDelimitedReader(sc, 1024*1024).ReadMsg(&pba) + if err != nil { + return nil, true, err // abort + } + + pk, err := cryptoenc.PubKeyFromProto(pba.PubKey) + if err != nil { + return nil, true, err // abort + } + + _recvMsg := authSigMessage{ + Key: pk, + Sig: pba.Sig, + } + return _recvMsg, false, nil + }, + ) + + // If error: + if trs.FirstError() != nil { + err = trs.FirstError() + return + } + + var _recvMsg = trs.FirstValue().(authSigMessage) + return _recvMsg, nil +} + +//-------------------------------------------------------------------------------- + +// Increment nonce little-endian by 1 with wraparound. +// Due to chacha20poly1305 expecting a 12 byte nonce we do not use the first four +// bytes. We only increment a 64 bit unsigned int in the remaining 8 bytes +// (little-endian in nonce[4:]). +func incrNonce(nonce *[aeadNonceSize]byte) { + counter := binary.LittleEndian.Uint64(nonce[4:]) + if counter == math.MaxUint64 { + // Terminates the session and makes sure the nonce would not re-used. + // See https://github.com/tendermint/tendermint/issues/3531 + panic("can't increase nonce without overflow") + } + counter++ + binary.LittleEndian.PutUint64(nonce[4:], counter) +} diff --git a/privval/socket_dialers.go b/privval/socket_dialers.go index b91b0a45a..9be84e02d 100644 --- a/privval/socket_dialers.go +++ b/privval/socket_dialers.go @@ -7,7 +7,6 @@ import ( "github.com/tendermint/tendermint/crypto" tmnet "github.com/tendermint/tendermint/libs/net" - p2pconn "github.com/tendermint/tendermint/p2p/conn" ) // Socket errors. @@ -28,7 +27,7 @@ func DialTCPFn(addr string, timeoutReadWrite time.Duration, privKey crypto.PrivK err = conn.SetDeadline(deadline) } if err == nil { - conn, err = p2pconn.MakeSecretConnection(conn, privKey) + conn, err = MakeSecretConnection(conn, privKey) } return conn, err } diff --git a/privval/socket_listeners.go b/privval/socket_listeners.go index ad49adac4..a6d031f8c 100644 --- a/privval/socket_listeners.go +++ b/privval/socket_listeners.go @@ -5,7 +5,6 @@ import ( "time" "github.com/tendermint/tendermint/crypto/ed25519" - p2pconn "github.com/tendermint/tendermint/p2p/conn" ) const ( @@ -76,7 +75,7 @@ func (ln *TCPListener) Accept() (net.Conn, error) { // Wrap the conn in our timeout and encryption wrappers timeoutConn := newTimeoutConn(tc, ln.timeoutReadWrite) - secretConn, err := p2pconn.MakeSecretConnection(timeoutConn, ln.secretConnKey) + secretConn, err := MakeSecretConnection(timeoutConn, ln.secretConnKey) if err != nil { return nil, err } diff --git a/proto/tendermint/privval/types.pb.go b/proto/tendermint/privval/types.pb.go index cab9975b1..da30f7527 100644 --- a/proto/tendermint/privval/types.pb.go +++ b/proto/tendermint/privval/types.pb.go @@ -661,6 +661,61 @@ func (*Message) XXX_OneofWrappers() []interface{} { } } +// AuthSigMessage is duplicated from p2p prior to the P2P refactor. +// It is used for the SecretConnection until we migrate privval to gRPC. +// https://github.com/tendermint/tendermint/issues/4698 +type AuthSigMessage struct { + PubKey crypto.PublicKey `protobuf:"bytes,1,opt,name=pub_key,json=pubKey,proto3" json:"pub_key"` + Sig []byte `protobuf:"bytes,2,opt,name=sig,proto3" json:"sig,omitempty"` +} + +func (m *AuthSigMessage) Reset() { *m = AuthSigMessage{} } +func (m *AuthSigMessage) String() string { return proto.CompactTextString(m) } +func (*AuthSigMessage) ProtoMessage() {} +func (*AuthSigMessage) Descriptor() ([]byte, []int) { + return fileDescriptor_cb4e437a5328cf9c, []int{10} +} +func (m *AuthSigMessage) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *AuthSigMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_AuthSigMessage.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *AuthSigMessage) XXX_Merge(src proto.Message) { + xxx_messageInfo_AuthSigMessage.Merge(m, src) +} +func (m *AuthSigMessage) XXX_Size() int { + return m.Size() +} +func (m *AuthSigMessage) XXX_DiscardUnknown() { + xxx_messageInfo_AuthSigMessage.DiscardUnknown(m) +} + +var xxx_messageInfo_AuthSigMessage proto.InternalMessageInfo + +func (m *AuthSigMessage) GetPubKey() crypto.PublicKey { + if m != nil { + return m.PubKey + } + return crypto.PublicKey{} +} + +func (m *AuthSigMessage) GetSig() []byte { + if m != nil { + return m.Sig + } + return nil +} + func init() { proto.RegisterEnum("tendermint.privval.Errors", Errors_name, Errors_value) proto.RegisterType((*RemoteSignerError)(nil), "tendermint.privval.RemoteSignerError") @@ -673,59 +728,62 @@ func init() { proto.RegisterType((*PingRequest)(nil), "tendermint.privval.PingRequest") proto.RegisterType((*PingResponse)(nil), "tendermint.privval.PingResponse") proto.RegisterType((*Message)(nil), "tendermint.privval.Message") + proto.RegisterType((*AuthSigMessage)(nil), "tendermint.privval.AuthSigMessage") } func init() { proto.RegisterFile("tendermint/privval/types.proto", fileDescriptor_cb4e437a5328cf9c) } var fileDescriptor_cb4e437a5328cf9c = []byte{ - // 750 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x55, 0x4d, 0x4f, 0x13, 0x41, - 0x18, 0xde, 0x85, 0x7e, 0xc0, 0x5b, 0x5a, 0xca, 0x80, 0x58, 0x1a, 0x5c, 0x6a, 0x8d, 0x4a, 0x7a, - 0x68, 0x0d, 0x26, 0x26, 0x06, 0x2f, 0x02, 0x1b, 0xdb, 0x34, 0x6c, 0xeb, 0xb4, 0x08, 0x21, 0x31, - 0x9b, 0x7e, 0x8c, 0xcb, 0x06, 0xba, 0x3b, 0xee, 0x6c, 0x49, 0x7a, 0xf6, 0xe6, 0xc9, 0xc4, 0x3f, - 0xe1, 0xd9, 0x5f, 0xc1, 0x91, 0xa3, 0x27, 0x63, 0xe0, 0x8f, 0x98, 0xce, 0x4e, 0xb7, 0xdb, 0x2f, - 0xa2, 0xe1, 0xb6, 0xf3, 0xbe, 0xef, 0x3c, 0x1f, 0x33, 0xcf, 0x66, 0x40, 0x71, 0x89, 0xd5, 0x26, - 0x4e, 0xc7, 0xb4, 0xdc, 0x02, 0x75, 0xcc, 0xcb, 0xcb, 0xc6, 0x45, 0xc1, 0xed, 0x51, 0xc2, 0xf2, - 0xd4, 0xb1, 0x5d, 0x1b, 0xa1, 0x61, 0x3f, 0x2f, 0xfa, 0xe9, 0xcd, 0xc0, 0x9e, 0x96, 0xd3, 0xa3, - 0xae, 0x5d, 0x38, 0x27, 0x3d, 0xb1, 0x63, 0xa4, 0xcb, 0x91, 0x82, 0x78, 0xe9, 0x35, 0xc3, 0x36, - 0x6c, 0xfe, 0x59, 0xe8, 0x7f, 0x79, 0xd5, 0x6c, 0x09, 0x56, 0x30, 0xe9, 0xd8, 0x2e, 0xa9, 0x99, - 0x86, 0x45, 0x1c, 0xd5, 0x71, 0x6c, 0x07, 0x21, 0x08, 0xb5, 0xec, 0x36, 0x49, 0xc9, 0x19, 0x79, - 0x3b, 0x8c, 0xf9, 0x37, 0xca, 0x40, 0xac, 0x4d, 0x58, 0xcb, 0x31, 0xa9, 0x6b, 0xda, 0x56, 0x6a, - 0x2e, 0x23, 0x6f, 0x2f, 0xe2, 0x60, 0x29, 0x9b, 0x83, 0x78, 0xb5, 0xdb, 0x2c, 0x93, 0x1e, 0x26, - 0x9f, 0xbb, 0x84, 0xb9, 0x68, 0x03, 0x16, 0x5a, 0x67, 0x0d, 0xd3, 0xd2, 0xcd, 0x36, 0x87, 0x5a, - 0xc4, 0x51, 0xbe, 0x2e, 0xb5, 0xb3, 0x5f, 0x65, 0x48, 0x0c, 0x86, 0x19, 0xb5, 0x2d, 0x46, 0xd0, - 0x2e, 0x44, 0x69, 0xb7, 0xa9, 0x9f, 0x93, 0x1e, 0x1f, 0x8e, 0xed, 0x6c, 0xe6, 0x03, 0x27, 0xe0, - 0xb9, 0xcd, 0x57, 0xbb, 0xcd, 0x0b, 0xb3, 0x55, 0x26, 0xbd, 0xbd, 0xd0, 0xd5, 0xef, 0x2d, 0x09, - 0x47, 0x28, 0x07, 0x41, 0xbb, 0x10, 0x26, 0x7d, 0xe9, 0x5c, 0x57, 0x6c, 0xe7, 0x69, 0x7e, 0xf2, - 0xf0, 0xf2, 0x13, 0x3e, 0xb1, 0xb7, 0x27, 0x7b, 0x02, 0xcb, 0xfd, 0xea, 0x07, 0xdb, 0x25, 0x03, - 0xe9, 0x39, 0x08, 0x5d, 0xda, 0x2e, 0x11, 0x4a, 0xd6, 0x83, 0x70, 0xde, 0x99, 0xf2, 0x61, 0x3e, - 0x33, 0x62, 0x73, 0x6e, 0xd4, 0xe6, 0x17, 0x19, 0x10, 0x27, 0x6c, 0x7b, 0xe0, 0xc2, 0xea, 0x8b, - 0x7f, 0x41, 0x17, 0x0e, 0x3d, 0x8e, 0x7b, 0xf9, 0x3b, 0x83, 0xd5, 0x7e, 0xb5, 0xea, 0xd8, 0xd4, - 0x66, 0x8d, 0x8b, 0x81, 0xc7, 0x57, 0xb0, 0x40, 0x45, 0x49, 0x28, 0x49, 0x4f, 0x2a, 0xf1, 0x37, - 0xf9, 0xb3, 0x77, 0xf9, 0xfd, 0x2e, 0xc3, 0xba, 0xe7, 0x77, 0x48, 0x26, 0x3c, 0xbf, 0xf9, 0x1f, - 0x36, 0xe1, 0x7d, 0xc8, 0x79, 0x2f, 0xff, 0x71, 0x88, 0x55, 0x4d, 0xcb, 0x10, 0xbe, 0xb3, 0x09, - 0x58, 0xf2, 0x96, 0x9e, 0xb2, 0xec, 0xcf, 0x30, 0x44, 0x0f, 0x09, 0x63, 0x0d, 0x83, 0xa0, 0x32, - 0x2c, 0x8b, 0x10, 0xea, 0x8e, 0x37, 0x2e, 0xc4, 0x3e, 0x9e, 0xc6, 0x38, 0x12, 0xf7, 0xa2, 0x84, - 0xe3, 0x74, 0x24, 0xff, 0x1a, 0x24, 0x87, 0x60, 0x1e, 0x99, 0xd0, 0x9f, 0xbd, 0x0b, 0xcd, 0x9b, - 0x2c, 0x4a, 0x38, 0x41, 0x47, 0xff, 0x90, 0xf7, 0xb0, 0xc2, 0x4c, 0xc3, 0xd2, 0xfb, 0x89, 0xf0, - 0xe5, 0xcd, 0x73, 0xc0, 0x27, 0xd3, 0x00, 0xc7, 0x42, 0x5d, 0x94, 0xf0, 0x32, 0x1b, 0xcb, 0xf9, - 0x29, 0xac, 0x31, 0x7e, 0x5f, 0x03, 0x50, 0x21, 0x33, 0xc4, 0x51, 0x9f, 0xcd, 0x42, 0x1d, 0xcd, - 0x73, 0x51, 0xc2, 0x88, 0x4d, 0xa6, 0xfc, 0x23, 0x3c, 0xe0, 0x72, 0x07, 0x97, 0xe8, 0x4b, 0x0e, - 0x73, 0xf0, 0xe7, 0xb3, 0xc0, 0xc7, 0x72, 0x5a, 0x94, 0xf0, 0x2a, 0x9b, 0x12, 0xdf, 0x4f, 0x90, - 0x12, 0xd2, 0x03, 0x04, 0x42, 0x7e, 0x84, 0x33, 0xe4, 0x66, 0xcb, 0x1f, 0x8f, 0x67, 0x51, 0xc2, - 0xeb, 0x6c, 0x7a, 0x70, 0x0f, 0x60, 0x89, 0x9a, 0x96, 0xe1, 0xab, 0x8f, 0x72, 0xec, 0xad, 0xa9, - 0x37, 0x38, 0x4c, 0x59, 0x51, 0xc2, 0x31, 0x3a, 0x5c, 0xa2, 0x77, 0x10, 0x17, 0x28, 0x42, 0xe2, - 0x02, 0x87, 0xc9, 0xcc, 0x86, 0xf1, 0x85, 0x2d, 0xd1, 0xc0, 0x7a, 0x2f, 0x0c, 0xf3, 0xac, 0xdb, - 0xc9, 0xfd, 0x90, 0x21, 0xc2, 0x43, 0xce, 0x10, 0x82, 0x84, 0x8a, 0x71, 0x05, 0xd7, 0xf4, 0x23, - 0xad, 0xac, 0x55, 0x8e, 0xb5, 0xa4, 0x84, 0x14, 0x48, 0xfb, 0x35, 0xf5, 0xa4, 0xaa, 0xee, 0xd7, - 0xd5, 0x03, 0x1d, 0xab, 0xb5, 0x6a, 0x45, 0xab, 0xa9, 0x49, 0x19, 0xa5, 0x60, 0x4d, 0xf4, 0xb5, - 0x8a, 0xbe, 0x5f, 0xd1, 0x34, 0x75, 0xbf, 0x5e, 0xaa, 0x68, 0xc9, 0x39, 0xf4, 0x08, 0x36, 0x44, - 0x67, 0x58, 0xd6, 0xeb, 0xa5, 0x43, 0xb5, 0x72, 0x54, 0x4f, 0xce, 0xa3, 0x87, 0xb0, 0x2a, 0xda, - 0x58, 0x7d, 0x7b, 0xe0, 0x37, 0x42, 0x01, 0xc4, 0x63, 0x5c, 0xaa, 0xab, 0x7e, 0x27, 0xbc, 0x57, - 0xbb, 0xba, 0x51, 0xe4, 0xeb, 0x1b, 0x45, 0xfe, 0x73, 0xa3, 0xc8, 0xdf, 0x6e, 0x15, 0xe9, 0xfa, - 0x56, 0x91, 0x7e, 0xdd, 0x2a, 0xd2, 0xe9, 0x6b, 0xc3, 0x74, 0xcf, 0xba, 0xcd, 0x7c, 0xcb, 0xee, - 0x14, 0x82, 0x6f, 0x57, 0xf0, 0x61, 0xec, 0xbf, 0x57, 0x93, 0x2f, 0x65, 0x33, 0xc2, 0x3b, 0x2f, - 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0xa5, 0x2a, 0xe5, 0x4a, 0x46, 0x07, 0x00, 0x00, + // 779 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x55, 0x4b, 0x4f, 0xdb, 0x4a, + 0x14, 0xb6, 0xc9, 0x0b, 0x4e, 0x1e, 0x84, 0x81, 0xcb, 0x0d, 0x11, 0xd7, 0xe4, 0xfa, 0xea, 0xb6, + 0x28, 0x8b, 0xa4, 0xa2, 0x52, 0xa5, 0x8a, 0x6e, 0x78, 0x58, 0x4d, 0x14, 0xe1, 0xa4, 0x93, 0x50, + 0x10, 0x52, 0x65, 0xe5, 0x31, 0x75, 0x2c, 0x88, 0xed, 0x7a, 0x1c, 0xa4, 0xac, 0xbb, 0xeb, 0xaa, + 0x52, 0xff, 0x44, 0xd7, 0xfd, 0x15, 0x2c, 0x59, 0x76, 0x55, 0x55, 0xf0, 0x47, 0xaa, 0x8c, 0x27, + 0x8e, 0xf3, 0x42, 0xad, 0xd8, 0xcd, 0x9c, 0x73, 0xe6, 0x3b, 0xdf, 0x37, 0xf3, 0xd9, 0x07, 0x24, + 0x97, 0x98, 0x1d, 0xe2, 0xf4, 0x0c, 0xd3, 0x2d, 0xda, 0x8e, 0x71, 0x7d, 0xdd, 0xbc, 0x2a, 0xba, + 0x03, 0x9b, 0xd0, 0x82, 0xed, 0x58, 0xae, 0x85, 0xd0, 0x38, 0x5f, 0xe0, 0xf9, 0xec, 0x76, 0xe0, + 0x4c, 0xdb, 0x19, 0xd8, 0xae, 0x55, 0xbc, 0x24, 0x03, 0x7e, 0x62, 0x22, 0xcb, 0x90, 0x82, 0x78, + 0xd9, 0x0d, 0xdd, 0xd2, 0x2d, 0xb6, 0x2c, 0x0e, 0x57, 0x5e, 0x54, 0x2e, 0xc3, 0x1a, 0x26, 0x3d, + 0xcb, 0x25, 0x75, 0x43, 0x37, 0x89, 0xa3, 0x38, 0x8e, 0xe5, 0x20, 0x04, 0xe1, 0xb6, 0xd5, 0x21, + 0x19, 0x31, 0x27, 0xee, 0x46, 0x30, 0x5b, 0xa3, 0x1c, 0xc4, 0x3b, 0x84, 0xb6, 0x1d, 0xc3, 0x76, + 0x0d, 0xcb, 0xcc, 0x2c, 0xe5, 0xc4, 0xdd, 0x15, 0x1c, 0x0c, 0xc9, 0x79, 0x48, 0xd6, 0xfa, 0xad, + 0x0a, 0x19, 0x60, 0xf2, 0xa1, 0x4f, 0xa8, 0x8b, 0xb6, 0x60, 0xb9, 0xdd, 0x6d, 0x1a, 0xa6, 0x66, + 0x74, 0x18, 0xd4, 0x0a, 0x8e, 0xb1, 0x7d, 0xb9, 0x23, 0x7f, 0x12, 0x21, 0x35, 0x2a, 0xa6, 0xb6, + 0x65, 0x52, 0x82, 0xf6, 0x21, 0x66, 0xf7, 0x5b, 0xda, 0x25, 0x19, 0xb0, 0xe2, 0xf8, 0xde, 0x76, + 0x21, 0x70, 0x03, 0x9e, 0xda, 0x42, 0xad, 0xdf, 0xba, 0x32, 0xda, 0x15, 0x32, 0x38, 0x0c, 0xdf, + 0xfc, 0xd8, 0x11, 0x70, 0xd4, 0x66, 0x20, 0x68, 0x1f, 0x22, 0x64, 0x48, 0x9d, 0xf1, 0x8a, 0xef, + 0xfd, 0x5f, 0x98, 0xbd, 0xbc, 0xc2, 0x8c, 0x4e, 0xec, 0x9d, 0x91, 0xcf, 0x61, 0x75, 0x18, 0x7d, + 0x6b, 0xb9, 0x64, 0x44, 0x3d, 0x0f, 0xe1, 0x6b, 0xcb, 0x25, 0x9c, 0xc9, 0x66, 0x10, 0xce, 0xbb, + 0x53, 0x56, 0xcc, 0x6a, 0x26, 0x64, 0x2e, 0x4d, 0xca, 0xfc, 0x28, 0x02, 0x62, 0x0d, 0x3b, 0x1e, + 0x38, 0x97, 0xfa, 0xec, 0x77, 0xd0, 0xb9, 0x42, 0xaf, 0xc7, 0xa3, 0xf4, 0x75, 0x61, 0x7d, 0x18, + 0xad, 0x39, 0x96, 0x6d, 0xd1, 0xe6, 0xd5, 0x48, 0xe3, 0x0b, 0x58, 0xb6, 0x79, 0x88, 0x33, 0xc9, + 0xce, 0x32, 0xf1, 0x0f, 0xf9, 0xb5, 0x0f, 0xe9, 0xfd, 0x22, 0xc2, 0xa6, 0xa7, 0x77, 0xdc, 0x8c, + 0x6b, 0x7e, 0xf5, 0x27, 0xdd, 0xb8, 0xf6, 0x71, 0xcf, 0x47, 0xe9, 0x4f, 0x42, 0xbc, 0x66, 0x98, + 0x3a, 0xd7, 0x2d, 0xa7, 0x20, 0xe1, 0x6d, 0x3d, 0x66, 0xf2, 0xb7, 0x08, 0xc4, 0x4e, 0x08, 0xa5, + 0x4d, 0x9d, 0xa0, 0x0a, 0xac, 0x72, 0x13, 0x6a, 0x8e, 0x57, 0xce, 0xc9, 0xfe, 0x3b, 0xaf, 0xe3, + 0x84, 0xdd, 0x4b, 0x02, 0x4e, 0xda, 0x13, 0xfe, 0x57, 0x21, 0x3d, 0x06, 0xf3, 0x9a, 0x71, 0xfe, + 0xf2, 0x43, 0x68, 0x5e, 0x65, 0x49, 0xc0, 0x29, 0x7b, 0xf2, 0x0b, 0x79, 0x03, 0x6b, 0xd4, 0xd0, + 0x4d, 0x6d, 0xe8, 0x08, 0x9f, 0x5e, 0x88, 0x01, 0xfe, 0x37, 0x0f, 0x70, 0xca, 0xd4, 0x25, 0x01, + 0xaf, 0xd2, 0x29, 0x9f, 0x5f, 0xc0, 0x06, 0x65, 0xef, 0x35, 0x02, 0xe5, 0x34, 0xc3, 0x0c, 0xf5, + 0xc9, 0x22, 0xd4, 0x49, 0x3f, 0x97, 0x04, 0x8c, 0xe8, 0xac, 0xcb, 0xdf, 0xc1, 0x5f, 0x8c, 0xee, + 0xe8, 0x11, 0x7d, 0xca, 0x11, 0x06, 0xfe, 0x74, 0x11, 0xf8, 0x94, 0x4f, 0x4b, 0x02, 0x5e, 0xa7, + 0x73, 0xec, 0xfb, 0x1e, 0x32, 0x9c, 0x7a, 0xa0, 0x01, 0xa7, 0x1f, 0x65, 0x1d, 0xf2, 0x8b, 0xe9, + 0x4f, 0xdb, 0xb3, 0x24, 0xe0, 0x4d, 0x3a, 0xdf, 0xb8, 0xc7, 0x90, 0xb0, 0x0d, 0x53, 0xf7, 0xd9, + 0xc7, 0x18, 0xf6, 0xce, 0xdc, 0x17, 0x1c, 0xbb, 0xac, 0x24, 0xe0, 0xb8, 0x3d, 0xde, 0xa2, 0xd7, + 0x90, 0xe4, 0x28, 0x9c, 0xe2, 0x32, 0x83, 0xc9, 0x2d, 0x86, 0xf1, 0x89, 0x25, 0xec, 0xc0, 0xfe, + 0x30, 0x02, 0x21, 0xda, 0xef, 0xc9, 0x1a, 0xa4, 0x0e, 0xfa, 0x6e, 0xb7, 0x6e, 0xe8, 0x23, 0xeb, + 0x3e, 0xea, 0xff, 0x99, 0x86, 0x10, 0x35, 0x74, 0xe6, 0xce, 0x04, 0x1e, 0x2e, 0xf3, 0x5f, 0x45, + 0x88, 0xb2, 0xaf, 0x88, 0x22, 0x04, 0x29, 0x05, 0xe3, 0x2a, 0xae, 0x6b, 0xa7, 0x6a, 0x45, 0xad, + 0x9e, 0xa9, 0x69, 0x01, 0x49, 0x90, 0xf5, 0x63, 0xca, 0x79, 0x4d, 0x39, 0x6a, 0x28, 0xc7, 0x1a, + 0x56, 0xea, 0xb5, 0xaa, 0x5a, 0x57, 0xd2, 0x22, 0xca, 0xc0, 0x06, 0xcf, 0xab, 0x55, 0xed, 0xa8, + 0xaa, 0xaa, 0xca, 0x51, 0xa3, 0x5c, 0x55, 0xd3, 0x4b, 0xe8, 0x1f, 0xd8, 0xe2, 0x99, 0x71, 0x58, + 0x6b, 0x94, 0x4f, 0x94, 0xea, 0x69, 0x23, 0x1d, 0x42, 0x7f, 0xc3, 0x3a, 0x4f, 0x63, 0xe5, 0xe0, + 0xd8, 0x4f, 0x84, 0x03, 0x88, 0x67, 0xb8, 0xdc, 0x50, 0xfc, 0x4c, 0xe4, 0xb0, 0x7e, 0x73, 0x27, + 0x89, 0xb7, 0x77, 0x92, 0xf8, 0xf3, 0x4e, 0x12, 0x3f, 0xdf, 0x4b, 0xc2, 0xed, 0xbd, 0x24, 0x7c, + 0xbf, 0x97, 0x84, 0x8b, 0x97, 0xba, 0xe1, 0x76, 0xfb, 0xad, 0x42, 0xdb, 0xea, 0x15, 0x83, 0xc3, + 0x31, 0x38, 0x79, 0x87, 0x03, 0x71, 0x76, 0x14, 0xb7, 0xa2, 0x2c, 0xf3, 0xfc, 0x57, 0x00, 0x00, + 0x00, 0xff, 0xff, 0x46, 0x64, 0xeb, 0xa4, 0xa7, 0x07, 0x00, 0x00, } func (m *RemoteSignerError) Marshal() (dAtA []byte, err error) { @@ -1258,6 +1316,46 @@ func (m *Message_PingResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { } return len(dAtA) - i, nil } +func (m *AuthSigMessage) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *AuthSigMessage) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *AuthSigMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Sig) > 0 { + i -= len(m.Sig) + copy(dAtA[i:], m.Sig) + i = encodeVarintTypes(dAtA, i, uint64(len(m.Sig))) + i-- + dAtA[i] = 0x12 + } + { + size, err := m.PubKey.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintTypes(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + return len(dAtA) - i, nil +} + func encodeVarintTypes(dAtA []byte, offset int, v uint64) int { offset -= sovTypes(v) base := offset @@ -1503,6 +1601,20 @@ func (m *Message_PingResponse) Size() (n int) { } return n } +func (m *AuthSigMessage) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = m.PubKey.Size() + n += 1 + l + sovTypes(uint64(l)) + l = len(m.Sig) + if l > 0 { + n += 1 + l + sovTypes(uint64(l)) + } + return n +} func sovTypes(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 @@ -2746,6 +2858,126 @@ func (m *Message) Unmarshal(dAtA []byte) error { } return nil } +func (m *AuthSigMessage) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: AuthSigMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: AuthSigMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PubKey", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthTypes + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.PubKey.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Sig", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthTypes + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Sig = append(m.Sig[:0], dAtA[iNdEx:postIndex]...) + if m.Sig == nil { + m.Sig = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipTypes(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthTypes + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthTypes + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipTypes(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 diff --git a/proto/tendermint/privval/types.proto b/proto/tendermint/privval/types.proto index 0fc8b61dc..0d712447c 100644 --- a/proto/tendermint/privval/types.proto +++ b/proto/tendermint/privval/types.proto @@ -74,3 +74,11 @@ message Message { PingResponse ping_response = 8; } } + +// AuthSigMessage is duplicated from p2p prior to the P2P refactor. +// It is used for the SecretConnection until we migrate privval to gRPC. +// https://github.com/tendermint/tendermint/issues/4698 +message AuthSigMessage { + tendermint.crypto.PublicKey pub_key = 1 [(gogoproto.nullable) = false]; + bytes sig = 2; +} From 39e81807a37c39240f3c7357cfeeb46dcc3fab84 Mon Sep 17 00:00:00 2001 From: Marko Date: Tue, 17 Nov 2020 10:30:35 +0100 Subject: [PATCH 14/27] buf: modify buf.yml, add buf generate (#5653) --- buf.gen.yaml | 13 +++++++++++++ buf.yaml | 2 ++ scripts/protocgen.sh | 15 +-------------- 3 files changed, 16 insertions(+), 14 deletions(-) create mode 100644 buf.gen.yaml diff --git a/buf.gen.yaml b/buf.gen.yaml new file mode 100644 index 000000000..dc56781dd --- /dev/null +++ b/buf.gen.yaml @@ -0,0 +1,13 @@ +# The version of the generation template. +# Required. +# The only currently-valid value is v1beta1. +version: v1beta1 + +# The plugins to run. +plugins: + # The name of the plugin. + - name: gogofaster + # The the relative output directory. + out: proto + # Any options to provide to the plugin. + opt: Mgoogle/protobuf/timestamp.proto=github.com/gogo/protobuf/types,Mgoogle/protobuf/duration.proto=github.com/golang/protobuf/ptypes/duration,plugins=grpc,paths=source_relative diff --git a/buf.yaml b/buf.yaml index d21611209..cc4aced57 100644 --- a/buf.yaml +++ b/buf.yaml @@ -1,3 +1,5 @@ +version: v1beta1 + build: roots: - proto diff --git a/scripts/protocgen.sh b/scripts/protocgen.sh index 16f0ec733..51b1cc6d3 100755 --- a/scripts/protocgen.sh +++ b/scripts/protocgen.sh @@ -2,20 +2,7 @@ set -eo pipefail -proto_dirs=$(find ./proto -path -prune -o -name '*.proto' -print0 | xargs -0 -n1 dirname | sort | uniq) -for dir in $proto_dirs; do - buf protoc \ - -I "proto" \ - -I "third_party/proto" \ - --gogofaster_out=\ -Mgoogle/protobuf/timestamp.proto=github.com/gogo/protobuf/types,\ -Mgoogle/protobuf/duration.proto=github.com/golang/protobuf/ptypes/duration,\ -plugins=grpc,paths=source_relative:. \ - $(find "${dir}" -maxdepth 1 -name '*.proto') -done - -cp -r ./tendermint/* ./proto/* -rm -rf tendermint +buf generate --path proto/tendermint mv ./proto/tendermint/abci/types.pb.go ./abci/types From fbf2309962cfecefa04426f687af30d4967d3bc5 Mon Sep 17 00:00:00 2001 From: Marko Date: Tue, 17 Nov 2020 11:49:57 +0100 Subject: [PATCH 15/27] ci: remove `add-path` (#5674) --- .github/workflows/tests.yml | 20 ++++---------------- crypto/ed25519/ed25519.go | 1 + 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 54a94ae97..abd79aad4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -25,7 +25,7 @@ jobs: steps: - uses: actions/setup-go@v2 with: - go-version: '^1.15.4' + go-version: "^1.15.4" - uses: actions/checkout@v2 - uses: technote-space/get-diff-action@v4 with: @@ -33,9 +33,6 @@ jobs: **/**.go go.mod go.sum - - name: Set GOBIN - run: | - echo "::add-path::$(go env GOPATH)/bin" - name: install run: make install install_abci if: "env.GIT_DIFF != ''" @@ -60,7 +57,7 @@ jobs: steps: - uses: actions/setup-go@v2 with: - go-version: '^1.15.4' + go-version: "^1.15.4" - uses: actions/checkout@v2 - uses: technote-space/get-diff-action@v4 with: @@ -68,9 +65,6 @@ jobs: **/**.go go.mod go.sum - - name: Set GOBIN - run: | - echo "::add-path::$(go env GOPATH)/bin" - uses: actions/cache@v2.1.3 with: path: ~/go/pkg/mod @@ -95,7 +89,7 @@ jobs: steps: - uses: actions/setup-go@v2 with: - go-version: '^1.15.4' + go-version: "^1.15.4" - uses: actions/checkout@v2 - uses: technote-space/get-diff-action@v4 with: @@ -103,9 +97,6 @@ jobs: **/**.go go.mod go.sum - - name: Set GOBIN - run: | - echo "::add-path::$(go env GOPATH)/bin" - uses: actions/cache@v2.1.3 with: path: ~/go/pkg/mod @@ -129,7 +120,7 @@ jobs: steps: - uses: actions/setup-go@v2 with: - go-version: '^1.15.4' + go-version: "^1.15.4" - uses: actions/checkout@v2 - uses: technote-space/get-diff-action@v4 with: @@ -137,9 +128,6 @@ jobs: **/**.go go.mod go.sum - - name: Set GOBIN - run: | - echo "::add-path::$(go env GOPATH)/bin" - uses: actions/cache@v2.1.3 with: path: ~/go/pkg/mod diff --git a/crypto/ed25519/ed25519.go b/crypto/ed25519/ed25519.go index 5bed1adde..30d470485 100644 --- a/crypto/ed25519/ed25519.go +++ b/crypto/ed25519/ed25519.go @@ -8,6 +8,7 @@ import ( "io" "github.com/hdevalence/ed25519consensus" + "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/crypto/tmhash" tmjson "github.com/tendermint/tendermint/libs/json" From 4e71357808bd32bd04b0f057efb20d15d74f627b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 Nov 2020 12:15:57 +0100 Subject: [PATCH 16/27] build(deps): Bump codecov/codecov-action from v1.0.14 to v1.0.15 (#5676) Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from v1.0.14 to v1.0.15. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Commits](https://github.com/codecov/codecov-action/compare/v1.0.14...239febf655bba88b16ff5dea1d3135ea8663a1f9) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/coverage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index f788b4962..0861ce402 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -94,7 +94,7 @@ jobs: - run: | cat ./*profile.out | grep -v "mode: atomic" >> coverage.txt if: env.GIT_DIFF - - uses: codecov/codecov-action@v1.0.14 + - uses: codecov/codecov-action@v1.0.15 with: file: ./coverage.txt if: env.GIT_DIFF From 909da42789f65018f4024967eee432cb146a97de Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Tue, 17 Nov 2020 14:23:16 +0100 Subject: [PATCH 17/27] light: make fraction parts uint64, ensuring that it is always positive (#5655) --- CHANGELOG_PENDING.md | 2 ++ libs/math/fraction.go | 24 +++++++++++++++--------- libs/math/fraction_test.go | 26 ++++++++++++++++++++++---- light/verifier_test.go | 10 ++++------ types/validator_set.go | 4 ++-- 5 files changed, 45 insertions(+), 21 deletions(-) diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 9b790f286..fe066695c 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -23,6 +23,8 @@ Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermi ### FEATURES +- [libs/math] \#5665 Make fractions unsigned integers (uint64) (@cmwaters) + ### IMPROVEMENTS - [crypto/ed25519] \#5632 Adopt zip215 `ed25519` verification. (@marbar3778) diff --git a/libs/math/fraction.go b/libs/math/fraction.go index 399bc1c18..e38636485 100644 --- a/libs/math/fraction.go +++ b/libs/math/fraction.go @@ -3,18 +3,18 @@ package math import ( "errors" "fmt" + "math" "strconv" "strings" ) -// Fraction defined in terms of a numerator divided by a denominator in int64 -// format. +// Fraction defined in terms of a numerator divided by a denominator in uint64 +// format. Fraction must be positive. type Fraction struct { // The portion of the denominator in the faction, e.g. 2 in 2/3. - Numerator int64 `json:"numerator"` - // The value by which the numerator is divided, e.g. 3 in 2/3. Must be - // positive. - Denominator int64 `json:"denominator"` + Numerator uint64 `json:"numerator"` + // The value by which the numerator is divided, e.g. 3 in 2/3. + Denominator uint64 `json:"denominator"` } func (fr Fraction) String() string { @@ -27,16 +27,22 @@ func (fr Fraction) String() string { func ParseFraction(f string) (Fraction, error) { o := strings.Split(f, "/") if len(o) != 2 { - return Fraction{}, errors.New("incorrect formating: should be like \"1/3\"") + return Fraction{}, errors.New("incorrect formating: should have a single slash i.e. \"1/3\"") } - numerator, err := strconv.ParseInt(o[0], 10, 64) + numerator, err := strconv.ParseUint(o[0], 10, 64) if err != nil { return Fraction{}, fmt.Errorf("incorrect formatting, err: %w", err) } - denominator, err := strconv.ParseInt(o[1], 10, 64) + denominator, err := strconv.ParseUint(o[1], 10, 64) if err != nil { return Fraction{}, fmt.Errorf("incorrect formatting, err: %w", err) } + if denominator == 0 { + return Fraction{}, errors.New("denominator can't be 0") + } + if numerator > math.MaxInt64 || denominator > math.MaxInt64 { + return Fraction{}, fmt.Errorf("value overflow, numerator and denominator must be less than %d", math.MaxInt64) + } return Fraction{Numerator: numerator, Denominator: denominator}, nil } diff --git a/libs/math/fraction_test.go b/libs/math/fraction_test.go index e4cabd32d..73ca0f6c8 100644 --- a/libs/math/fraction_test.go +++ b/libs/math/fraction_test.go @@ -23,15 +23,33 @@ func TestParseFraction(t *testing.T) { exp: Fraction{15, 5}, err: false, }, + // test divide by zero error + { + f: "2/0", + exp: Fraction{}, + err: true, + }, + // test negative { f: "-1/2", - exp: Fraction{-1, 2}, - err: false, + exp: Fraction{}, + err: true, }, { f: "1/-2", - exp: Fraction{1, -2}, - err: false, + exp: Fraction{}, + err: true, + }, + // test overflow + { + f: "9223372036854775808/2", + exp: Fraction{}, + err: true, + }, + { + f: "2/9223372036854775808", + exp: Fraction{}, + err: true, }, { f: "2/3/4", diff --git a/light/verifier_test.go b/light/verifier_test.go index 712f3dac5..9e10810b2 100644 --- a/light/verifier_test.go +++ b/light/verifier_test.go @@ -318,12 +318,10 @@ func TestValidateTrustLevel(t *testing.T) { 4: {tmmath.Fraction{Numerator: 4, Denominator: 5}, true}, // invalid - 5: {tmmath.Fraction{Numerator: 6, Denominator: 5}, false}, - 6: {tmmath.Fraction{Numerator: -1, Denominator: 3}, false}, - 7: {tmmath.Fraction{Numerator: 0, Denominator: 1}, false}, - 8: {tmmath.Fraction{Numerator: -1, Denominator: -3}, false}, - 9: {tmmath.Fraction{Numerator: 0, Denominator: 0}, false}, - 10: {tmmath.Fraction{Numerator: 1, Denominator: 0}, false}, + 5: {tmmath.Fraction{Numerator: 6, Denominator: 5}, false}, + 6: {tmmath.Fraction{Numerator: 0, Denominator: 1}, false}, + 7: {tmmath.Fraction{Numerator: 0, Denominator: 0}, false}, + 8: {tmmath.Fraction{Numerator: 1, Denominator: 0}, false}, } for _, tc := range testCases { diff --git a/types/validator_set.go b/types/validator_set.go index c9f6194bd..7cff0405d 100644 --- a/types/validator_set.go +++ b/types/validator_set.go @@ -788,11 +788,11 @@ func (vals *ValidatorSet) VerifyCommitLightTrusting(chainID string, commit *Comm ) // Safely calculate voting power needed. - totalVotingPowerMulByNumerator, overflow := safeMul(vals.TotalVotingPower(), trustLevel.Numerator) + totalVotingPowerMulByNumerator, overflow := safeMul(vals.TotalVotingPower(), int64(trustLevel.Numerator)) if overflow { return errors.New("int64 overflow while calculating voting power needed. please provide smaller trustLevel numerator") } - votingPowerNeeded := totalVotingPowerMulByNumerator / trustLevel.Denominator + votingPowerNeeded := totalVotingPowerMulByNumerator / int64(trustLevel.Denominator) for idx, commitSig := range commit.Signatures { // No need to verify absent or nil votes. From b2d72dce7efefe2d9c992f2eb5d1ab83818f0e48 Mon Sep 17 00:00:00 2001 From: Marko Date: Tue, 17 Nov 2020 14:55:23 +0100 Subject: [PATCH 18/27] e2e: use ed25519 for secretConn (remote signer) (#5678) ## Description Hardcode ed25519 to dialTCPFn in e2e tests. I will backport `DefaultRequestHandler` fixes This will be replaced when grpc is implemented. --- privval/signer_requestHandler.go | 11 +++++++---- test/e2e/app/main.go | 3 ++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/privval/signer_requestHandler.go b/privval/signer_requestHandler.go index ea020b682..682863b19 100644 --- a/privval/signer_requestHandler.go +++ b/privval/signer_requestHandler.go @@ -24,14 +24,17 @@ func DefaultValidationRequestHandler( switch r := req.Sum.(type) { case *privvalproto.Message_PubKeyRequest: if r.PubKeyRequest.GetChainId() != chainID { - res = mustWrapMsg(&privvalproto.SignedVoteResponse{ - Vote: tmproto.Vote{}, Error: &privvalproto.RemoteSignerError{ + res = mustWrapMsg(&privvalproto.PubKeyResponse{ + PubKey: cryptoproto.PublicKey{}, Error: &privvalproto.RemoteSignerError{ Code: 0, Description: "unable to provide pubkey"}}) return res, fmt.Errorf("want chainID: %s, got chainID: %s", r.PubKeyRequest.GetChainId(), chainID) } var pubKey crypto.PubKey pubKey, err = privVal.GetPubKey() + if err != nil { + return res, err + } pk, err := cryptoenc.PubKeyToProto(pubKey) if err != nil { return res, err @@ -64,8 +67,8 @@ func DefaultValidationRequestHandler( case *privvalproto.Message_SignProposalRequest: if r.SignProposalRequest.GetChainId() != chainID { - res = mustWrapMsg(&privvalproto.SignedVoteResponse{ - Vote: tmproto.Vote{}, Error: &privvalproto.RemoteSignerError{ + res = mustWrapMsg(&privvalproto.SignedProposalResponse{ + Proposal: tmproto.Proposal{}, Error: &privvalproto.RemoteSignerError{ Code: 0, Description: "unable to sign proposal"}}) return res, fmt.Errorf("want chainID: %s, got chainID: %s", r.SignProposalRequest.GetChainId(), chainID) diff --git a/test/e2e/app/main.go b/test/e2e/app/main.go index 53e053c4d..25d995e2c 100644 --- a/test/e2e/app/main.go +++ b/test/e2e/app/main.go @@ -12,6 +12,7 @@ import ( "github.com/tendermint/tendermint/abci/server" "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/crypto/ed25519" tmflags "github.com/tendermint/tendermint/libs/cli/flags" "github.com/tendermint/tendermint/libs/log" tmnet "github.com/tendermint/tendermint/libs/net" @@ -178,7 +179,7 @@ func startSigner(cfg *Config) error { var dialFn privval.SocketDialer switch protocol { case "tcp": - dialFn = privval.DialTCPFn(address, 3*time.Second, filePV.Key.PrivKey) + dialFn = privval.DialTCPFn(address, 3*time.Second, ed25519.GenPrivKey()) case "unix": dialFn = privval.DialUnixFn(address) default: From f2f6a788098bb1b39b0c572dd2a1c1f76d094d9e Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Tue, 17 Nov 2020 19:37:35 +0400 Subject: [PATCH 19/27] docs: warn developers about calling blocking funcs in Receive (#5679) Refs #2888 --- blockchain/v0/reactor.go | 2 ++ blockchain/v1/reactor.go | 2 ++ blockchain/v2/reactor.go | 2 ++ consensus/reactor.go | 2 ++ evidence/reactor.go | 2 ++ mempool/reactor.go | 2 ++ p2p/base_reactor.go | 3 +++ p2p/pex/pex_reactor.go | 2 ++ statesync/reactor.go | 2 ++ 9 files changed, 19 insertions(+) diff --git a/blockchain/v0/reactor.go b/blockchain/v0/reactor.go index 50f2e164e..dd3878669 100644 --- a/blockchain/v0/reactor.go +++ b/blockchain/v0/reactor.go @@ -204,6 +204,8 @@ func (bcR *BlockchainReactor) respondToPeer(msg *bcproto.BlockRequest, } // Receive implements Reactor by handling 4 types of messages (look below). +// XXX: do not call any methods that can block or incur heavy processing. +// https://github.com/tendermint/tendermint/issues/2888 func (bcR *BlockchainReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) { msg, err := bc.DecodeMsg(msgBytes) if err != nil { diff --git a/blockchain/v1/reactor.go b/blockchain/v1/reactor.go index c16139153..78ce71e18 100644 --- a/blockchain/v1/reactor.go +++ b/blockchain/v1/reactor.go @@ -250,6 +250,8 @@ func (bcR *BlockchainReactor) RemovePeer(peer p2p.Peer, reason interface{}) { } // Receive implements Reactor by handling 4 types of messages (look below). +// XXX: do not call any methods that can block or incur heavy processing. +// https://github.com/tendermint/tendermint/issues/2888 func (bcR *BlockchainReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) { msg, err := bc.DecodeMsg(msgBytes) if err != nil { diff --git a/blockchain/v2/reactor.go b/blockchain/v2/reactor.go index 0c43db179..5da117c94 100644 --- a/blockchain/v2/reactor.go +++ b/blockchain/v2/reactor.go @@ -455,6 +455,8 @@ func (r *BlockchainReactor) Stop() error { } // Receive implements Reactor by handling different message types. +// XXX: do not call any methods that can block or incur heavy processing. +// https://github.com/tendermint/tendermint/issues/2888 func (r *BlockchainReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) { msg, err := bc.DecodeMsg(msgBytes) if err != nil { diff --git a/consensus/reactor.go b/consensus/reactor.go index c4a2ef9fd..9fb41c37c 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -220,6 +220,8 @@ func (conR *Reactor) RemovePeer(peer p2p.Peer, reason interface{}) { // Peer state updates can happen in parallel, but processing of // proposals, block parts, and votes are ordered by the receiveRoutine // NOTE: blocks on consensus state for proposals, block parts, and votes +// XXX: do not call any methods that can block or incur heavy processing. +// https://github.com/tendermint/tendermint/issues/2888 func (conR *Reactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) { if !conR.IsRunning() { conR.Logger.Debug("Receive", "src", src, "chId", chID, "bytes", msgBytes) diff --git a/evidence/reactor.go b/evidence/reactor.go index 421e4bc18..951d64d71 100644 --- a/evidence/reactor.go +++ b/evidence/reactor.go @@ -66,6 +66,8 @@ func (evR *Reactor) AddPeer(peer p2p.Peer) { // Receive implements Reactor. // It adds any received evidence to the evpool. +// XXX: do not call any methods that can block or incur heavy processing. +// https://github.com/tendermint/tendermint/issues/2888 func (evR *Reactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) { evis, err := decodeMsg(msgBytes) if err != nil { diff --git a/mempool/reactor.go b/mempool/reactor.go index b4f76b9e8..6bf0ce7d7 100644 --- a/mempool/reactor.go +++ b/mempool/reactor.go @@ -160,6 +160,8 @@ func (memR *Reactor) RemovePeer(peer p2p.Peer, reason interface{}) { // Receive implements Reactor. // It adds any received transactions to the mempool. +// XXX: do not call any methods that can block or incur heavy processing. +// https://github.com/tendermint/tendermint/issues/2888 func (memR *Reactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) { msg, err := memR.decodeMsg(msgBytes) if err != nil { diff --git a/p2p/base_reactor.go b/p2p/base_reactor.go index 86b0d980a..59faf4c3f 100644 --- a/p2p/base_reactor.go +++ b/p2p/base_reactor.go @@ -44,6 +44,9 @@ type Reactor interface { // copying. // // CONTRACT: msgBytes are not nil. + // + // XXX: do not call any methods that can block or incur heavy processing. + // https://github.com/tendermint/tendermint/issues/2888 Receive(chID byte, peer Peer, msgBytes []byte) } diff --git a/p2p/pex/pex_reactor.go b/p2p/pex/pex_reactor.go index 71b71b054..fc701bfdd 100644 --- a/p2p/pex/pex_reactor.go +++ b/p2p/pex/pex_reactor.go @@ -236,6 +236,8 @@ func (r *Reactor) logErrAddrBook(err error) { } // Receive implements Reactor by handling incoming PEX messages. +// XXX: do not call any methods that can block or incur heavy processing. +// https://github.com/tendermint/tendermint/issues/2888 func (r *Reactor) Receive(chID byte, src Peer, msgBytes []byte) { msg, err := decodeMsg(msgBytes) if err != nil { diff --git a/statesync/reactor.go b/statesync/reactor.go index 4f4310f84..0abd91cf1 100644 --- a/statesync/reactor.go +++ b/statesync/reactor.go @@ -90,6 +90,8 @@ func (r *Reactor) RemovePeer(peer p2p.Peer, reason interface{}) { } // Receive implements p2p.Reactor. +// XXX: do not call any methods that can block or incur heavy processing. +// https://github.com/tendermint/tendermint/issues/2888 func (r *Reactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) { if !r.IsRunning() { return From e3728e7709d731c2f73f95859a9a24630a86413e Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Wed, 18 Nov 2020 15:27:47 +0100 Subject: [PATCH 20/27] p2p: remove unused MakePoWTarget() (#5684) --- CHANGELOG_PENDING.md | 1 + p2p/key.go | 26 -------------------------- p2p/key_test.go | 28 ---------------------------- 3 files changed, 1 insertion(+), 54 deletions(-) diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index fe066695c..6560cdf59 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -16,6 +16,7 @@ Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermi - P2P Protocol - Go API + - [p2p] Removed unused function `MakePoWTarget`. (@erikgrinaker) - [libs/os] Kill() and {Must,}{Read,Write}File() functions have been removed. (@alessio) diff --git a/p2p/key.go b/p2p/key.go index f0f3dfd03..57cdd4de1 100644 --- a/p2p/key.go +++ b/p2p/key.go @@ -1,9 +1,7 @@ package p2p import ( - "bytes" "encoding/hex" - "fmt" "io/ioutil" "github.com/tendermint/tendermint/crypto" @@ -94,27 +92,3 @@ func (nodeKey *NodeKey) SaveAs(filePath string) error { } return nil } - -//------------------------------------------------------------------------------ - -// MakePoWTarget returns the big-endian encoding of 2^(targetBits - difficulty) - 1. -// It can be used as a Proof of Work target. -// NOTE: targetBits must be a multiple of 8 and difficulty must be less than targetBits. -func MakePoWTarget(difficulty, targetBits uint) []byte { - if targetBits%8 != 0 { - panic(fmt.Sprintf("targetBits (%d) not a multiple of 8", targetBits)) - } - if difficulty >= targetBits { - panic(fmt.Sprintf("difficulty (%d) >= targetBits (%d)", difficulty, targetBits)) - } - targetBytes := targetBits / 8 - zeroPrefixLen := (int(difficulty) / 8) - prefix := bytes.Repeat([]byte{0}, zeroPrefixLen) - mod := (difficulty % 8) - if mod > 0 { - nonZeroPrefix := byte(1<<(8-mod) - 1) - prefix = append(prefix, nonZeroPrefix) - } - tailLen := int(targetBytes) - len(prefix) - return append(prefix, bytes.Repeat([]byte{0xFF}, tailLen)...) -} diff --git a/p2p/key_test.go b/p2p/key_test.go index 7593df239..22cccf743 100644 --- a/p2p/key_test.go +++ b/p2p/key_test.go @@ -1,7 +1,6 @@ package p2p import ( - "bytes" "os" "path/filepath" "testing" @@ -52,30 +51,3 @@ func TestNodeKeySaveAs(t *testing.T) { assert.NoError(t, err) assert.FileExists(t, filePath) } - -//---------------------------------------------------------- - -func padBytes(bz []byte, targetBytes int) []byte { - return append(bz, bytes.Repeat([]byte{0xFF}, targetBytes-len(bz))...) -} - -func TestPoWTarget(t *testing.T) { - - targetBytes := 20 - cases := []struct { - difficulty uint - target []byte - }{ - {0, padBytes([]byte{}, targetBytes)}, - {1, padBytes([]byte{127}, targetBytes)}, - {8, padBytes([]byte{0}, targetBytes)}, - {9, padBytes([]byte{0, 127}, targetBytes)}, - {10, padBytes([]byte{0, 63}, targetBytes)}, - {16, padBytes([]byte{0, 0}, targetBytes)}, - {17, padBytes([]byte{0, 0, 127}, targetBytes)}, - } - - for _, c := range cases { - assert.Equal(t, MakePoWTarget(c.difficulty, 20*8), c.target) - } -} From b435d9aae5295f778eaad3df8cdf822591ecf9ea Mon Sep 17 00:00:00 2001 From: Tess Rinearson Date: Wed, 18 Nov 2020 18:10:43 +0100 Subject: [PATCH 21/27] upgrading: update 0.34 instructions with updates since RC4 (#5685) Co-authored-by: Marko --- UPGRADING.md | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/UPGRADING.md b/UPGRADING.md index 7152a9400..065e9822d 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -8,8 +8,12 @@ This guide provides instructions for upgrading to specific versions of Tendermin This release is not compatible with previous blockchains due to changes to the encoding format (see "Protocol Buffers," below) and the block header (see "Blockchain Protocol"). +Note also that Tendermint 0.34 also requires Go 1.15 or higher. + ### ABCI Changes +* The `ABCIVersion` is now `0.17.0`. + * New ABCI methods (`ListSnapshots`, `LoadSnapshotChunk`, `OfferSnapshot`, and `ApplySnapshotChunk`) were added to support the new State Sync feature. Previously, syncing a new node to a preexisting network could take days; but with State Sync, @@ -61,7 +65,7 @@ fields. Tendermint now relies on the application to tell it which transactions to index. This means that in the `config.toml`, generated by Tendermint, there is no longer a way to specify which -transactions to index. `tx.height` & `tx.hash` will always be indexed when using the `kv` indexer. +transactions to index. `tx.height` and `tx.hash` will always be indexed when using the `kv` indexer. Applications must now choose to either a) enable indexing for all transactions, or b) allow node operators to decide which transactions to index. @@ -107,7 +111,7 @@ Tendermint 0.34 includes new and updated consensus parameters. #### Evidence Parameters -* `MaxNum`, which caps the total amount of evidence by a absolute number. The default is 50. +* `MaxBytes`, which caps the total amount of evidence. The default is 1048576 (1 MB). ### Crypto @@ -168,16 +172,31 @@ Other user-relevant changes include: ### `privval` Package All requests are now accompanied by the chain ID from the network. -This is a optional field and can be ignored by key management systems. -It is recommended to check the chain ID if using the same key management system for multiple chains. +This is a optional field and can be ignored by key management systems; +however, if you are using the same key management system for multiple different +blockchains, we recommend that you check the chain ID. + ### RPC -`/unsafe_start_cpu_profiler`, `/unsafe_stop_cpu_profiler` and -`/unsafe_write_heap_profile` were removed. -For profiling, please use the pprof server, which can -be enabled through `--rpc.pprof_laddr=X` flag or `pprof_laddr=X` config setting -in the rpc section. +* `/unsafe_start_cpu_profiler`, `/unsafe_stop_cpu_profiler` and + `/unsafe_write_heap_profile` were removed. + For profiling, please use the pprof server, which can + be enabled through `--rpc.pprof_laddr=X` flag or `pprof_laddr=X` config setting + in the rpc section. +* The `Content-Type` header returned on RPC calls is now (correctly) set as `application/json`. + +### Version + +Version is now set through Go linker flags `ld_flags`. Applications that are using tendermint as a library should set this at compile time. + +Example: + +```sh +go install -mod=readonly -ldflags "-X github.com/tendermint/tendermint/version.TMCoreSemVer=$(go list -m github.com/tendermint/tendermint | sed 's/ /\@/g') -s -w " -trimpath ./cmd +``` + +Additionally, the exported constant `version.Version` is now `version.TMCoreSemVer`. ## v0.33.4 From 68dc751a8c681eeb86c5178f3cd8c2e07fb761eb Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Thu, 19 Nov 2020 12:02:58 +0100 Subject: [PATCH 22/27] light: ensure required header fields are present for verification (#5677) --- light/verifier.go | 107 ++++++++++++++++++++++++++++------------------ 1 file changed, 66 insertions(+), 41 deletions(-) diff --git a/light/verifier.go b/light/verifier.go index 3c2eaca22..0ea4c7332 100644 --- a/light/verifier.go +++ b/light/verifier.go @@ -29,6 +29,7 @@ var ( // // maxClockDrift defines how much untrustedHeader.Time can drift into the // future. +// trustedHeader must have a ChainID, Height and Time func VerifyNonAdjacent( trustedHeader *types.SignedHeader, // height=X trustedVals *types.ValidatorSet, // height=X or height=X+1 @@ -39,6 +40,8 @@ func VerifyNonAdjacent( maxClockDrift time.Duration, trustLevel tmmath.Fraction) error { + checkRequiredHeaderFields(trustedHeader) + if untrustedHeader.Height == trustedHeader.Height+1 { return errors.New("headers must be non adjacent in height") } @@ -90,6 +93,7 @@ func VerifyNonAdjacent( // // maxClockDrift defines how much untrustedHeader.Time can drift into the // future. +// trustedHeader must have a ChainID, Height, Time and NextValidatorsHash func VerifyAdjacent( trustedHeader *types.SignedHeader, // height=X untrustedHeader *types.SignedHeader, // height=X+1 @@ -98,6 +102,12 @@ func VerifyAdjacent( now time.Time, maxClockDrift time.Duration) error { + checkRequiredHeaderFields(trustedHeader) + + if len(trustedHeader.NextValidatorsHash) == 0 { + panic("next validators hash in trusted header is empty") + } + if untrustedHeader.Height != trustedHeader.Height+1 { return errors.New("headers must be adjacent in height") } @@ -150,47 +160,6 @@ func Verify( return VerifyAdjacent(trustedHeader, untrustedHeader, untrustedVals, trustingPeriod, now, maxClockDrift) } -func verifyNewHeaderAndVals( - untrustedHeader *types.SignedHeader, - untrustedVals *types.ValidatorSet, - trustedHeader *types.SignedHeader, - now time.Time, - maxClockDrift time.Duration) error { - - if err := untrustedHeader.ValidateBasic(trustedHeader.ChainID); err != nil { - return fmt.Errorf("untrustedHeader.ValidateBasic failed: %w", err) - } - - if untrustedHeader.Height <= trustedHeader.Height { - return fmt.Errorf("expected new header height %d to be greater than one of old header %d", - untrustedHeader.Height, - trustedHeader.Height) - } - - if !untrustedHeader.Time.After(trustedHeader.Time) { - return fmt.Errorf("expected new header time %v to be after old header time %v", - untrustedHeader.Time, - trustedHeader.Time) - } - - if !untrustedHeader.Time.Before(now.Add(maxClockDrift)) { - return fmt.Errorf("new header has a time from the future %v (now: %v; max clock drift: %v)", - untrustedHeader.Time, - now, - maxClockDrift) - } - - if !bytes.Equal(untrustedHeader.ValidatorsHash, untrustedVals.Hash()) { - return fmt.Errorf("expected new header validators (%X) to match those that were supplied (%X) at height %d", - untrustedHeader.ValidatorsHash, - untrustedVals.Hash(), - untrustedHeader.Height, - ) - } - - return nil -} - // ValidateTrustLevel checks that trustLevel is within the allowed range [1/3, // 1]. If not, it returns an error. 1/3 is the minimum amount of trust needed // which does not break the security model. @@ -243,3 +212,59 @@ func VerifyBackwards(untrustedHeader, trustedHeader *types.Header) error { return nil } + +func verifyNewHeaderAndVals( + untrustedHeader *types.SignedHeader, + untrustedVals *types.ValidatorSet, + trustedHeader *types.SignedHeader, + now time.Time, + maxClockDrift time.Duration) error { + + if err := untrustedHeader.ValidateBasic(trustedHeader.ChainID); err != nil { + return fmt.Errorf("untrustedHeader.ValidateBasic failed: %w", err) + } + + if untrustedHeader.Height <= trustedHeader.Height { + return fmt.Errorf("expected new header height %d to be greater than one of old header %d", + untrustedHeader.Height, + trustedHeader.Height) + } + + if !untrustedHeader.Time.After(trustedHeader.Time) { + return fmt.Errorf("expected new header time %v to be after old header time %v", + untrustedHeader.Time, + trustedHeader.Time) + } + + if !untrustedHeader.Time.Before(now.Add(maxClockDrift)) { + return fmt.Errorf("new header has a time from the future %v (now: %v; max clock drift: %v)", + untrustedHeader.Time, + now, + maxClockDrift) + } + + if !bytes.Equal(untrustedHeader.ValidatorsHash, untrustedVals.Hash()) { + return fmt.Errorf("expected new header validators (%X) to match those that were supplied (%X) at height %d", + untrustedHeader.ValidatorsHash, + untrustedVals.Hash(), + untrustedHeader.Height, + ) + } + + return nil +} + +func checkRequiredHeaderFields(h *types.SignedHeader) { + if h.Height == 0 { + panic("height in trusted header must be set (non zero") + } + + zeroTime := time.Time{} + if h.Time == zeroTime { + panic("time in trusted header must be set") + } + + if h.ChainID == "" { + panic("chain ID in trusted header must be set") + } +} From f97a498cee4818da68dc1c1dac3215f2e5f25311 Mon Sep 17 00:00:00 2001 From: Tess Rinearson Date: Thu, 19 Nov 2020 14:23:08 +0100 Subject: [PATCH 23/27] scripts: make linkifier default to 'pull' rather than 'issue' (#5689) --- scripts/linkify_changelog.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/linkify_changelog.py b/scripts/linkify_changelog.py index 16647c05f..bc446c769 100644 --- a/scripts/linkify_changelog.py +++ b/scripts/linkify_changelog.py @@ -3,11 +3,11 @@ import re # This script goes through the provided file, and replaces any " \#", # with the valid mark down formatted link to it. e.g. -# " [\#number](https://github.com/tendermint/tendermint/issues/) -# Note that if the number is for a PR, github will auto-redirect you when you click the link. +# " [\#number](https://github.com/tendermint/tendermint/pull/) +# Note that if the number is for a an issue, github will auto-redirect you when you click the link. # It is safe to run the script multiple times in succession. # # Example usage $ python3 linkify_changelog.py ../CHANGELOG_PENDING.md for line in fileinput.input(inplace=1): - line = re.sub(r"\s\\#([0-9]*)", r" [\\#\1](https://github.com/tendermint/tendermint/issues/\1)", line.rstrip()) + line = re.sub(r"\s\\#([0-9]*)", r" [\\#\1](https://github.com/tendermint/tendermint/pull/\1)", line.rstrip()) print(line) \ No newline at end of file From 3246283cf2011200c801d7f78380592acbe8b733 Mon Sep 17 00:00:00 2001 From: Tess Rinearson Date: Thu, 19 Nov 2020 16:20:30 +0100 Subject: [PATCH 24/27] .vscode: remove directory (#5626) --- .vscode/settings.json | 8 -------- CONTRIBUTING.md | 15 +++++++++++++++ 2 files changed, 15 insertions(+), 8 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 3a42e525f..000000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "protoc": { - "options": [ - "--proto_path=${workspaceRoot}/proto", - "--proto_path=${workspaceRoot}/third_party/proto" - ] - } -} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d94ecb137..54e0489c8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -127,6 +127,21 @@ make install You should now be able to run `make proto-gen` from inside the root Tendermint directory to generate new files from proto files. +### Visual Studio Code + +If you are a VS Code user, you may want to add the following to your `.vscode/settings.json`: + +```json +{ + "protoc": { + "options": [ + "--proto_path=${workspaceRoot}/proto", + "--proto_path=${workspaceRoot}/third_party/proto" + ] + } +} +``` + ## Changelog Every fix, improvement, feature, or breaking change should be made in a From 64101f5ac9901be47de12d99a8c9b0e3c33e1df7 Mon Sep 17 00:00:00 2001 From: Tess Rinearson Date: Thu, 19 Nov 2020 16:34:38 +0100 Subject: [PATCH 25/27] .gitignore: sort (#5690) --- .gitignore | 65 ++++++++++++++++++++++++------------------------------ 1 file changed, 29 insertions(+), 36 deletions(-) diff --git a/.gitignore b/.gitignore index 3aa5112d8..d5442f35d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,50 +1,43 @@ -*.swp -*.swo -.bak *.bak +*.iml +*.log +*.swo +*.swp +*/.glide +*/vendor .DS_Store -build/* -artifacts/* -rpc/test/.tendermint -.tendermint -remote_dump +.bak +.idea/ .revision -vendor +.tendermint +.tendermint-lite +.terraform .vagrant -test/e2e/build -test/maverick/maverick -test/e2e/networks/*/ -test/p2p/data/ -test/logs +.vendor-new/ +.vscode/ +abci-cli +addrbook.json +artifacts/* +build/* coverage.txt +docs/.vuepress/dist docs/_build docs/dist -docs/.vuepress/dist -*.log -abci-cli docs/node_modules/ index.html.md - -scripts/wal2json/wal2json -scripts/cutWALUntil/cutWALUntil - -.idea/ -*.iml - -.vscode/ - libs/pubsub/query/fuzz_test/output +profile\.out +remote_dump +rpc/test/.tendermint +scripts/cutWALUntil/cutWALUntil +scripts/wal2json/wal2json shunit2 - -.tendermint-lite -addrbook.json - -*/vendor -.vendor-new/ -*/.glide -.terraform terraform.tfstate terraform.tfstate.backup terraform.tfstate.d - -profile\.out +test/e2e/build +test/e2e/networks/*/ +test/logs +test/maverick/maverick +test/p2p/data/ +vendor \ No newline at end of file From 6bee97160ff81fdf9261962a5cfc4ee295646ea8 Mon Sep 17 00:00:00 2001 From: Tess Rinearson Date: Thu, 19 Nov 2020 17:44:08 +0100 Subject: [PATCH 26/27] changelog: squash changelog from 0.34 RCs into one (#5691) "Squashes" the changelog from RCs 2-6 into one changelog message for 0.34.0, and adds the changelog pending. --- CHANGELOG.md | 354 +++++++++++++++---------------------------- CHANGELOG_PENDING.md | 3 +- 2 files changed, 119 insertions(+), 238 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ab1eabd1..fe534485a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,302 +1,184 @@ # Changelog -## v0.34.0-rc6 +## v0.34.0 -*November 5, 2020* +*November 19, 2020* -Friendly reminder: We have a [bug bounty program](https://hackerone.com/tendermint). - -### BREAKING CHANGES - -- [evidence] [\#5610](https://github.com/tendermint/tendermint/issues/5610) Make it possible for abci evidence to be formed from tm evidence (@cmwaters) - - This makes breaking changes to `DuplicateVoteEvidence` and `LightClientAttackEvidence` - such that ABCI evidence can be reconstructed from these types, and is necessary for nodes - to successfully sync to block heights with recorded evidence -- [version] [\#5527](https://github.com/tendermint/tendermint/pull/5527) We've updated our release tooling, and `version.Version` is now `version.TMCoreSemVer` (@marbar3778) - -### IMPROVEMENTS - -- [statesync] [\#5516](https://github.com/tendermint/tendermint/issues/5516) Check that all heights necessary to rebuild state for a snapshot exist before adding the snapshot to the pool. (@erikgrinaker) - -### BUG FIXES - -- [blockchain/v2] [\#5499](https://github.com/tendermint/tendermint/issues/5499) Fix "duplicate block enqueued by processor" panic (@melekes) -- [abci/grpc] [\#5520](https://github.com/tendermint/tendermint/issues/5520) Return async responses in order, to avoid mempool panics. (@erikgrinaker) -- [blockchain/v2] [\#5530](https://github.com/tendermint/tendermint/issues/5530) Fix "processed height 4541 but expected height 4540" panic (@melekes) -- [consensus/wal] Fix WAL autorepair by opening target WAL in read/write mode (@erikgrinaker) -- [block] [\#5567](https://github.com/tendermint/tendermint/issues/5567) Fix MaxCommitSigBytes (@cmwaters) -- [blockchain/v2] [\#5553](https://github.com/tendermint/tendermint/issues/5553) Make the removal of an already removed peer a noop (@melekes) -- [evidence] [\#5574](https://github.com/tendermint/tendermint/issues/5574) Fix bug where node sends committed evidence to peer (@cmwaters) -- [privval] [\#5583](https://github.com/tendermint/tendermint/issues/5583) Make `Vote`, `Proposal` & `PubKey` non-nullable in Responses (@marbar3778) -- [light] [\#5392](https://github.com/tendermint/tendermint/issues/5392) Make light client home directory same as the full node default (@cmwaters) - - -## v0.34.0-rc5 - -*October 13, 2020* - -Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermint). - -### BREAKING CHANGES - -- CLI/RPC/Config - -- Apps - - [ABCI] \#5447 Remove `SetOption` method from `ABCI.Client` interface - -- P2P Protocol - -- Go API - - [evidence] [\#5499](https://github.com/tendermint/tendermint/pull/5449) `MaxNum` evidence consensus parameter has been changed to `MaxBytes` (@cmwaters) - -- Blockchain Protocol - -### FEATURES - -### IMPROVEMENTS - -- [privval] \#5434 `NewSignerDialerEndpoint` can now be given `SignerServiceEndpointOption` (@erikgrinaker) - -- [config] \#5433 `statesync.rpc_servers` is now properly set when writing the configuration file (@erikgrinaker) - -### BUG FIXES - -- [privval] \#5441 Fix faulty ping message encoding causing nil message errors in logs (@erikgrinaker) - -## v0.34.0-rc4 - -*September 24, 2020* - -Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermint). - -### BREAKING CHANGES - -- CLI/RPC/Config - - [config] [\#5315](https://github.com/tendermint/tendermint/issues/5315) Rename `prof_laddr` to `pprof_laddr` and move it to `rpc` section (@melekes) - - [rpc] [\#5315](https://github.com/tendermint/tendermint/issues/5315) Remove `/unsafe_start_cpu_profiler`, `/unsafe_stop_cpu_profiler` and `/unsafe_write_heap_profile`. Please use pprof functionality instead (@melekes) - - [rpc/client, rpc/jsonrpc/client] [\#5347](https://github.com/tendermint/tendermint/issues/5347) All client methods now accept `context.Context` as 1st param (@melekes) - -- Apps - - [abci] [\#5324](https://github.com/tendermint/tendermint/pull/5324) abci evidence type is an enum with two types of possible evidence (@cmwaters) - -- P2P Protocol - - [mempool] [\#5321](https://github.com/tendermint/tendermint/issues/5321) Batch transactions when broadcasting them to peers (@melekes) `MaxBatchBytes` new config setting defines the max size of one batch. - -- Go API - - [evidence] [\#5317](https://github.com/tendermint/tendermint/issues/5317) Remove ConflictingHeaders evidence type & CompositeEvidence Interface. (@marbar3778) - - [evidence] [\#5318](https://github.com/tendermint/tendermint/issues/5318) Remove LunaticValidator evidence type. (@marbar3778) - - [evidence] [\#5319](https://github.com/tendermint/tendermint/issues/5319) Remove Amnesia & potentialAmnesia evidence types and removed POLC. (@marbar3778) - - [evidence] [\#5361](https://github.com/tendermint/tendermint/pull/5361) Add LightClientAttackEvidence and change evidence interface (@cmwaters) - - [params] [\#5319](https://github.com/tendermint/tendermint/issues/5319) Remove `ProofofTrialPeriod` from evidence params (@marbar3778) - - [light] [\#5347](https://github.com/tendermint/tendermint/issues/5347) `NewClient`, `NewHTTPClient`, `VerifyHeader` and `VerifyLightBlockAtHeight` now accept `context.Context` as 1st param (@melekes) - - [state] [\#5348](https://github.com/tendermint/tendermint/issues/5348) Define an Interface for the state store. (@marbar3778) - -### FEATURES - -- [privval] [\#5239](https://github.com/tendermint/tendermint/issues/5239) Add `chainID` to requests from client. (@marbar3778) -- [config] [\#5147](https://github.com/tendermint/tendermint/issues/5147) Add `--consensus.double_sign_check_height` flag and `DoubleSignCheckHeight` config variable. See [ADR-51](https://github.com/tendermint/tendermint/blob/master/docs/architecture/adr-051-double-signing-risk-reduction.md) -- [light] [\#5298](https://github.com/tendermint/tendermint/pull/5298) Morph validator set and signed header into light block (@cmwaters) -- [evidence] [\#5361](https://github.com/tendermint/tendermint/pull/5361) Add LightClientAttackEvidence and refactor evidence lifecycle (@cmwaters) - -### IMPROVEMENTS - -- [blockchain] [\#5278](https://github.com/tendermint/tendermint/issues/5278) Verify only +2/3 of the signatures in a block when fast syncing. (@marbar3778) -- [rpc] [\#5293](https://github.com/tendermint/tendermint/issues/5293) `/dial_peers` has added `private` and `unconditional` as parameters. (@marbar3778) -- [types] [\#5340](https://github.com/tendermint/tendermint/issues/5340) Add check in `Header.ValidateBasic()` for block protocol version (@marbar3778) -- [statesync] [\#5399](https://github.com/tendermint/tendermint/issues/5399) Add `discovery_time` configuration setting, and reduce default to 15s. (@erikgrinaker) - -### BUG FIXES - -- [blockchain] [\#5249](https://github.com/tendermint/tendermint/issues/5249) Fix fast sync halt with initial height > 1 (@erikgrinaker) -- [statesync] [\#5302](https://github.com/tendermint/tendermint/issues/5302) Fix genesis state propagation to state sync routine (@erikgrinaker) -- [statesync] [\#5320](https://github.com/tendermint/tendermint/issues/5320) Broadcast snapshot request to all pre-connected peers on start (@erikgrinaker) -- [consensus] [\#5329](https://github.com/tendermint/tendermint/issues/5329) Fix wrong proposer schedule for validators returned by `InitChain` (@erikgrinaker) -- [store] [\#5382](https://github.com/tendermint/tendermint/issues/5382) Fix race conditions when loading/saving/pruning blocks (@erikgrinaker) -- [light] [\#5307](https://github.com/tendermint/tendermint/pull/5307) Persist correct proposer priority in light client validator sets (@cmwaters) -- [docker] [\#5385](https://github.com/tendermint/tendermint/issues/5385) Fix incorrect `time_iota_ms` default setting causing block timestamp drift (@erikgrinaker) -- [abci] [\#5395](https://github.com/tendermint/tendermint/issues/5395) Fix socket client error for state sync responses (@erikgrinaker) - - -## v0.34.0-rc3 - -*August 13, 2020* - -Special thanks to external contributors on this release: @SadPencil - -Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermint). - -### BREAKING CHANGES: - -- Blockchain Protocol - - [\#5193](https://github.com/tendermint/tendermint/pull/5193) Header hashes are no longer empty for empty inputs, notably `DataHash`, `EvidenceHash`, and `LastResultsHash` (@erikgrinaker) - -- Go API - - [evidence] [\#5181](https://github.com/tendermint/tendermint/pull/5181) Phantom validator evidence was removed (@cmwaters) - - [merkle] [\#5193](https://github.com/tendermint/tendermint/pull/5193) `HashFromByteSlices` and `ProofsFromByteSlices` now return a hash for empty inputs, following RFC6962 (@erikgrinaker) - - [crypto] [\#5214](https://github.com/tendermint/tendermint/issues/5214) Change `GenPrivKeySecp256k1` to `GenPrivKeyFromSecret` to be consistent with other keys (@marbar3778) - - [state] [\#5191](https://github.com/tendermint/tendermint/pull/5191) Add `State.InitialHeight` field to record initial block height, must be `1` (not `0`) to start from 1 (@erikgrinaker) - - [state] [\#5231](https://github.com/tendermint/tendermint/issues/5231) `LoadStateFromDBOrGenesisFile()` and `LoadStateFromDBOrGenesisDoc()` no longer saves the state in the database if not found, the genesis state is simply returned (@erikgrinaker) - - [crypto] [\#5236](https://github.com/tendermint/tendermint/issues/5236) `VerifyBytes` is now `VerifySignature` on the `crypto.PubKey` interface (@marbar3778) - -### FEATURES: - -- [abci] [\#5174](https://github.com/tendermint/tendermint/pull/5174) Add amnesia evidence and remove mock and potential amnesia evidence from abci (@cmwaters) -- [abci] [\#5191](https://github.com/tendermint/tendermint/pull/5191) Add `InitChain.InitialHeight` field giving the initial block height (@erikgrinaker) -- [abci] [\#5227](https://github.com/tendermint/tendermint/pull/5227) Add `ResponseInitChain.app_hash` which is recorded in genesis block (@erikgrinaker) -- [genesis] [\#5191](https://github.com/tendermint/tendermint/pull/5191) Add `initial_height` field to specify the initial chain height (defaults to `1`) (@erikgrinaker) -- [db] [\#5233](https://github.com/tendermint/tendermint/issues/5233) Add support for `badgerdb` database backend (@erikgrinaker) - -### IMPROVEMENTS: - -- [evidence] [\#5219](https://github.com/tendermint/tendermint/pull/5219) Change the source of evidence time to block time (@cmwaters) - -### BUG FIXES: - -- [evidence] [\#5170](https://github.com/tendermint/tendermint/pull/5170) change abci evidence time to the time the infraction happened not the time the evidence was committed on the block (@cmwaters) -- [node] [\#5211](https://github.com/tendermint/tendermint/issues/5211) Don't attempt fast sync when the ABCI application specifies ourself as the only validator via `InitChain` (@erikgrinaker) -- [libs/rand] [\#5215](https://github.com/tendermint/tendermint/pull/5215) Fix out-of-memory error on unexpected argument of Str() (@SadPencil) - - -## v0.34.0-rc2 - -*July 30, 2020* +Holy smokes, this is a big one! For a more reader-friendly overview of the changes in 0.34.0 +(and of the changes you need to accommodate as a user), check out [UPGRADING.md](UPGRADING.md). Special thanks to external contributors on this release: @james-ray, @fedekunze, @favadi, @alessio, -@joe-bowman, @cuonglm +@joe-bowman, @cuonglm, @SadPencil and @dongsam. -Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermint). +And as always, friendly reminder, that we have a [bug bounty program](https://hackerone.com/tendermint). -### BREAKING CHANGES: +### BREAKING CHANGES - CLI/RPC/Config - - [evidence] [\#4959](https://github.com/tendermint/tendermint/issues/4959) Add json tags to `DuplicateVoteEvidence` - - [light] [\#4946](https://github.com/tendermint/tendermint/issues/4946) `tendermint lite` cmd has been renamed to `tendermint light` - - [privval] [\#4582](https://github.com/tendermint/tendermint/issues/4582) `round` in private_validator_state.json is no longer a string in json it is now a number + - [config] [\#5315](https://github.com/tendermint/tendermint/pull/5315) Rename `prof_laddr` to `pprof_laddr` and move it to `rpc` section (@melekes) + - [evidence] [\#4959](https://github.com/tendermint/tendermint/pull/4959) Add JSON tags to `DuplicateVoteEvidence` (@marbar3778) + - [light] [\#4946](https://github.com/tendermint/tendermint/pull/4946) `tendermint lite` command has been renamed to `tendermint light` (@marbar3778) + - [privval] [\#4582](https://github.com/tendermint/tendermint/pull/4582) `round` in private_validator_state.json is no longer JSON string; instead it is a number (@marbar3778) - [rpc] [\#4792](https://github.com/tendermint/tendermint/pull/4792) `/validators` are now sorted by voting power (@melekes) - - [rpc] [\#4937](https://github.com/tendermint/tendermint/issues/4937) Return an error when `page` pagination param is 0 in `/validators`, `tx_search` (@melekes) - - [rpc] [\#5137](https://github.com/tendermint/tendermint/issues/5137) The json tags of `gasWanted` & `gasUsed` in `ResponseCheckTx` & `ResponseDeliverTx` have been made snake_case. (`gas_wanted` & `gas_used`) + - [rpc] [\#4947](https://github.com/tendermint/tendermint/pull/4947) Return an error when `page` pagination param is 0 in `/validators`, `tx_search` (@melekes) + - [rpc] [\#5137](https://github.com/tendermint/tendermint/pull/5137) JSON tags of `gasWanted` and `gasUsed` in `ResponseCheckTx` and `ResponseDeliverTx` have been made snake_case (`gas_wanted` and `gas_used`) (@marbar3778) + - [rpc] [\#5315](https://github.com/tendermint/tendermint/pull/5315) Remove `/unsafe_start_cpu_profiler`, `/unsafe_stop_cpu_profiler` and `/unsafe_write_heap_profile`. Please use pprof functionality instead (@melekes) + - [rpc/client, rpc/jsonrpc/client] [\#5347](https://github.com/tendermint/tendermint/pull/5347) All client methods now accept `context.Context` as 1st param (@melekes) - Apps - - [abci] [\#4704](https://github.com/tendermint/tendermint/pull/4704) Add ABCI methods `ListSnapshots`, `LoadSnapshotChunk`, `OfferSnapshot`, and `ApplySnapshotChunk` for state sync snapshots. `ABCIVersion` bumped to 0.17.0. - - [abci] [\#4989](https://github.com/tendermint/tendermint/issues/4989) `Proof` within `ResponseQuery` has been renamed to `ProofOps` - - [abci] `CheckTxType` Protobuf enum names are now uppercase, to follow Protobuf style guide + - [abci] [\#4704](https://github.com/tendermint/tendermint/pull/4704) Add ABCI methods `ListSnapshots`, `LoadSnapshotChunk`, `OfferSnapshot`, and `ApplySnapshotChunk` for state sync snapshots. `ABCIVersion` bumped to 0.17.0. (@erikgrinaker) + - [abci] [\#4989](https://github.com/tendermint/tendermint/pull/4989) `Proof` within `ResponseQuery` has been renamed to `ProofOps` (@marbar3778) + - [abci] [\#5096](https://github.com/tendermint/tendermint/pull/5096) `CheckTxType` Protobuf enum names are now uppercase, to follow Protobuf style guide (@erikgrinaker) + - [abci] [\#5324](https://github.com/tendermint/tendermint/pull/5324) ABCI evidence type is now an enum with two types of possible evidence (@cmwaters) - P2P Protocol - - [blockchain] [\#4637](https://github.com/tendermint/tendermint/issues/4637) Migrate blockchain reactor(s) to Protobuf encoding - - [evidence] [\#4949](https://github.com/tendermint/tendermint/issues/4949) Migrate evidence reactor to Protobuf encoding - - [mempool] [\#4940](https://github.com/tendermint/tendermint/issues/4940) Migrate mempool from to Protobuf encoding - - [p2p/pex] [\#4973](https://github.com/tendermint/tendermint/issues/4973) Migrate `p2p/pex` reactor to Protobuf encoding - - [statesync] [\#4943](https://github.com/tendermint/tendermint/issues/4943) Migrate state sync reactor to Protobuf encoding + - [blockchain] [\#4637](https://github.com/tendermint/tendermint/pull/4637) Migrate blockchain reactor(s) to Protobuf encoding (@marbar3778) + - [evidence] [\#4949](https://github.com/tendermint/tendermint/pull/4949) Migrate evidence reactor to Protobuf encoding (@marbar3778) + - [mempool] [\#4940](https://github.com/tendermint/tendermint/pull/4940) Migrate mempool from to Protobuf encoding (@marbar3778) + - [mempool] [\#5321](https://github.com/tendermint/tendermint/pull/5321) Batch transactions when broadcasting them to peers (@melekes) + - `MaxBatchBytes` new config setting defines the max size of one batch. + - [p2p/pex] [\#4973](https://github.com/tendermint/tendermint/pull/4973) Migrate `p2p/pex` reactor to Protobuf encoding (@marbar3778) + - [statesync] [\#4943](https://github.com/tendermint/tendermint/pull/4943) Migrate state sync reactor to Protobuf encoding (@marbar3778) - Blockchain Protocol - - [evidence] [\#4780](https://github.com/tendermint/tendermint/pull/4780) Cap evidence to an absolute number (@cmwaters) - - Add `max_num` to consensus evidence parameters (default: 50 items). - - [evidence] [\#4725](https://github.com/tendermint/tendermint/issues/4725) Remove `Pubkey` from `DuplicateVoteEvidence` - - [state] [\#4845](https://github.com/tendermint/tendermint/issues/4845) Include `GasWanted` and `GasUsed` into `LastResultsHash` (@melekes) + - [evidence] [\#4725](https://github.com/tendermint/tendermint/pull/4725) Remove `Pubkey` from `DuplicateVoteEvidence` (@marbar3778) + - [evidence] [\#5499](https://github.com/tendermint/tendermint/pull/5449) Cap evidence to a maximum number of bytes (supercedes [\#4780](https://github.com/tendermint/tendermint/pull/4780)) (@cmwaters) + - [merkle] [\#5193](https://github.com/tendermint/tendermint/pull/5193) Header hashes are no longer empty for empty inputs, notably `DataHash`, `EvidenceHash`, and `LastResultsHash` (@erikgrinaker) + - [state] [\#4845](https://github.com/tendermint/tendermint/pull/4845) Include `GasWanted` and `GasUsed` into `LastResultsHash` (@melekes) - [types] [\#4792](https://github.com/tendermint/tendermint/pull/4792) Sort validators by voting power to enable faster commit verification (@melekes) - On-disk serialization - - [state] [\#4679](https://github.com/tendermint/tendermint/issues/4679) Migrate state module to Protobuf encoding + - [state] [\#4679](https://github.com/tendermint/tendermint/pull/4679) Migrate state module to Protobuf encoding (@marbar3778) - `BlockStoreStateJSON` is now `BlockStoreState` and is encoded as binary in the database - - [store] [\#4778](https://github.com/tendermint/tendermint/issues/4778) Migrate store module to Protobuf encoding + - [store] [\#4778](https://github.com/tendermint/tendermint/pull/4778) Migrate store module to Protobuf encoding (@marbar3778) - Light client, private validator - - [light] [\#4964](https://github.com/tendermint/tendermint/issues/4964) Migrate light module migration to Protobuf encoding - - [privval] [\#4985](https://github.com/tendermint/tendermint/issues/4985) Migrate `privval` module to Protobuf encoding + - [light] [\#4964](https://github.com/tendermint/tendermint/pull/4964) Migrate light module migration to Protobuf encoding (@marbar3778) + - [privval] [\#4985](https://github.com/tendermint/tendermint/pull/4985) Migrate `privval` module to Protobuf encoding (@marbar3778) - Go API - - [light] [\#4946](https://github.com/tendermint/tendermint/issues/4946) Rename `lite2` pkg to `light`. Remove `lite` implementation. + - [consensus] [\#4582](https://github.com/tendermint/tendermint/pull/4582) RoundState: `Round`, `LockedRound` & `CommitRound` are now `int32` (@marbar3778) + - [consensus] [\#4582](https://github.com/tendermint/tendermint/pull/4582) HeightVoteSet: `round` is now `int32` (@marbar3778) - [crypto] [\#4721](https://github.com/tendermint/tendermint/pull/4721) Remove `SimpleHashFromMap()` and `SimpleProofsFromMap()` (@erikgrinaker) - - [crypto] [\#4940](https://github.com/tendermint/tendermint/issues/4940) All keys have become `[]byte` instead of `[]byte`. The byte method no longer returns the marshaled value but just the `[]byte` form of the data. - - [crypto] \4988 Removal of key type multisig + - [crypto] [\#4940](https://github.com/tendermint/tendermint/pull/4940) All keys have become `[]byte` instead of `[]byte`. The byte method no longer returns the marshaled value but just the `[]byte` form of the data. (@marbar3778) + - [crypto] [\#4988](https://github.com/tendermint/tendermint/pull/4988) Removal of key type multisig (@marbar3778) - The key has been moved to the [Cosmos-SDK](https://github.com/cosmos/cosmos-sdk/blob/master/crypto/types/multisig/multisignature.go) - - [crypto] [\#4989](https://github.com/tendermint/tendermint/issues/4989) Remove `Simple` prefixes from `SimpleProof`, `SimpleValueOp` & `SimpleProofNode`. + - [crypto] [\#4989](https://github.com/tendermint/tendermint/pull/4989) Remove `Simple` prefixes from `SimpleProof`, `SimpleValueOp` & `SimpleProofNode`. (@marbar3778) - `merkle.Proof` has been renamed to `ProofOps`. - Protobuf messages `Proof` & `ProofOp` has been moved to `proto/crypto/merkle` - `SimpleHashFromByteSlices` has been renamed to `HashFromByteSlices` - `SimpleHashFromByteSlicesIterative` has been renamed to `HashFromByteSlicesIterative` - `SimpleProofsFromByteSlices` has been renamed to `ProofsFromByteSlices` - - [crypto] [\#4941](https://github.com/tendermint/tendermint/issues/4941) Remove suffixes from all keys. + - [crypto] [\#4941](https://github.com/tendermint/tendermint/pull/4941) Remove suffixes from all keys. (@marbar3778) - ed25519: type `PrivKeyEd25519` is now `PrivKey` - ed25519: type `PubKeyEd25519` is now `PubKey` - secp256k1: type`PrivKeySecp256k1` is now `PrivKey` - secp256k1: type`PubKeySecp256k1` is now `PubKey` - sr25519: type `PrivKeySr25519` is now `PrivKey` - sr25519: type `PubKeySr25519` is now `PubKey` - - multisig: type `PubKeyMultisigThreshold` is now `PubKey` - - [libs] [\#4831](https://github.com/tendermint/tendermint/issues/4831) Remove `Bech32` pkg from Tendermint. This pkg now lives in the [cosmos-sdk](https://github.com/cosmos/cosmos-sdk/tree/4173ea5ebad906dd9b45325bed69b9c655504867/types/bech32) - - [rpc/client] [\#4947](https://github.com/tendermint/tendermint/issues/4947) `Validators`, `TxSearch` `page`/`per_page` params become pointers (@melekes) - - `UnconfirmedTxs` `limit` param is a pointer - - [proto] [\#5025](https://github.com/tendermint/tendermint/issues/5025) All proto files have been moved to `/proto` directory. + - [crypto] [\#5214](https://github.com/tendermint/tendermint/pull/5214) Change `GenPrivKeySecp256k1` to `GenPrivKeyFromSecret` to be consistent with other keys (@marbar3778) + - [crypto] [\#5236](https://github.com/tendermint/tendermint/pull/5236) `VerifyBytes` is now `VerifySignature` on the `crypto.PubKey` interface (@marbar3778) + - [evidence] [\#5361](https://github.com/tendermint/tendermint/pull/5361) Add LightClientAttackEvidence and change evidence interface (@cmwaters) + - [libs] [\#4831](https://github.com/tendermint/tendermint/pull/4831) Remove `Bech32` pkg from Tendermint. This pkg now lives in the [cosmos-sdk](https://github.com/cosmos/cosmos-sdk/tree/4173ea5ebad906dd9b45325bed69b9c655504867/types/bech32) (@marbar3778) + - [light] [\#4946](https://github.com/tendermint/tendermint/pull/4946) Rename `lite2` pkg to `light`. Remove `lite` implementation. (@marbar3778) + - [light] [\#5347](https://github.com/tendermint/tendermint/pull/5347) `NewClient`, `NewHTTPClient`, `VerifyHeader` and `VerifyLightBlockAtHeight` now accept `context.Context` as 1st param (@melekes) + - [merkle] [\#5193](https://github.com/tendermint/tendermint/pull/5193) `HashFromByteSlices` and `ProofsFromByteSlices` now return a hash for empty inputs, following RFC6962 (@erikgrinaker) + - [proto] [\#5025](https://github.com/tendermint/tendermint/pull/5025) All proto files have been moved to `/proto` directory. (@marbar3778) - Using the recommended the file layout from buf, [see here for more info](https://buf.build/docs/lint-checkers#file_layout) - - [state] [\#4679](https://github.com/tendermint/tendermint/issues/4679) `TxResult` is a Protobuf type defined in `abci` types directory - - [types] [\#4939](https://github.com/tendermint/tendermint/issues/4939) `SignedMsgType` has moved to a Protobuf enum types - - [types] [\#4962](https://github.com/tendermint/tendermint/issues/4962) `ConsensusParams`, `BlockParams`, `EvidenceParams`, `ValidatorParams` & `HashedParams` are now Protobuf types - - [types] [\#4852](https://github.com/tendermint/tendermint/issues/4852) Vote & Proposal `SignBytes` is now func `VoteSignBytes` & `ProposalSignBytes` - - [types] [\#4798](https://github.com/tendermint/tendermint/issues/4798) Simplify `VerifyCommitTrusting` func + remove extra validation (@melekes) - - [types] [\#4845](https://github.com/tendermint/tendermint/issues/4845) Remove `ABCIResult` - - [types] [\#5029](https://github.com/tendermint/tendermint/issues/5029) Rename all values from `PartsHeader` to `PartSetHeader` to have consistency - - [types] [\#4939](https://github.com/tendermint/tendermint/issues/4939) `Total` in `Parts` & `PartSetHeader` has been changed from a `int` to a `uint32` - - [types] [\#4939](https://github.com/tendermint/tendermint/issues/4939) Vote: `ValidatorIndex` & `Round` are now `int32` - - [types] [\#4939](https://github.com/tendermint/tendermint/issues/4939) Proposal: `POLRound` & `Round` are now `int32` - - [types] [\#4939](https://github.com/tendermint/tendermint/issues/4939) Block: `Round` is now `int32` - - [consensus] [\#4582](https://github.com/tendermint/tendermint/issues/4582) RoundState: `Round`, `LockedRound` & `CommitRound` are now `int32` - - [consensus] [\#4582](https://github.com/tendermint/tendermint/issues/4582) HeightVoteSet: `round` is now `int32` - - [rpc/jsonrpc/server] [\#5141](https://github.com/tendermint/tendermint/issues/5141) Remove `WriteRPCResponseArrayHTTP` (use `WriteRPCResponseHTTP` instead) (@melekes) + - [rpc/client] [\#4947](https://github.com/tendermint/tendermint/pull/4947) `Validators`, `TxSearch` `page`/`per_page` params become pointers (@melekes) + - `UnconfirmedTxs` `limit` param is a pointer + - [rpc/jsonrpc/server] [\#5141](https://github.com/tendermint/tendermint/pull/5141) Remove `WriteRPCResponseArrayHTTP` (use `WriteRPCResponseHTTP` instead) (@melekes) + - [state] [\#4679](https://github.com/tendermint/tendermint/pull/4679) `TxResult` is a Protobuf type defined in `abci` types directory (@marbar3778) + - [state] [\#5191](https://github.com/tendermint/tendermint/pull/5191) Add `State.InitialHeight` field to record initial block height, must be `1` (not `0`) to start from 1 (@erikgrinaker) + - [state] [\#5231](https://github.com/tendermint/tendermint/pull/5231) `LoadStateFromDBOrGenesisFile()` and `LoadStateFromDBOrGenesisDoc()` no longer saves the state in the database if not found, the genesis state is simply returned (@erikgrinaker) + - [state] [\#5348](https://github.com/tendermint/tendermint/pull/5348) Define an Interface for the state store. (@marbar3778) + - [types] [\#4939](https://github.com/tendermint/tendermint/pull/4939) `SignedMsgType` has moved to a Protobuf enum types (@marbar3778) + - [types] [\#4962](https://github.com/tendermint/tendermint/pull/4962) `ConsensusParams`, `BlockParams`, `EvidenceParams`, `ValidatorParams` & `HashedParams` are now Protobuf types (@marbar3778) + - [types] [\#4852](https://github.com/tendermint/tendermint/pull/4852) Vote & Proposal `SignBytes` is now func `VoteSignBytes` & `ProposalSignBytes` (@marbar3778) + - [types] [\#4798](https://github.com/tendermint/tendermint/pull/4798) Simplify `VerifyCommitTrusting` func + remove extra validation (@melekes) + - [types] [\#4845](https://github.com/tendermint/tendermint/pull/4845) Remove `ABCIResult` (@melekes) + - [types] [\#5029](https://github.com/tendermint/tendermint/pull/5029) Rename all values from `PartsHeader` to `PartSetHeader` to have consistency (@marbar3778) + - [types] [\#4939](https://github.com/tendermint/tendermint/pull/4939) `Total` in `Parts` & `PartSetHeader` has been changed from a `int` to a `uint32` (@marbar3778) + - [types] [\#4939](https://github.com/tendermint/tendermint/pull/4939) Vote: `ValidatorIndex` & `Round` are now `int32` (@marbar3778) + - [types] [\#4939](https://github.com/tendermint/tendermint/pull/4939) Proposal: `POLRound` & `Round` are now `int32` (@marbar3778) + - [types] [\#4939](https://github.com/tendermint/tendermint/pull/4939) Block: `Round` is now `int32` (@marbar3778) -### FEATURES: +### FEATURES -- [abci] [\#5031](https://github.com/tendermint/tendermint/issues/5031) Add `AppVersion` to consensus parameters (@james-ray) - - ... making it possible to update your ABCI application version via `EndBlock` response +- [abci] [\#5031](https://github.com/tendermint/tendermint/pull/5031) Add `AppVersion` to consensus parameters (@james-ray) + - This makes it possible to update your ABCI application version via `EndBlock` response +- [abci] [\#5174](https://github.com/tendermint/tendermint/pull/5174) Remove `MockEvidence` in favor of testing with actual evidence types (`DuplicateVoteEvidence` & `LightClientAttackEvidence`) (@cmwaters) +- [abci] [\#5191](https://github.com/tendermint/tendermint/pull/5191) Add `InitChain.InitialHeight` field giving the initial block height (@erikgrinaker) +- [abci] [\#5227](https://github.com/tendermint/tendermint/pull/5227) Add `ResponseInitChain.app_hash` which is recorded in genesis block (@erikgrinaker) +- [config] [\#5147](https://github.com/tendermint/tendermint/pull/5147) Add `--consensus.double_sign_check_height` flag and `DoubleSignCheckHeight` config variable. See [ADR-51](https://github.com/tendermint/tendermint/blob/master/docs/architecture/adr-051-double-signing-risk-reduction.md) (@dongsam) +- [db] [\#5233](https://github.com/tendermint/tendermint/pull/5233) Add support for `badgerdb` database backend (@erikgrinaker) - [evidence] [\#4532](https://github.com/tendermint/tendermint/pull/4532) Handle evidence from light clients (@melekes) -- [evidence] [#4821](https://github.com/tendermint/tendermint/pull/4821) Amnesia evidence can be detected, verified and committed (@cmwaters) -- [light] [\#4532](https://github.com/tendermint/tendermint/pull/4532) Submit conflicting headers, if any, to a full node & all witnesses (@melekes) -- [p2p] [\#4981](https://github.com/tendermint/tendermint/issues/4981) Expose `SaveAs` func on NodeKey (@melekes) +- [evidence] [#4821](https://github.com/tendermint/tendermint/pull/4821) Amnesia (light client attack) evidence can be detected, verified and committed (@cmwaters) +- [genesis] [\#5191](https://github.com/tendermint/tendermint/pull/5191) Add `initial_height` field to specify the initial chain height (defaults to `1`) (@erikgrinaker) +- [libs/math] [\#5665](https://github.com/tendermint/tendermint/pull/5665) Make fractions unsigned integers (uint64) (@cmwaters) +- [light] [\#5298](https://github.com/tendermint/tendermint/pull/5298) Morph validator set and signed header into light block (@cmwaters) +- [p2p] [\#4981](https://github.com/tendermint/tendermint/pull/4981) Expose `SaveAs` func on NodeKey (@melekes) +- [privval] [\#5239](https://github.com/tendermint/tendermint/pull/5239) Add `chainID` to requests from client. (@marbar3778) - [rpc] [\#4532](https://github.com/tendermint/tendermint/pull/4923) Support `BlockByHash` query (@fedekunze) -- [rpc] [\#4979](https://github.com/tendermint/tendermint/issues/4979) Support EXISTS operator in `/tx_search` query (@melekes) -- [rpc] [\#5017](https://github.com/tendermint/tendermint/issues/5017) Add `/check_tx` endpoint to check transactions without executing them or adding them to the mempool (@melekes) -- [statesync] Add state sync support, where a new node can be rapidly bootstrapped by fetching state snapshots from peers instead of replaying blocks. See the `[statesync]` config section. +- [rpc] [\#4979](https://github.com/tendermint/tendermint/pull/4979) Support EXISTS operator in `/tx_search` query (@melekes) +- [rpc] [\#5017](https://github.com/tendermint/tendermint/pull/5017) Add `/check_tx` endpoint to check transactions without executing them or adding them to the mempool (@melekes) - [rpc] [\#5108](https://github.com/tendermint/tendermint/pull/5108) Subscribe using the websocket for new evidence events (@cmwaters) +- [statesync] Add state sync support, where a new node can be rapidly bootstrapped by fetching state snapshots from peers instead of replaying blocks. See the `[statesync]` config section. +- [evidence] [\#5361](https://github.com/tendermint/tendermint/pull/5361) Add LightClientAttackEvidence and refactor evidence lifecycle - for more information see [ADR-059](https://github.com/tendermint/tendermint/blob/master/docs/architecture/adr-059-evidence-composition-and-lifecycle.md) (@cmwaters) -### IMPROVEMENTS: +### IMPROVEMENTS -- [consensus] [\#4578](https://github.com/tendermint/tendermint/issues/4578) Attempt to repair the consensus WAL file (`data/cs.wal/wal`) automatically in case of corruption (@alessio) +- [blockchain] [\#5278](https://github.com/tendermint/tendermint/pull/5278) Verify only +2/3 of the signatures in a block when fast syncing. (@marbar3778) +- [consensus] [\#4578](https://github.com/tendermint/tendermint/pull/4578) Attempt to repair the consensus WAL file (`data/cs.wal/wal`) automatically in case of corruption (@alessio) - The original WAL file will be backed up to `data/cs.wal/wal.CORRUPTED`. -- [evidence] [\#4722](https://github.com/tendermint/tendermint/pull/4722) Improved evidence db (@cmwaters) +- [consensus] [\#5143](https://github.com/tendermint/tendermint/pull/5143) Only call `privValidator.GetPubKey` once per block (@melekes) +- [evidence] [\#4722](https://github.com/tendermint/tendermint/pull/4722) Consolidate evidence store and pool types to improve evidence DB (@cmwaters) - [evidence] [\#4839](https://github.com/tendermint/tendermint/pull/4839) Reject duplicate evidence from being proposed (@cmwaters) -- [evidence] [\#4892](https://github.com/tendermint/tendermint/pull/4892) Remove redundant header from phantom validator evidence (@cmwaters) +- [evidence] [\#5219](https://github.com/tendermint/tendermint/pull/5219) Change the source of evidence time to block time (@cmwaters) +- [libs] [\#5126](https://github.com/tendermint/tendermint/pull/5126) Add a sync package which wraps sync.(RW)Mutex & deadlock.(RW)Mutex and use a build flag (deadlock) in order to enable deadlock checking (@marbar3778) - [light] [\#4935](https://github.com/tendermint/tendermint/pull/4935) Fetch and compare a new header with witnesses in parallel (@melekes) -- [light] [\#4929](https://github.com/tendermint/tendermint/pull/4929) compare header w/ witnesses only when doing bisection (@melekes) -- [light] [\#4916](https://github.com/tendermint/tendermint/pull/4916) validate basic for inbound validator sets and headers before further processing them (@cmwaters) -- [p2p/conn] [\#4795](https://github.com/tendermint/tendermint/issues/4795) Return err on `signChallenge()` instead of panic +- [light] [\#4929](https://github.com/tendermint/tendermint/pull/4929) Compare header with witnesses only when doing bisection (@melekes) +- [light] [\#4916](https://github.com/tendermint/tendermint/pull/4916) Validate basic for inbound validator sets and headers before further processing them (@cmwaters) +- [mempool] Add RemoveTxByKey() exported function for custom mempool cleaning (@p4u) +- [p2p/conn] [\#4795](https://github.com/tendermint/tendermint/pull/4795) Return err on `signChallenge()` instead of panic +- [privval] [\#5437](https://github.com/tendermint/tendermint/pull/5437) `NewSignerDialerEndpoint` can now be given `SignerServiceEndpointOption` (@erikgrinaker) +- [rpc] [\#4968](https://github.com/tendermint/tendermint/pull/4968) JSON encoding is now handled by `libs/json`, not Amino (@erikgrinaker) +- [rpc] [\#5293](https://github.com/tendermint/tendermint/pull/5293) `/dial_peers` has added `private` and `unconditional` as parameters. (@marbar3778) - [state] [\#4781](https://github.com/tendermint/tendermint/pull/4781) Export `InitStateVersion` for the initial state version (@erikgrinaker) - [txindex] [\#4466](https://github.com/tendermint/tendermint/pull/4466) Allow to index an event at runtime (@favadi) - `abci.EventAttribute` replaces `KV.Pair` -- [libs] [\#5126](https://github.com/tendermint/tendermint/issues/5126) Add a sync package which wraps sync.(RW)Mutex & deadlock.(RW)Mutex and use a build flag (deadlock) in order to enable deadlock checking - [types] [\#4905](https://github.com/tendermint/tendermint/pull/4905) Add `ValidateBasic` to validator and validator set (@cmwaters) -- [rpc] [\#4968](https://github.com/tendermint/tendermint/issues/4968) JSON encoding is now handled by `libs/json`, not Amino -- [mempool] Add RemoveTxByKey() exported function for custom mempool cleaning (@p4u) -- [consensus] [\#5143](https://github.com/tendermint/tendermint/issues/5143) Only call `privValidator.GetPubKey` once per block (@melekes) +- [types] [\#5340](https://github.com/tendermint/tendermint/pull/5340) Add check in `Header.ValidateBasic()` for block protocol version (@marbar3778) +- [types] [\#5490](https://github.com/tendermint/tendermint/pull/5490) Use `Commit` and `CommitSig` max sizes instead of vote max size to calculate the maximum block size. (@cmwaters) -### BUG FIXES: -- [blockchain/v2] Correctly set block store base in status responses (@erikgrinaker) +### BUG FIXES + +- [abci/grpc] [\#5520](https://github.com/tendermint/tendermint/pull/5520) Return async responses in order, to avoid mempool panics. (@erikgrinaker) +- [blockchain/v2] [\#4971](https://github.com/tendermint/tendermint/pull/4971) Correctly set block store base in status responses (@erikgrinaker) +- [blockchain/v2] [\#5499](https://github.com/tendermint/tendermint/pull/5499) Fix "duplicate block enqueued by processor" panic (@melekes) +- [blockchain/v2] [\#5530](https://github.com/tendermint/tendermint/pull/5530) Fix out of order block processing panic (@melekes) +- [blockchain/v2] [\#5553](https://github.com/tendermint/tendermint/pull/5553) Make the removal of an already removed peer a noop (@melekes) - [consensus] [\#4895](https://github.com/tendermint/tendermint/pull/4895) Cache the address of the validator to reduce querying a remote KMS (@joe-bowman) -- [consensus] [\#4970](https://github.com/tendermint/tendermint/issues/4970) Stricter on `LastCommitRound` check (@cuonglm) -- [p2p][\#5136](https://github.com/tendermint/tendermint/pull/5136) Fix error for peer with the same ID but different IPs (@valardragon) -- [proxy] [\#5078](https://github.com/tendermint/tendermint/issues/5078) Fix a bug, where TM does not exit when ABCI app crashes (@melekes) - - -## v0.34.0-rc1 - -This release was removed, as a premature GitHub tag was recorded on sum.golang.org causing checksum errors. +- [consensus] [\#4970](https://github.com/tendermint/tendermint/pull/4970) Don't allow `LastCommitRound` to be negative (@cuonglm) +- [consensus] [\#5329](https://github.com/tendermint/tendermint/pull/5329) Fix wrong proposer schedule for validators returned by `InitChain` (@erikgrinaker) +- [docker] [\#5385](https://github.com/tendermint/tendermint/pull/5385) Fix incorrect `time_iota_ms` default setting causing block timestamp drift (@erikgrinaker) +- [evidence] [\#5170](https://github.com/tendermint/tendermint/pull/5170) Change ABCI evidence time to the time the infraction happened not the time the evidence was committed on the block (@cmwaters) +- [evidence] [\#5610](https://github.com/tendermint/tendermint/pull/5610) Make it possible for ABCI evidence to be formed from Tendermint evidence (@cmwaters) +- [libs/rand] [\#5215](https://github.com/tendermint/tendermint/pull/5215) Fix out-of-memory error on unexpected argument of Str() (@SadPencil) +- [light] [\#5307](https://github.com/tendermint/tendermint/pull/5307) Persist correct proposer priority in light client validator sets (@cmwaters) +- [p2p] [\#5136](https://github.com/tendermint/tendermint/pull/5136) Fix error for peer with the same ID but different IPs (@valardragon) +- [privval] [\#5638](https://github.com/tendermint/tendermint/pull/5638) Increase read/write timeout to 5s and calculate ping interval based on it (@JoeKash) +- [proxy] [\#5078](https://github.com/tendermint/tendermint/pull/5078) Force Tendermint to exit when ABCI app crashes (@melekes) +- [rpc] [\#5660](https://github.com/tendermint/tendermint/pull/5660) Set `application/json` as the `Content-Type` header in RPC responses. (@alexanderbez) +- [store] [\#5382](https://github.com/tendermint/tendermint/pull/5382) Fix race conditions when loading/saving/pruning blocks (@erikgrinaker) ## v0.33.8 diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 6560cdf59..5dc5f18d8 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -24,7 +24,7 @@ Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermi ### FEATURES -- [libs/math] \#5665 Make fractions unsigned integers (uint64) (@cmwaters) + ### IMPROVEMENTS @@ -33,6 +33,5 @@ Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermi ### BUG FIXES -- [rpc] \#5660 Set `application/json` as the `Content-Type` header in RPC responses. - [types] \#5523 Change json naming of `PartSetHeader` within `BlockID` from `parts` to `part_set_header` (@marbar3778) - [privval] \#5638 Increase read/write timeout to 5s and calculate ping interval based on it (@JoeKash) From 8b29622fe22e3431c08f4c5471ab0bc908dda05e Mon Sep 17 00:00:00 2001 From: Tess Rinearson Date: Thu, 19 Nov 2020 19:10:45 +0100 Subject: [PATCH 27/27] .goreleaser: add windows, remove arm (32 bit) (#5692) This updates the goreleaser tooling to build the same binaries that were built for the 0.34.0 release. --- .goreleaser.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.goreleaser.yml b/.goreleaser.yml index 9fb5933af..494f7c245 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -14,9 +14,9 @@ builds: goos: - darwin - linux + - windows goarch: - amd64 - - arm - arm64 checksum: