From 8d9d19a11353adf52a59a17e01165d7680b6d8f6 Mon Sep 17 00:00:00 2001 From: Sam Kleinman Date: Thu, 9 Jun 2022 18:22:47 -0400 Subject: [PATCH 01/61] e2e/generator: enlarge nightly test suite (#8731) --- test/e2e/generator/generate.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/test/e2e/generator/generate.go b/test/e2e/generator/generate.go index 0e8470111..01e04a418 100644 --- a/test/e2e/generator/generate.go +++ b/test/e2e/generator/generate.go @@ -14,14 +14,14 @@ var ( // testnetCombinations defines global testnet options, where we generate a // separate testnet for each combination (Cartesian product) of options. testnetCombinations = map[string][]interface{}{ - "topology": {"single", "quad", "large"}, - "initialHeight": {0, 1000}, + "topology": {"single", "quad", "large"}, "initialState": { map[string]string{}, map[string]string{"initial01": "a", "initial02": "b", "initial03": "c"}, }, "validators": {"genesis", "initchain"}, "abci": {"builtin", "outofprocess"}, + "txSize": {1024, 2048, 4096, 8192}, } // The following specify randomly chosen values for testnet nodes. @@ -63,11 +63,11 @@ var ( } // the following specify random chosen values for the entire testnet - evidence = uniformChoice{0, 1, 10} - txSize = uniformChoice{1024, 4096} // either 1kb or 4kb - ipv6 = uniformChoice{false, true} - keyType = uniformChoice{types.ABCIPubKeyTypeEd25519, types.ABCIPubKeyTypeSecp256k1} - abciDelays = uniformChoice{"none", "small", "large"} + initialHeight = uniformChoice{0, 1000} + evidence = uniformChoice{0, 1, 10} + ipv6 = uniformChoice{false, true} + keyType = uniformChoice{types.ABCIPubKeyTypeEd25519, types.ABCIPubKeyTypeSecp256k1} + abciDelays = uniformChoice{"none", "small", "large"} voteExtensionEnableHeightOffset = uniformChoice{int64(0), int64(10), int64(100)} voteExtensionEnabled = uniformChoice{true, false} @@ -109,7 +109,6 @@ type Options struct { func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, error) { manifest := e2e.Manifest{ IPv6: ipv6.Choose(r).(bool), - InitialHeight: int64(opt["initialHeight"].(int)), InitialState: opt["initialState"].(map[string]string), Validators: &map[string]int64{}, ValidatorUpdates: map[string]map[string]int64{}, @@ -117,9 +116,11 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er KeyType: keyType.Choose(r).(string), Evidence: evidence.Choose(r).(int), QueueType: "priority", - TxSize: txSize.Choose(r).(int), + TxSize: opt["txSize"].(int), } + manifest.InitialHeight = int64(initialHeight.Choose(r).(int)) + if voteExtensionEnabled.Choose(r).(bool) { manifest.VoteExtensionsEnableHeight = manifest.InitialHeight + voteExtensionEnableHeightOffset.Choose(r).(int64) } From b0e48ca5f3019256ac87708e7065bd4c10c40caa Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Fri, 10 Jun 2022 17:17:33 +0200 Subject: [PATCH 02/61] remove unused config variables (#8738) --- cmd/tendermint/commands/testnet.go | 1 - config/config.go | 10 ---------- config/toml.go | 3 --- 3 files changed, 14 deletions(-) diff --git a/cmd/tendermint/commands/testnet.go b/cmd/tendermint/commands/testnet.go index 82c8cc6f9..5ba76621c 100644 --- a/cmd/tendermint/commands/testnet.go +++ b/cmd/tendermint/commands/testnet.go @@ -239,7 +239,6 @@ Example: for i := 0; i < nValidators+nNonValidators; i++ { nodeDir := filepath.Join(outputDir, fmt.Sprintf("%s%d", nodeDirPrefix, i)) config.SetRoot(nodeDir) - config.P2P.AllowDuplicateIP = true if populatePersistentPeers { persistentPeersWithoutSelf := make([]string, 0) for j := 0; j < len(persistentPeers); j++ { diff --git a/config/config.go b/config/config.go index 43b3fc10f..d875b8569 100644 --- a/config/config.go +++ b/config/config.go @@ -638,9 +638,6 @@ type P2PConfig struct { //nolint: maligned // other peers) PrivatePeerIDs string `mapstructure:"private-peer-ids"` - // Toggle to disable guard against peers connecting from the same ip. - AllowDuplicateIP bool `mapstructure:"allow-duplicate-ip"` - // Time to wait before flushing messages out on the connection FlushThrottleTimeout time.Duration `mapstructure:"flush-throttle-timeout"` @@ -657,10 +654,6 @@ type P2PConfig struct { //nolint: maligned HandshakeTimeout time.Duration `mapstructure:"handshake-timeout"` DialTimeout time.Duration `mapstructure:"dial-timeout"` - // Testing params. - // Force dial to fail - TestDialFail bool `mapstructure:"test-dial-fail"` - // Makes it possible to configure which queue backend the p2p // layer uses. Options are: "fifo" and "priority", // with the default being "priority". @@ -685,10 +678,8 @@ func DefaultP2PConfig() *P2PConfig { SendRate: 5120000, // 5 mB/s RecvRate: 5120000, // 5 mB/s PexReactor: true, - AllowDuplicateIP: false, HandshakeTimeout: 20 * time.Second, DialTimeout: 3 * time.Second, - TestDialFail: false, QueueType: "priority", } } @@ -715,7 +706,6 @@ func (cfg *P2PConfig) ValidateBasic() error { func TestP2PConfig() *P2PConfig { cfg := DefaultP2PConfig() cfg.ListenAddress = "tcp://127.0.0.1:36656" - cfg.AllowDuplicateIP = true cfg.FlushThrottleTimeout = 10 * time.Millisecond return cfg } diff --git a/config/toml.go b/config/toml.go index 4db4f4e65..cf6635d45 100644 --- a/config/toml.go +++ b/config/toml.go @@ -319,9 +319,6 @@ pex = {{ .P2P.PexReactor }} # Warning: IPs will be exposed at /net_info, for more information https://github.com/tendermint/tendermint/issues/3055 private-peer-ids = "{{ .P2P.PrivatePeerIDs }}" -# Toggle to disable guard against peers connecting from the same ip. -allow-duplicate-ip = {{ .P2P.AllowDuplicateIP }} - # Peer connection configuration. handshake-timeout = "{{ .P2P.HandshakeTimeout }}" dial-timeout = "{{ .P2P.DialTimeout }}" From 7172862786cabaeb8ac06a6d646955a2faa6da31 Mon Sep 17 00:00:00 2001 From: Sam Kleinman Date: Sat, 11 Jun 2022 08:21:55 -0400 Subject: [PATCH 03/61] e2e: split test cases to avoid hitting timeouts (#8741) --- .github/workflows/e2e-manual.yml | 4 ++-- .github/workflows/e2e-nightly-master.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/e2e-manual.yml b/.github/workflows/e2e-manual.yml index 6da4c3342..56a406b7a 100644 --- a/.github/workflows/e2e-manual.yml +++ b/.github/workflows/e2e-manual.yml @@ -11,7 +11,7 @@ jobs: strategy: fail-fast: false matrix: - group: ['00', '01', '02', '03'] + group: ['00', '01', '02', '03', '04'] runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -29,7 +29,7 @@ jobs: - name: Generate testnets working-directory: test/e2e # When changing -g, also change the matrix groups above - run: ./build/generator -g 4 -d networks/nightly/ + run: ./build/generator -g 5 -d networks/nightly/ - name: Run ${{ matrix.p2p }} p2p testnets working-directory: test/e2e diff --git a/.github/workflows/e2e-nightly-master.yml b/.github/workflows/e2e-nightly-master.yml index 7fac05f2d..367364efb 100644 --- a/.github/workflows/e2e-nightly-master.yml +++ b/.github/workflows/e2e-nightly-master.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - group: ['00', '01', '02', '03'] + group: ['00', '01', '02', '03', "04"] runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -34,7 +34,7 @@ jobs: - name: Generate testnets working-directory: test/e2e # When changing -g, also change the matrix groups above - run: ./build/generator -g 4 -d networks/nightly/ + run: ./build/generator -g 5 -d networks/nightly/ - name: Run ${{ matrix.p2p }} p2p testnets working-directory: test/e2e From 06175129ed7927e58ab070c9e33699d8b0e78e1c Mon Sep 17 00:00:00 2001 From: Jasmina Malicevic Date: Mon, 13 Jun 2022 14:09:57 +0200 Subject: [PATCH 04/61] abci-cli: added `PrepareProposal` command to cli (#8656) * Prepare prosal cli --- CHANGELOG_PENDING.md | 3 +- abci/cmd/abci-cli/abci-cli.go | 80 +++++++++++++++++++++++++++++++- abci/tests/server/client.go | 13 ++++++ abci/tests/test_cli/ex1.abci | 2 + abci/tests/test_cli/ex1.abci.out | 10 ++++ abci/tests/test_cli/test.sh | 2 + 6 files changed, 107 insertions(+), 3 deletions(-) diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index c3e191e7e..f33198403 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -31,7 +31,8 @@ Special thanks to external contributors on this release: - [abci] \#7984 Remove the locks preventing concurrent use of ABCI applications by Tendermint. (@tychoish) - [abci] \#8605 Remove info, log, events, gasUsed and mempoolError fields from ResponseCheckTx as they are not used by Tendermint. (@jmalicevic) - [abci] \#8664 Move `app_hash` parameter from `Commit` to `FinalizeBlock`. (@sergio-mena) - + - [abci] \#8656 Added cli command for `PrepareProposal`. (@jmalicevic) + - P2P Protocol - [p2p] \#7035 Remove legacy P2P routing implementation and associated configuration options. (@tychoish) diff --git a/abci/cmd/abci-cli/abci-cli.go b/abci/cmd/abci-cli/abci-cli.go index b09f3c9a7..237493e0e 100644 --- a/abci/cmd/abci-cli/abci-cli.go +++ b/abci/cmd/abci-cli/abci-cli.go @@ -2,6 +2,7 @@ package main import ( "bufio" + "bytes" "encoding/hex" "errors" "fmt" @@ -130,6 +131,7 @@ func addCommands(cmd *cobra.Command, logger log.Logger) { cmd.AddCommand(commitCmd) cmd.AddCommand(versionCmd) cmd.AddCommand(testCmd) + cmd.AddCommand(prepareProposalCmd) cmd.AddCommand(getQueryCmd()) // examples @@ -170,7 +172,7 @@ This command opens an interactive console for running any of the other commands without opening a new connection each time `, Args: cobra.ExactArgs(0), - ValidArgs: []string{"echo", "info", "finalize_block", "check_tx", "commit", "query"}, + ValidArgs: []string{"echo", "info", "query", "check_tx", "prepare_proposal", "finalize_block", "commit"}, RunE: cmdConsole, } @@ -224,6 +226,14 @@ var versionCmd = &cobra.Command{ }, } +var prepareProposalCmd = &cobra.Command{ + Use: "prepare_proposal", + Short: "prepare proposal", + Long: "prepare proposal", + Args: cobra.MinimumNArgs(1), + RunE: cmdPrepareProposal, +} + func getQueryCmd() *cobra.Command { cmd := &cobra.Command{ Use: "query", @@ -335,6 +345,13 @@ func cmdTest(cmd *cobra.Command, args []string) error { }, nil, []byte{0, 0, 0, 0, 0, 0, 0, 5}) }, func() error { return servertest.Commit(ctx, client) }, + func() error { + return servertest.PrepareProposal(ctx, client, [][]byte{ + {0x01}, + }, []types.TxRecord_TxAction{ + types.TxRecord_UNMODIFIED, + }, nil) + }, }) } @@ -435,6 +452,8 @@ func muxOnCommands(cmd *cobra.Command, pArgs []string) error { return cmdInfo(cmd, actualArgs) case "query": return cmdQuery(cmd, actualArgs) + case "prepare_proposal": + return cmdPrepareProposal(cmd, actualArgs) default: return cmdUnimplemented(cmd, pArgs) } @@ -605,6 +624,64 @@ func cmdQuery(cmd *cobra.Command, args []string) error { return nil } +func inTxArray(txByteArray [][]byte, tx []byte) bool { + for _, txTmp := range txByteArray { + if bytes.Equal(txTmp, tx) { + return true + } + + } + return false +} +func cmdPrepareProposal(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + printResponse(cmd, args, response{ + Code: codeBad, + Info: "Must provide at least one transaction", + Log: "Must provide at least one transaction", + }) + return nil + } + txsBytesArray := make([][]byte, len(args)) + + for i, arg := range args { + txBytes, err := stringOrHexToBytes(arg) + if err != nil { + return err + } + txsBytesArray[i] = txBytes + } + + res, err := client.PrepareProposal(cmd.Context(), &types.RequestPrepareProposal{ + Txs: txsBytesArray, + // kvstore has to have this parameter in order not to reject a tx as the default value is 0 + MaxTxBytes: 65536, + }) + if err != nil { + return err + } + resps := make([]response, 0, len(res.TxResults)+1) + for _, tx := range res.TxRecords { + existingTx := inTxArray(txsBytesArray, tx.Tx) + if tx.Action == types.TxRecord_UNKNOWN || + (existingTx && tx.Action == types.TxRecord_ADDED) || + (!existingTx && (tx.Action == types.TxRecord_UNMODIFIED || tx.Action == types.TxRecord_REMOVED)) { + resps = append(resps, response{ + Code: codeBad, + Log: "Failed. Tx: " + string(tx.GetTx()) + " action: " + tx.Action.String(), + }) + } else { + resps = append(resps, response{ + Code: code.CodeTypeOK, + Log: "Succeeded. Tx: " + string(tx.Tx) + " action: " + tx.Action.String(), + }) + } + } + + printResponse(cmd, args, resps...) + return nil +} + func makeKVStoreCmd(logger log.Logger) func(*cobra.Command, []string) error { return func(cmd *cobra.Command, args []string) error { // Create the application - in memory or persisted to disk @@ -649,7 +726,6 @@ func printResponse(cmd *cobra.Command, args []string, rsps ...response) { fmt.Printf("-> code: OK\n") } else { fmt.Printf("-> code: %d\n", rsp.Code) - } if len(rsp.Data) != 0 { diff --git a/abci/tests/server/client.go b/abci/tests/server/client.go index cddb42ec0..7762c8d03 100644 --- a/abci/tests/server/client.go +++ b/abci/tests/server/client.go @@ -70,6 +70,19 @@ func FinalizeBlock(ctx context.Context, client abciclient.Client, txBytes [][]by return nil } +func PrepareProposal(ctx context.Context, client abciclient.Client, txBytes [][]byte, codeExp []types.TxRecord_TxAction, dataExp []byte) error { + res, _ := client.PrepareProposal(ctx, &types.RequestPrepareProposal{Txs: txBytes}) + for i, tx := range res.TxRecords { + if tx.Action != codeExp[i] { + fmt.Println("Failed test: PrepareProposal") + fmt.Printf("PrepareProposal response code was unexpected. Got %v expected %v.", + tx.Action, codeExp) + return errors.New("PrepareProposal error") + } + } + fmt.Println("Passed test: PrepareProposal") + return nil +} func CheckTx(ctx context.Context, client abciclient.Client, txBytes []byte, codeExp uint32, dataExp []byte) error { res, _ := client.CheckTx(ctx, &types.RequestCheckTx{Tx: txBytes}) code, data := res.Code, res.Data diff --git a/abci/tests/test_cli/ex1.abci b/abci/tests/test_cli/ex1.abci index 56355dc94..dc9e213ec 100644 --- a/abci/tests/test_cli/ex1.abci +++ b/abci/tests/test_cli/ex1.abci @@ -1,5 +1,6 @@ echo hello info +prepare_proposal "abc" finalize_block "abc" commit info @@ -7,3 +8,4 @@ query "abc" finalize_block "def=xyz" "ghi=123" commit query "def" +prepare_proposal "preparedef" \ No newline at end of file diff --git a/abci/tests/test_cli/ex1.abci.out b/abci/tests/test_cli/ex1.abci.out index 9a35290b0..f4f342dfb 100644 --- a/abci/tests/test_cli/ex1.abci.out +++ b/abci/tests/test_cli/ex1.abci.out @@ -8,6 +8,10 @@ -> data: {"size":0} -> data.hex: 0x7B2273697A65223A307D +> prepare_proposal "abc" +-> code: OK +-> log: Succeeded. Tx: abc action: UNMODIFIED + > finalize_block "abc" -> code: OK -> code: OK @@ -48,3 +52,9 @@ -> value: xyz -> value.hex: 78797A +> prepare_proposal "preparedef" +-> code: OK +-> log: Succeeded. Tx: def action: ADDED +-> code: OK +-> log: Succeeded. Tx: preparedef action: REMOVED + diff --git a/abci/tests/test_cli/test.sh b/abci/tests/test_cli/test.sh index 9c02ce6f5..d160d59c9 100755 --- a/abci/tests/test_cli/test.sh +++ b/abci/tests/test_cli/test.sh @@ -30,6 +30,8 @@ function testExample() { cat "${INPUT}.out.new" echo "Expected:" cat "${INPUT}.out" + echo "Diff:" + diff "${INPUT}.out" "${INPUT}.out.new" exit 1 fi From 82907c84fa53501696b21931c9e97c7ee99d2f75 Mon Sep 17 00:00:00 2001 From: Marko Date: Mon, 13 Jun 2022 19:20:54 +0200 Subject: [PATCH 05/61] sink/psql: json marshal instead of proto (#8637) Storing transaction records as JSON makes it simpler for clients of the index. --- CHANGELOG_PENDING.md | 3 ++- internal/state/indexer/sink/psql/psql.go | 9 ++++++--- internal/state/indexer/sink/psql/psql_test.go | 12 ++++++++---- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index f33198403..608abc06d 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -32,7 +32,8 @@ Special thanks to external contributors on this release: - [abci] \#8605 Remove info, log, events, gasUsed and mempoolError fields from ResponseCheckTx as they are not used by Tendermint. (@jmalicevic) - [abci] \#8664 Move `app_hash` parameter from `Commit` to `FinalizeBlock`. (@sergio-mena) - [abci] \#8656 Added cli command for `PrepareProposal`. (@jmalicevic) - + - [sink/psql] \#8637 tx_results emitted from psql sink are now json encoded, previously they were protobuf encoded + - P2P Protocol - [p2p] \#7035 Remove legacy P2P routing implementation and associated configuration options. (@tychoish) diff --git a/internal/state/indexer/sink/psql/psql.go b/internal/state/indexer/sink/psql/psql.go index c06383264..57f5e5c3d 100644 --- a/internal/state/indexer/sink/psql/psql.go +++ b/internal/state/indexer/sink/psql/psql.go @@ -9,8 +9,7 @@ import ( "strings" "time" - "github.com/gogo/protobuf/proto" - + "github.com/gogo/protobuf/jsonpb" abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/internal/pubsub/query" "github.com/tendermint/tendermint/internal/state/indexer" @@ -177,12 +176,16 @@ INSERT INTO `+tableBlocks+` (height, chain_id, created_at) }) } +var ( + jsonpbMarshaller = jsonpb.Marshaler{} +) + func (es *EventSink) IndexTxEvents(txrs []*abci.TxResult) error { ts := time.Now().UTC() for _, txr := range txrs { // Encode the result message in protobuf wire format for indexing. - resultData, err := proto.Marshal(txr) + resultData, err := jsonpbMarshaller.MarshalToString(txr) if err != nil { return fmt.Errorf("marshaling tx_result: %w", err) } diff --git a/internal/state/indexer/sink/psql/psql_test.go b/internal/state/indexer/sink/psql/psql_test.go index 72d14b5d8..2625d7245 100644 --- a/internal/state/indexer/sink/psql/psql_test.go +++ b/internal/state/indexer/sink/psql/psql_test.go @@ -1,6 +1,7 @@ package psql import ( + "bytes" "context" "database/sql" "flag" @@ -12,7 +13,7 @@ import ( "time" "github.com/adlio/schema" - "github.com/gogo/protobuf/proto" + "github.com/gogo/protobuf/jsonpb" "github.com/ory/dockertest" "github.com/ory/dockertest/docker" "github.com/stretchr/testify/assert" @@ -151,6 +152,8 @@ func TestType(t *testing.T) { assert.Equal(t, indexer.PSQL, psqlSink.Type()) } +var jsonpbUnmarshaller = jsonpb.Unmarshaler{} + func TestIndexing(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -278,13 +281,14 @@ func loadTxResult(hash []byte) (*abci.TxResult, error) { hashString := fmt.Sprintf("%X", hash) var resultData []byte if err := testDB().QueryRow(` -SELECT tx_result FROM `+tableTxResults+` WHERE tx_hash = $1; -`, hashString).Scan(&resultData); err != nil { + SELECT tx_result FROM `+tableTxResults+` WHERE tx_hash = $1; + `, hashString).Scan(&resultData); err != nil { return nil, fmt.Errorf("lookup transaction for hash %q failed: %v", hashString, err) } + reader := bytes.NewBuffer(resultData) txr := new(abci.TxResult) - if err := proto.Unmarshal(resultData, txr); err != nil { + if err := jsonpbUnmarshaller.Unmarshal(reader, txr); err != nil { return nil, fmt.Errorf("unmarshaling txr: %w", err) } From 21bbbe3e2a1ad133241abdefdaf6874f72fbbc4d Mon Sep 17 00:00:00 2001 From: Jeeyong Um Date: Tue, 14 Jun 2022 16:50:55 +0800 Subject: [PATCH 06/61] mempool: fix typos in test (#8746) --- internal/mempool/priority_queue_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/mempool/priority_queue_test.go b/internal/mempool/priority_queue_test.go index ddc84806d..90f611162 100644 --- a/internal/mempool/priority_queue_test.go +++ b/internal/mempool/priority_queue_test.go @@ -90,7 +90,7 @@ func TestTxPriorityQueue_GetEvictableTxs(t *testing.T) { expectedLen int }{ { - name: "larest priority; single tx", + name: "largest priority; single tx", priority: int64(max + 1), txSize: 5, totalSize: totalSize, @@ -98,7 +98,7 @@ func TestTxPriorityQueue_GetEvictableTxs(t *testing.T) { expectedLen: 1, }, { - name: "larest priority; multi tx", + name: "largest priority; multi tx", priority: int64(max + 1), txSize: 17, totalSize: totalSize, @@ -106,7 +106,7 @@ func TestTxPriorityQueue_GetEvictableTxs(t *testing.T) { expectedLen: 4, }, { - name: "larest priority; out of capacity", + name: "largest priority; out of capacity", priority: int64(max + 1), txSize: totalSize + 1, totalSize: totalSize, From a4cf8939b88b70072f40a87e87516496ee3cc347 Mon Sep 17 00:00:00 2001 From: Jeeyong Um Date: Tue, 14 Jun 2022 18:54:35 +0800 Subject: [PATCH 07/61] mempool: fix error message check in test (#8750) --- internal/mempool/mempool_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/mempool/mempool_test.go b/internal/mempool/mempool_test.go index 538cb3e1f..42fb13bdc 100644 --- a/internal/mempool/mempool_test.go +++ b/internal/mempool/mempool_test.go @@ -631,7 +631,7 @@ func TestTxMempool_CheckTxPostCheckError(t *testing.T) { require.NoError(t, txmp.CheckTx(ctx, tx, callback, TxInfo{SenderID: 0})) } else { err = txmp.CheckTx(ctx, tx, callback, TxInfo{SenderID: 0}) - fmt.Print(err.Error()) + require.EqualError(t, err, "test error") } }) } From a2908c29d5aa6d9437edb052d52bedb254e2c963 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Jun 2022 11:06:36 +0000 Subject: [PATCH 08/61] build(deps): Bump github.com/vektra/mockery/v2 from 2.12.3 to 2.13.0 (#8748) Bumps [github.com/vektra/mockery/v2](https://github.com/vektra/mockery) from 2.12.3 to 2.13.0.
Release notes

Sourced from github.com/vektra/mockery/v2's releases.

v2.13.0

Changelog

  • f9f6d38 Merge pull request #477 from LandonTClipp/generics_release
  • 6498bd6 Updating dependencies

v2.13.0-beta.1

Changelog

  • 9dba1fd Merge pull request #454 from Gevrai/gejo-move-expecter-test
  • cde38d9 Move generated ExpecterTest to mocks directory

v2.13.0-beta.0

Changelog

  • dc5539e Add support for generating mocks for generic interfaces
  • 33c4bf3 Generics mocking fixes
  • a727d70 Genrics support: rename and comment methods
  • 4c0f6c8 Merge conflict resolution
  • 46c61f0 Merge pull request #456 from cruickshankpg/mock-generic-interfaces
  • ed55a47 Update x/tools to pick up fix for golang/go#51629
Commits
  • f9f6d38 Merge pull request #477 from LandonTClipp/generics_release
  • 6498bd6 Updating dependencies
  • 9dba1fd Merge pull request #454 from Gevrai/gejo-move-expecter-test
  • cde38d9 Move generated ExpecterTest to mocks directory
  • 46c61f0 Merge pull request #456 from cruickshankpg/mock-generic-interfaces
  • 4c0f6c8 Merge conflict resolution
  • ed55a47 Update x/tools to pick up fix for golang/go#51629
  • a727d70 Genrics support: rename and comment methods
  • 33c4bf3 Generics mocking fixes
  • dc5539e Add support for generating mocks for generic interfaces
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/vektra/mockery/v2&package-manager=go_modules&previous-version=2.12.3&new-version=2.13.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- go.mod | 19 +++++++++---------- go.sum | 37 ++++++++++++++++++------------------- 2 files changed, 27 insertions(+), 29 deletions(-) diff --git a/go.mod b/go.mod index 5d005836f..7a5a1a907 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/spf13/viper v1.12.0 github.com/stretchr/testify v1.7.2 github.com/tendermint/tm-db v0.6.6 - golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 + golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2 golang.org/x/sync v0.0.0-20220513210516-0976fa681c29 google.golang.org/grpc v1.47.0 @@ -42,7 +42,7 @@ require ( github.com/creachadair/taskgroup v0.3.2 github.com/golangci/golangci-lint v1.46.0 github.com/google/go-cmp v0.5.8 - github.com/vektra/mockery/v2 v2.12.3 + github.com/vektra/mockery/v2 v2.13.0 gotest.tools v2.2.0+incompatible ) @@ -169,7 +169,7 @@ require ( github.com/opencontainers/image-spec v1.0.2 // indirect github.com/opencontainers/runc v1.0.3 // indirect github.com/pelletier/go-toml v1.9.5 // indirect - github.com/pelletier/go-toml/v2 v2.0.1 // indirect + github.com/pelletier/go-toml/v2 v2.0.2 // indirect github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d // indirect github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/pkg/errors v0.9.1 // indirect @@ -199,7 +199,7 @@ require ( github.com/ssgreg/nlreturn/v2 v2.2.1 // indirect github.com/stbenjam/no-sprintf-host-port v0.1.1 // indirect github.com/stretchr/objx v0.1.1 // indirect - github.com/subosito/gotenv v1.3.0 // indirect + github.com/subosito/gotenv v1.4.0 // indirect github.com/sylvia7788/contextcheck v1.0.4 // indirect github.com/tdakkota/asciicheck v0.1.1 // indirect github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect @@ -219,15 +219,14 @@ require ( go.uber.org/multierr v1.8.0 // indirect go.uber.org/zap v1.21.0 // indirect golang.org/x/exp/typeparams v0.0.0-20220218215828-6cf2b201936e // indirect - golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect - golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect - golang.org/x/term v0.0.0-20220411215600-e5f449aeb171 // indirect + golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect + golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d // indirect + golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 // indirect golang.org/x/text v0.3.7 // indirect - golang.org/x/tools v0.1.11-0.20220316014157-77aa08bb151a // indirect - golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect + golang.org/x/tools v0.1.11 // indirect google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd // indirect google.golang.org/protobuf v1.28.0 // indirect - gopkg.in/ini.v1 v1.66.4 // indirect + gopkg.in/ini.v1 v1.66.6 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect honnef.co/go/tools v0.3.1 // indirect diff --git a/go.sum b/go.sum index bf2c4f067..297b27a74 100644 --- a/go.sum +++ b/go.sum @@ -874,8 +874,9 @@ github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3v github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.0.0-beta.8/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= github.com/pelletier/go-toml/v2 v2.0.0/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= -github.com/pelletier/go-toml/v2 v2.0.1 h1:8e3L2cCQzLFi2CR4g7vGFuFxX7Jl1kKX8gW+iV0GUKU= github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= +github.com/pelletier/go-toml/v2 v2.0.2 h1:+jQXlF3scKIcSEKkdHzXhCTDLPFi5r1wnK6yPS+49Gw= +github.com/pelletier/go-toml/v2 v2.0.2/go.mod h1:MovirKjgVRESsAvNZlAjtFwV867yGuwRkXbG66OzopI= github.com/performancecopilot/speed/v4 v4.0.0/go.mod h1:qxrSyuDGrTOWfV+uKRFhfxw6h/4HXRGUiZiufxo49BM= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d h1:CdDQnGF8Nq9ocOS/xlSptM1N3BbrA6/kmaep5ggwaIA= @@ -966,7 +967,6 @@ github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/cors v1.8.2 h1:KCooALfAYGs415Cwu5ABvv9n9509fSiG5SQJn/AQo4U= github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.26.1/go.mod h1:/wSSJWX7lVrsOwlbyTRSOJvqRlc+WjWlfes+CiJ+tmc= github.com/rs/zerolog v1.27.0 h1:1T7qCieN22GVc8S4Q2yuexzBb1EqjbgjSH9RohbMjKs= github.com/rs/zerolog v1.27.0/go.mod h1:7frBqO0oezxmnO7GF86FY++uy8I0Tk/If5ni1G9Qc0U= github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= @@ -980,7 +980,6 @@ github.com/ryanrolds/sqlclosecheck v0.3.0 h1:AZx+Bixh8zdUBxUA1NxbxVAS78vTPq4rCb8 github.com/ryanrolds/sqlclosecheck v0.3.0/go.mod h1:1gREqxyTGR3lVtpngyFo3hZAgk0KCtEdgEkHwDbigdA= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= -github.com/sagikazarmark/crypt v0.4.0/go.mod h1:ALv2SRj7GxYV4HO9elxH9nS6M9gW+xDNxqmyJ6RfDFM= github.com/sagikazarmark/crypt v0.5.0/go.mod h1:l+nzl7KWh51rpzp2h7t4MZWyiEWdhNpOAnclKvg+mdA= github.com/sagikazarmark/crypt v0.6.0/go.mod h1:U8+INwJo3nBv1m6A/8OBXAq7Jnpspk5AxSgDyEQcea8= github.com/sanposhiho/wastedassign/v2 v2.0.6 h1:+6/hQIHKNJAUixEj6EmOngGIisyeI+T3335lYTyxRoA= @@ -1022,7 +1021,6 @@ github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/afero v1.8.0/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo= github.com/spf13/afero v1.8.2 h1:xehSyVa0YnHWsJ49JFljMpg1HX19V6NDZ1fkm1Xznbo= github.com/spf13/afero v1.8.2/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= @@ -1045,7 +1043,6 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= -github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU= github.com/spf13/viper v1.11.0/go.mod h1:djo0X/bA5+tYVoCn+C7cAYJGcVn/qYLFTG8gdUsX7Zk= github.com/spf13/viper v1.12.0 h1:CZ7eSOd3kZoaYDLbXnmzgQI5RlciuXBMA+18HwHRfZQ= github.com/spf13/viper v1.12.0/go.mod h1:b6COn30jlNxbm/V2IqWiNWkJ+vZNiMNksliPCiuKtSI= @@ -1071,8 +1068,9 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/subosito/gotenv v1.3.0 h1:mjC+YW8QpAdXibNi+vNWgzmgBH4+5l5dCXv8cNysBLI= github.com/subosito/gotenv v1.3.0/go.mod h1:YzJjq/33h7nrwdY+iHMhEOEEbW0ovIz0tB6t6PwAXzs= +github.com/subosito/gotenv v1.4.0 h1:yAzM1+SmVcz5R4tXGsNMu1jUl2aOJXoiWUCEwwnGrvs= +github.com/subosito/gotenv v1.4.0/go.mod h1:mZd6rFysKEcUhUHXJk0C/08wAgyDBFuwEYL7vWWGaGo= github.com/sylvia7788/contextcheck v1.0.4 h1:MsiVqROAdr0efZc/fOCt0c235qm9XJqHtWwM+2h2B04= github.com/sylvia7788/contextcheck v1.0.4/go.mod h1:vuPKJMQ7MQ91ZTqfdyreNKwZjyUg6KO+IebVyQDedZQ= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= @@ -1118,8 +1116,8 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC github.com/valyala/fasthttp v1.30.0/go.mod h1:2rsYD01CKFrjjsvFxx75KlEUNpWNBY9JWD3K/7o2Cus= github.com/valyala/quicktemplate v1.7.0/go.mod h1:sqKJnoaOF88V07vkO+9FL8fb9uZg/VPSJnLYn+LmLk8= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= -github.com/vektra/mockery/v2 v2.12.3 h1:74h0R+p75tdr3QNwiNz3MXeCwSP/I5bYUbZY6oT4t20= -github.com/vektra/mockery/v2 v2.12.3/go.mod h1:8vf4KDDUptfkyypzdHLuE7OE2xA7Gdt60WgIS8PgD+U= +github.com/vektra/mockery/v2 v2.13.0 h1:jzHQuiWMbLK52usAz/3wyIf07gZnACOsTJ8/AcHA/2s= +github.com/vektra/mockery/v2 v2.13.0/go.mod h1:bnD1T8tExSgPD1ripLkDbr60JA9VtQeu12P3wgLZd7M= github.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8/go.mod h1:dniwbG03GafCjFohMDmz6Zc6oCuiqgH6tGNyXTkHzXE= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= @@ -1221,11 +1219,10 @@ golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210915214749-c084706c2272/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= -golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220313003712-b769efc7c000/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 h1:kUhD7nTDoI3fVd9G4ORWrbV5NY0liEs/Jg2pv5f+bBA= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM= +golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1271,8 +1268,9 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= -golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1487,13 +1485,15 @@ golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d h1:Zu/JngovGLVi6t2J3nmAf3AoTDwuzw85YZ3b9o4yU7s= +golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.0.0-20220411215600-e5f449aeb171 h1:EH1Deb8WZJ0xc0WK//leUHXcX9aLE5SymusoTmMZye8= golang.org/x/term v0.0.0-20220411215600-e5f449aeb171/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 h1:CBpWXWQpIRjzmkkA+M7q9Fqnwd2mZr3AFqexg8YTfoM= +golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1611,14 +1611,14 @@ golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.9-0.20211228192929-ee1ca4ffc4da/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= -golang.org/x/tools v0.1.11-0.20220316014157-77aa08bb151a h1:ofrrl6c6NG5/IOSx/R1cyiQxxjqlur0h/TvbUhkH0II= golang.org/x/tools v0.1.11-0.20220316014157-77aa08bb151a/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= +golang.org/x/tools v0.1.11 h1:loJ25fNOEhSXfHrpoGj91eCUThwdNX6u24rO1xnNteY= +golang.org/x/tools v0.1.11/go.mod h1:SgwaegtQh8clINPpECJMqnxLv9I09HLqnW3RMqW0CA4= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df h1:5Pf6pFKu98ODmgnpvkJ3kFUOQGGLIzLIkbzUHp47618= golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= @@ -1793,7 +1793,6 @@ google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= @@ -1830,9 +1829,9 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.66.3/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.66.4 h1:SsAcf+mM7mRZo2nJNGt8mZCjG8ZRaNGMURJw7BsIST4= gopkg.in/ini.v1 v1.66.4/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.66.6 h1:LATuAqN/shcYAOkv3wl2L4rkaKqkcgTBQjOyYDvcPKI= +gopkg.in/ini.v1 v1.66.6/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= From 7971f4a2fca1ee3054fbc996e2bc041de63d3b89 Mon Sep 17 00:00:00 2001 From: Sam Kleinman Date: Tue, 14 Jun 2022 12:45:05 -0400 Subject: [PATCH 09/61] p2p: self-add node should not error (#8753) --- internal/p2p/peermanager.go | 12 +++++++++++- internal/p2p/peermanager_test.go | 14 ++++++++------ node/node.go | 2 +- node/seed.go | 2 +- node/setup.go | 2 ++ 5 files changed, 23 insertions(+), 9 deletions(-) diff --git a/internal/p2p/peermanager.go b/internal/p2p/peermanager.go index 7391de4ea..65741f63f 100644 --- a/internal/p2p/peermanager.go +++ b/internal/p2p/peermanager.go @@ -15,6 +15,7 @@ import ( dbm "github.com/tendermint/tm-db" tmsync "github.com/tendermint/tendermint/internal/libs/sync" + "github.com/tendermint/tendermint/libs/log" p2pproto "github.com/tendermint/tendermint/proto/tendermint/p2p" "github.com/tendermint/tendermint/types" ) @@ -145,6 +146,8 @@ type PeerManagerOptions struct { // persistentPeers provides fast PersistentPeers lookups. It is built // by optimize(). persistentPeers map[types.NodeID]bool + + Logger log.Logger } // Validate validates the options. @@ -264,6 +267,7 @@ type PeerManager struct { rand *rand.Rand dialWaker *tmsync.Waker // wakes up DialNext() on relevant peer changes evictWaker *tmsync.Waker // wakes up EvictNext() on relevant peer changes + logger log.Logger mtx sync.Mutex store *peerStore @@ -298,6 +302,7 @@ func NewPeerManager(selfID types.NodeID, peerDB dbm.DB, options PeerManagerOptio rand: rand.New(rand.NewSource(time.Now().UnixNano())), // nolint:gosec dialWaker: tmsync.NewWaker(), evictWaker: tmsync.NewWaker(), + logger: log.NewNopLogger(), store: store, dialing: map[types.NodeID]bool{}, @@ -308,6 +313,11 @@ func NewPeerManager(selfID types.NodeID, peerDB dbm.DB, options PeerManagerOptio evicting: map[types.NodeID]bool{}, subscriptions: map[*PeerUpdates]*PeerUpdates{}, } + + if options.Logger != nil { + peerManager.logger = options.Logger + } + if err = peerManager.configurePeers(); err != nil { return nil, err } @@ -390,7 +400,7 @@ func (m *PeerManager) Add(address NodeAddress) (bool, error) { return false, err } if address.NodeID == m.selfID { - return false, fmt.Errorf("can't add self (%v) to peer store", m.selfID) + return false, nil } m.mtx.Lock() diff --git a/internal/p2p/peermanager_test.go b/internal/p2p/peermanager_test.go index 47e8462a4..bb79fe771 100644 --- a/internal/p2p/peermanager_test.go +++ b/internal/p2p/peermanager_test.go @@ -265,8 +265,9 @@ func TestPeerManager_Add(t *testing.T) { require.Error(t, err) // Adding self should error - _, err = peerManager.Add(p2p.NodeAddress{Protocol: "memory", NodeID: selfID}) - require.Error(t, err) + ok, err := peerManager.Add(p2p.NodeAddress{Protocol: "memory", NodeID: selfID}) + require.False(t, ok) + require.NoError(t, err) } func TestPeerManager_DialNext(t *testing.T) { @@ -841,13 +842,14 @@ func TestPeerManager_Dialed_Connected(t *testing.T) { require.Error(t, peerManager.Dialed(b)) } -func TestPeerManager_Dialed_Self(t *testing.T) { +func TestPeerManager_Adding_Self(t *testing.T) { peerManager, err := p2p.NewPeerManager(selfID, dbm.NewMemDB(), p2p.PeerManagerOptions{}) require.NoError(t, err) - // Dialing self should error. - _, err = peerManager.Add(p2p.NodeAddress{Protocol: "memory", NodeID: selfID}) - require.Error(t, err) + // Ingesting self should not error. + ok, err := peerManager.Add(p2p.NodeAddress{Protocol: "memory", NodeID: selfID}) + require.False(t, ok) + require.NoError(t, err) } func TestPeerManager_Dialed_MaxConnected(t *testing.T) { diff --git a/node/node.go b/node/node.go index 1bda1f0f7..f2c4cd6a8 100644 --- a/node/node.go +++ b/node/node.go @@ -203,7 +203,7 @@ func makeNode( } } - peerManager, peerCloser, err := createPeerManager(cfg, dbProvider, nodeKey.ID) + peerManager, peerCloser, err := createPeerManager(logger, cfg, dbProvider, nodeKey.ID) closers = append(closers, peerCloser) if err != nil { return nil, combineCloseError( diff --git a/node/seed.go b/node/seed.go index a0b71e411..3ba4d86d4 100644 --- a/node/seed.go +++ b/node/seed.go @@ -67,7 +67,7 @@ func makeSeedNode( // Setup Transport and Switch. p2pMetrics := p2p.PrometheusMetrics(cfg.Instrumentation.Namespace, "chain_id", genDoc.ChainID) - peerManager, closer, err := createPeerManager(cfg, dbProvider, nodeKey.ID) + peerManager, closer, err := createPeerManager(logger, cfg, dbProvider, nodeKey.ID) if err != nil { return nil, combineCloseError( fmt.Errorf("failed to create peer manager: %w", err), diff --git a/node/setup.go b/node/setup.go index 51a048249..3240ba0e7 100644 --- a/node/setup.go +++ b/node/setup.go @@ -199,6 +199,7 @@ func createEvidenceReactor( } func createPeerManager( + logger log.Logger, cfg *config.Config, dbProvider config.DBProvider, nodeID types.NodeID, @@ -226,6 +227,7 @@ func createPeerManager( maxUpgradeConns := uint16(4) options := p2p.PeerManagerOptions{ + Logger: logger.With("module", "peermanager"), SelfAddress: selfAddr, MaxConnected: maxConns, MaxConnectedUpgrade: maxUpgradeConns, From bf1cb89bb7c34d75e9aade09f7304fd376ce084b Mon Sep 17 00:00:00 2001 From: Sam Kleinman Date: Tue, 14 Jun 2022 16:55:10 -0400 Subject: [PATCH 10/61] Revert "p2p: self-add node should not error (tendermint#8753)" (#8757) --- internal/p2p/peermanager.go | 12 +----------- internal/p2p/peermanager_test.go | 14 ++++++-------- node/node.go | 2 +- node/seed.go | 2 +- node/setup.go | 2 -- 5 files changed, 9 insertions(+), 23 deletions(-) diff --git a/internal/p2p/peermanager.go b/internal/p2p/peermanager.go index 65741f63f..7391de4ea 100644 --- a/internal/p2p/peermanager.go +++ b/internal/p2p/peermanager.go @@ -15,7 +15,6 @@ import ( dbm "github.com/tendermint/tm-db" tmsync "github.com/tendermint/tendermint/internal/libs/sync" - "github.com/tendermint/tendermint/libs/log" p2pproto "github.com/tendermint/tendermint/proto/tendermint/p2p" "github.com/tendermint/tendermint/types" ) @@ -146,8 +145,6 @@ type PeerManagerOptions struct { // persistentPeers provides fast PersistentPeers lookups. It is built // by optimize(). persistentPeers map[types.NodeID]bool - - Logger log.Logger } // Validate validates the options. @@ -267,7 +264,6 @@ type PeerManager struct { rand *rand.Rand dialWaker *tmsync.Waker // wakes up DialNext() on relevant peer changes evictWaker *tmsync.Waker // wakes up EvictNext() on relevant peer changes - logger log.Logger mtx sync.Mutex store *peerStore @@ -302,7 +298,6 @@ func NewPeerManager(selfID types.NodeID, peerDB dbm.DB, options PeerManagerOptio rand: rand.New(rand.NewSource(time.Now().UnixNano())), // nolint:gosec dialWaker: tmsync.NewWaker(), evictWaker: tmsync.NewWaker(), - logger: log.NewNopLogger(), store: store, dialing: map[types.NodeID]bool{}, @@ -313,11 +308,6 @@ func NewPeerManager(selfID types.NodeID, peerDB dbm.DB, options PeerManagerOptio evicting: map[types.NodeID]bool{}, subscriptions: map[*PeerUpdates]*PeerUpdates{}, } - - if options.Logger != nil { - peerManager.logger = options.Logger - } - if err = peerManager.configurePeers(); err != nil { return nil, err } @@ -400,7 +390,7 @@ func (m *PeerManager) Add(address NodeAddress) (bool, error) { return false, err } if address.NodeID == m.selfID { - return false, nil + return false, fmt.Errorf("can't add self (%v) to peer store", m.selfID) } m.mtx.Lock() diff --git a/internal/p2p/peermanager_test.go b/internal/p2p/peermanager_test.go index bb79fe771..47e8462a4 100644 --- a/internal/p2p/peermanager_test.go +++ b/internal/p2p/peermanager_test.go @@ -265,9 +265,8 @@ func TestPeerManager_Add(t *testing.T) { require.Error(t, err) // Adding self should error - ok, err := peerManager.Add(p2p.NodeAddress{Protocol: "memory", NodeID: selfID}) - require.False(t, ok) - require.NoError(t, err) + _, err = peerManager.Add(p2p.NodeAddress{Protocol: "memory", NodeID: selfID}) + require.Error(t, err) } func TestPeerManager_DialNext(t *testing.T) { @@ -842,14 +841,13 @@ func TestPeerManager_Dialed_Connected(t *testing.T) { require.Error(t, peerManager.Dialed(b)) } -func TestPeerManager_Adding_Self(t *testing.T) { +func TestPeerManager_Dialed_Self(t *testing.T) { peerManager, err := p2p.NewPeerManager(selfID, dbm.NewMemDB(), p2p.PeerManagerOptions{}) require.NoError(t, err) - // Ingesting self should not error. - ok, err := peerManager.Add(p2p.NodeAddress{Protocol: "memory", NodeID: selfID}) - require.False(t, ok) - require.NoError(t, err) + // Dialing self should error. + _, err = peerManager.Add(p2p.NodeAddress{Protocol: "memory", NodeID: selfID}) + require.Error(t, err) } func TestPeerManager_Dialed_MaxConnected(t *testing.T) { diff --git a/node/node.go b/node/node.go index f2c4cd6a8..1bda1f0f7 100644 --- a/node/node.go +++ b/node/node.go @@ -203,7 +203,7 @@ func makeNode( } } - peerManager, peerCloser, err := createPeerManager(logger, cfg, dbProvider, nodeKey.ID) + peerManager, peerCloser, err := createPeerManager(cfg, dbProvider, nodeKey.ID) closers = append(closers, peerCloser) if err != nil { return nil, combineCloseError( diff --git a/node/seed.go b/node/seed.go index 3ba4d86d4..a0b71e411 100644 --- a/node/seed.go +++ b/node/seed.go @@ -67,7 +67,7 @@ func makeSeedNode( // Setup Transport and Switch. p2pMetrics := p2p.PrometheusMetrics(cfg.Instrumentation.Namespace, "chain_id", genDoc.ChainID) - peerManager, closer, err := createPeerManager(logger, cfg, dbProvider, nodeKey.ID) + peerManager, closer, err := createPeerManager(cfg, dbProvider, nodeKey.ID) if err != nil { return nil, combineCloseError( fmt.Errorf("failed to create peer manager: %w", err), diff --git a/node/setup.go b/node/setup.go index 3240ba0e7..51a048249 100644 --- a/node/setup.go +++ b/node/setup.go @@ -199,7 +199,6 @@ func createEvidenceReactor( } func createPeerManager( - logger log.Logger, cfg *config.Config, dbProvider config.DBProvider, nodeID types.NodeID, @@ -227,7 +226,6 @@ func createPeerManager( maxUpgradeConns := uint16(4) options := p2p.PeerManagerOptions{ - Logger: logger.With("module", "peermanager"), SelfAddress: selfAddr, MaxConnected: maxConns, MaxConnectedUpgrade: maxUpgradeConns, From 979a6a1b13331ae90efb1c03b9dc69bc227d1909 Mon Sep 17 00:00:00 2001 From: Sam Kleinman Date: Tue, 14 Jun 2022 19:12:53 -0400 Subject: [PATCH 11/61] p2p: accept should not abort on first error (#8759) --- internal/p2p/router.go | 17 ++++--- internal/p2p/router_test.go | 96 +++++++++++++------------------------ 2 files changed, 43 insertions(+), 70 deletions(-) diff --git a/internal/p2p/router.go b/internal/p2p/router.go index 8b77541de..511fa0fb9 100644 --- a/internal/p2p/router.go +++ b/internal/p2p/router.go @@ -470,14 +470,17 @@ func (r *Router) dialSleep(ctx context.Context) { func (r *Router) acceptPeers(ctx context.Context, transport Transport) { for { conn, err := transport.Accept(ctx) - switch err { - case nil: - case io.EOF: - r.logger.Debug("stopping accept routine", "transport", transport) + switch { + case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded): + r.logger.Debug("stopping accept routine", "transport", transport, "err", "context canceled") return - default: + case errors.Is(err, io.EOF): + r.logger.Debug("stopping accept routine", "transport", transport, "err", "EOF") + return + case err != nil: + // in this case we got an error from the net.Listener. r.logger.Error("failed to accept connection", "transport", transport, "err", err) - return + continue } incomingIP := conn.RemoteEndpoint().IP @@ -489,7 +492,7 @@ func (r *Router) acceptPeers(ctx context.Context, transport Transport) { "close_err", closeErr, ) - return + continue } // Spawn a goroutine for the handshake, to avoid head-of-line blocking. diff --git a/internal/p2p/router_test.go b/internal/p2p/router_test.go index 663e6b81c..e0910fb21 100644 --- a/internal/p2p/router_test.go +++ b/internal/p2p/router_test.go @@ -442,78 +442,48 @@ func TestRouter_AcceptPeers(t *testing.T) { } } -func TestRouter_AcceptPeers_Error(t *testing.T) { - t.Cleanup(leaktest.Check(t)) +func TestRouter_AcceptPeers_Errors(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + for _, err := range []error{io.EOF, context.Canceled, context.DeadlineExceeded} { + t.Run(err.Error(), func(t *testing.T) { + t.Cleanup(leaktest.Check(t)) - // Set up a mock transport that returns an error, which should prevent - // the router from calling Accept again. - mockTransport := &mocks.Transport{} - mockTransport.On("String").Maybe().Return("mock") - mockTransport.On("Accept", mock.Anything).Once().Return(nil, errors.New("boom")) - mockTransport.On("Close").Return(nil) - mockTransport.On("Listen", mock.Anything).Return(nil) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() - // Set up and start the router. - peerManager, err := p2p.NewPeerManager(selfID, dbm.NewMemDB(), p2p.PeerManagerOptions{}) - require.NoError(t, err) + // Set up a mock transport that returns io.EOF once, which should prevent + // the router from calling Accept again. + mockTransport := &mocks.Transport{} + mockTransport.On("String").Maybe().Return("mock") + mockTransport.On("Accept", mock.Anything).Once().Return(nil, io.EOF) + mockTransport.On("Close").Return(nil) + mockTransport.On("Listen", mock.Anything).Return(nil) - router, err := p2p.NewRouter( - log.NewNopLogger(), - p2p.NopMetrics(), - selfKey, - peerManager, - func() *types.NodeInfo { return &selfInfo }, - mockTransport, - nil, - p2p.RouterOptions{}, - ) - require.NoError(t, err) + // Set up and start the router. + peerManager, err := p2p.NewPeerManager(selfID, dbm.NewMemDB(), p2p.PeerManagerOptions{}) + require.NoError(t, err) - require.NoError(t, router.Start(ctx)) - time.Sleep(time.Second) - router.Stop() + router, err := p2p.NewRouter( + log.NewNopLogger(), + p2p.NopMetrics(), + selfKey, + peerManager, + func() *types.NodeInfo { return &selfInfo }, + mockTransport, + nil, + p2p.RouterOptions{}, + ) + require.NoError(t, err) - mockTransport.AssertExpectations(t) -} + require.NoError(t, router.Start(ctx)) + time.Sleep(time.Second) + router.Stop() -func TestRouter_AcceptPeers_ErrorEOF(t *testing.T) { - t.Cleanup(leaktest.Check(t)) + mockTransport.AssertExpectations(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + }) - // Set up a mock transport that returns io.EOF once, which should prevent - // the router from calling Accept again. - mockTransport := &mocks.Transport{} - mockTransport.On("String").Maybe().Return("mock") - mockTransport.On("Accept", mock.Anything).Once().Return(nil, io.EOF) - mockTransport.On("Close").Return(nil) - mockTransport.On("Listen", mock.Anything).Return(nil) - - // Set up and start the router. - peerManager, err := p2p.NewPeerManager(selfID, dbm.NewMemDB(), p2p.PeerManagerOptions{}) - require.NoError(t, err) - - router, err := p2p.NewRouter( - log.NewNopLogger(), - p2p.NopMetrics(), - selfKey, - peerManager, - func() *types.NodeInfo { return &selfInfo }, - mockTransport, - nil, - p2p.RouterOptions{}, - ) - require.NoError(t, err) - - require.NoError(t, router.Start(ctx)) - time.Sleep(time.Second) - router.Stop() - - mockTransport.AssertExpectations(t) + } } func TestRouter_AcceptPeers_HeadOfLineBlocking(t *testing.T) { From 51b3f111dceb2ffb9ee0e756283303aba608579c Mon Sep 17 00:00:00 2001 From: Sam Kleinman Date: Tue, 14 Jun 2022 19:48:48 -0400 Subject: [PATCH 12/61] p2p: fix mconn transport accept test (#8762) Fix minor test incongruency missed earlier. --- internal/p2p/router_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/p2p/router_test.go b/internal/p2p/router_test.go index e0910fb21..86af19385 100644 --- a/internal/p2p/router_test.go +++ b/internal/p2p/router_test.go @@ -455,7 +455,7 @@ func TestRouter_AcceptPeers_Errors(t *testing.T) { // the router from calling Accept again. mockTransport := &mocks.Transport{} mockTransport.On("String").Maybe().Return("mock") - mockTransport.On("Accept", mock.Anything).Once().Return(nil, io.EOF) + mockTransport.On("Accept", mock.Anything).Once().Return(nil, err) mockTransport.On("Close").Return(nil) mockTransport.On("Listen", mock.Anything).Return(nil) From f0b0f34f3f0c1e59145cb07f25a1b84f3c82d86e Mon Sep 17 00:00:00 2001 From: Roman Date: Wed, 15 Jun 2022 06:32:22 -0400 Subject: [PATCH 13/61] refactor: improve string representation for a vote against the proposal (#8745) --- types/vote.go | 19 ++++++++++----- types/vote_set.go | 4 ++-- types/vote_test.go | 60 +++++++++++++++++++++++++++++++++++++++------- 3 files changed, 67 insertions(+), 16 deletions(-) diff --git a/types/vote.go b/types/vote.go index f7006b8cd..821a63018 100644 --- a/types/vote.go +++ b/types/vote.go @@ -13,7 +13,8 @@ import ( ) const ( - nilVoteStr string = "nil-Vote" + absentVoteStr string = "Vote{absent}" + nilVoteStr string = "nil" // The maximum supported number of bytes in a vote extension. MaxVoteExtensionSize int = 1024 * 1024 @@ -189,7 +190,7 @@ func (vote *Vote) Copy() *Vote { // 10. timestamp func (vote *Vote) String() string { if vote == nil { - return nilVoteStr + return absentVoteStr } var typeString string @@ -202,16 +203,22 @@ func (vote *Vote) String() string { panic("Unknown vote type") } - return fmt.Sprintf("Vote{%v:%X %v/%02d/%v(%v) %X %X %X @ %s}", + var blockHashString string + if len(vote.BlockID.Hash) > 0 { + blockHashString = fmt.Sprintf("%X", tmbytes.Fingerprint(vote.BlockID.Hash)) + } else { + blockHashString = nilVoteStr + } + + return fmt.Sprintf("Vote{%v:%X %v/%d %s %s %X %d @ %s}", vote.ValidatorIndex, tmbytes.Fingerprint(vote.ValidatorAddress), vote.Height, vote.Round, - vote.Type, typeString, - tmbytes.Fingerprint(vote.BlockID.Hash), + blockHashString, tmbytes.Fingerprint(vote.Signature), - tmbytes.Fingerprint(vote.Extension), + len(vote.Extension), CanonicalTime(vote.Timestamp), ) } diff --git a/types/vote_set.go b/types/vote_set.go index 6d83ac85d..7ca69f302 100644 --- a/types/vote_set.go +++ b/types/vote_set.go @@ -505,7 +505,7 @@ func (voteSet *VoteSet) StringIndented(indent string) string { voteStrings := make([]string, len(voteSet.votes)) for i, vote := range voteSet.votes { if vote == nil { - voteStrings[i] = nilVoteStr + voteStrings[i] = absentVoteStr } else { voteStrings[i] = vote.String() } @@ -570,7 +570,7 @@ func (voteSet *VoteSet) voteStrings() []string { voteStrings := make([]string, len(voteSet.votes)) for i, vote := range voteSet.votes { if vote == nil { - voteStrings[i] = nilVoteStr + voteStrings[i] = absentVoteStr } else { voteStrings[i] = vote.String() } diff --git a/types/vote_test.go b/types/vote_test.go index d0819d7c4..917de2e4b 100644 --- a/types/vote_test.go +++ b/types/vote_test.go @@ -2,6 +2,7 @@ package types import ( "context" + "fmt" "testing" "time" @@ -16,6 +17,22 @@ import ( tmproto "github.com/tendermint/tendermint/proto/tendermint/types" ) +const ( + //nolint: lll + preCommitTestStr = `Vote{56789:6AF1F4111082 12345/2 Precommit 8B01023386C3 000000000000 0 @ 2017-12-25T03:00:01.234Z}` + //nolint: lll + preVoteTestStr = `Vote{56789:6AF1F4111082 12345/2 Prevote 8B01023386C3 000000000000 0 @ 2017-12-25T03:00:01.234Z}` +) + +var ( + // nolint: lll + nilVoteTestStr = fmt.Sprintf(`Vote{56789:6AF1F4111082 12345/2 Precommit %s 000000000000 0 @ 2017-12-25T03:00:01.234Z}`, nilVoteStr) + formatNonEmptyVoteExtensionFn = func(voteExtensionLength int) string { + // nolint: lll + return fmt.Sprintf(`Vote{56789:6AF1F4111082 12345/2 Precommit 8B01023386C3 000000000000 %d @ 2017-12-25T03:00:01.234Z}`, voteExtensionLength) + } +) + func examplePrevote(t *testing.T) *Vote { t.Helper() return exampleVote(t, byte(tmproto.PrevoteType)) @@ -321,16 +338,43 @@ func TestVoteVerify(t *testing.T) { } func TestVoteString(t *testing.T) { - str := examplePrecommit(t).String() - expected := `Vote{56789:6AF1F4111082 12345/02/SIGNED_MSG_TYPE_PRECOMMIT(Precommit) 8B01023386C3 000000000000 000000000000 @ 2017-12-25T03:00:01.234Z}` //nolint:lll //ignore line length for tests - if str != expected { - t.Errorf("got unexpected string for Vote. Expected:\n%v\nGot:\n%v", expected, str) + testcases := map[string]struct { + vote *Vote + expectedResult string + }{ + "pre-commit": { + vote: examplePrecommit(t), + expectedResult: preCommitTestStr, + }, + "pre-vote": { + vote: examplePrevote(t), + expectedResult: preVoteTestStr, + }, + "absent vote": { + expectedResult: absentVoteStr, + }, + "nil vote": { + vote: func() *Vote { + v := examplePrecommit(t) + v.BlockID.Hash = nil + return v + }(), + expectedResult: nilVoteTestStr, + }, + "non-empty vote extension": { + vote: func() *Vote { + v := examplePrecommit(t) + v.Extension = []byte{1, 2} + return v + }(), + expectedResult: formatNonEmptyVoteExtensionFn(2), + }, } - str2 := examplePrevote(t).String() - expected = `Vote{56789:6AF1F4111082 12345/02/SIGNED_MSG_TYPE_PREVOTE(Prevote) 8B01023386C3 000000000000 000000000000 @ 2017-12-25T03:00:01.234Z}` //nolint:lll //ignore line length for tests - if str2 != expected { - t.Errorf("got unexpected string for Vote. Expected:\n%v\nGot:\n%v", expected, str2) + for name, tc := range testcases { + t.Run(name, func(t *testing.T) { + require.Equal(t, tc.expectedResult, tc.vote.String()) + }) } } From 134bfefbe52d40dcbe5637bf48242e6584cdbb27 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Jun 2022 10:46:37 +0000 Subject: [PATCH 14/61] build(deps): Bump github.com/vektra/mockery/v2 from 2.13.0 to 2.13.1 (#8766) Bumps [github.com/vektra/mockery/v2](https://github.com/vektra/mockery) from 2.13.0 to 2.13.1.
Release notes

Sourced from github.com/vektra/mockery/v2's releases.

v2.13.1

Changelog

  • f04b040 Fix infinity mocking caused interface in mock
  • 9d7c819 Merge pull request #472 from grongor/fix-infinite-mocking
Commits
  • 9d7c819 Merge pull request #472 from grongor/fix-infinite-mocking
  • f04b040 Fix infinity mocking caused interface in mock
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/vektra/mockery/v2&package-manager=go_modules&previous-version=2.13.0&new-version=2.13.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7a5a1a907..2ecb3ea48 100644 --- a/go.mod +++ b/go.mod @@ -42,7 +42,7 @@ require ( github.com/creachadair/taskgroup v0.3.2 github.com/golangci/golangci-lint v1.46.0 github.com/google/go-cmp v0.5.8 - github.com/vektra/mockery/v2 v2.13.0 + github.com/vektra/mockery/v2 v2.13.1 gotest.tools v2.2.0+incompatible ) diff --git a/go.sum b/go.sum index 297b27a74..5f391f394 100644 --- a/go.sum +++ b/go.sum @@ -1116,8 +1116,8 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC github.com/valyala/fasthttp v1.30.0/go.mod h1:2rsYD01CKFrjjsvFxx75KlEUNpWNBY9JWD3K/7o2Cus= github.com/valyala/quicktemplate v1.7.0/go.mod h1:sqKJnoaOF88V07vkO+9FL8fb9uZg/VPSJnLYn+LmLk8= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= -github.com/vektra/mockery/v2 v2.13.0 h1:jzHQuiWMbLK52usAz/3wyIf07gZnACOsTJ8/AcHA/2s= -github.com/vektra/mockery/v2 v2.13.0/go.mod h1:bnD1T8tExSgPD1ripLkDbr60JA9VtQeu12P3wgLZd7M= +github.com/vektra/mockery/v2 v2.13.1 h1:Lqs7aZiC7TwZO76fJ/4Zsb3NaO4F7cuuz0mZLYeNwtQ= +github.com/vektra/mockery/v2 v2.13.1/go.mod h1:bnD1T8tExSgPD1ripLkDbr60JA9VtQeu12P3wgLZd7M= github.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8/go.mod h1:dniwbG03GafCjFohMDmz6Zc6oCuiqgH6tGNyXTkHzXE= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= From 1062ae73d63d49d37ce83177d8de6a09018b36cc Mon Sep 17 00:00:00 2001 From: Sam Kleinman Date: Wed, 15 Jun 2022 09:04:13 -0400 Subject: [PATCH 15/61] e2e/ci: add extra split to 0.36 (#8770) --- .github/workflows/e2e-nightly-36x.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e-nightly-36x.yml b/.github/workflows/e2e-nightly-36x.yml index 2067602e7..9ac797146 100644 --- a/.github/workflows/e2e-nightly-36x.yml +++ b/.github/workflows/e2e-nightly-36x.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - group: ['00', '01', '02', '03'] + group: ['00', '01', '02', '03', '04'] runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -35,7 +35,7 @@ jobs: - name: Generate testnets working-directory: test/e2e # When changing -g, also change the matrix groups above - run: ./build/generator -g 4 -d networks/nightly + run: ./build/generator -g 5 -d networks/nightly - name: Run testnets in group ${{ matrix.group }} working-directory: test/e2e From 56e329aa9e1fa19f8947288f047688abad652526 Mon Sep 17 00:00:00 2001 From: Emmanuel T Odeke Date: Wed, 15 Jun 2022 21:46:50 -0600 Subject: [PATCH 16/61] cmd/tendermint/commands/debug: guard against PID int overflows (#8764) --- cmd/tendermint/commands/debug/kill.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/cmd/tendermint/commands/debug/kill.go b/cmd/tendermint/commands/debug/kill.go index a6c1ac7d8..7755817a6 100644 --- a/cmd/tendermint/commands/debug/kill.go +++ b/cmd/tendermint/commands/debug/kill.go @@ -33,10 +33,14 @@ $ tendermint debug kill 34255 /path/to/tm-debug.zip`, Args: cobra.ExactArgs(2), RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() - pid, err := strconv.ParseInt(args[0], 10, 64) + // Using Atoi so that the size of an integer can be automatically inferred. + pid, err := strconv.Atoi(args[0]) if err != nil { return err } + if pid <= 0 { + return fmt.Errorf("PID value must be > 0; given value %q, got %d", args[0], pid) + } outFile := args[1] if outFile == "" { @@ -95,7 +99,7 @@ $ tendermint debug kill 34255 /path/to/tm-debug.zip`, } logger.Info("killing Tendermint process") - if err := killProc(int(pid), tmpDir); err != nil { + if err := killProc(pid, tmpDir); err != nil { return err } @@ -113,6 +117,9 @@ $ tendermint debug kill 34255 /path/to/tm-debug.zip`, // if the output file cannot be created or the tail command cannot be started. // An error is not returned if any subsequent syscall fails. func killProc(pid int, dir string) error { + if pid <= 0 { + return fmt.Errorf("PID must be > 0, got %d", pid) + } // pipe STDERR output from tailing the Tendermint process to a file // // NOTE: This will only work on UNIX systems. From 8854ce4e68ad719a4b231795e8252552360e7e2e Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Thu, 16 Jun 2022 18:04:10 +0200 Subject: [PATCH 17/61] e2e: reactivate network test (#8635) --- test/e2e/runner/test.go | 2 +- test/e2e/tests/net_test.go | 29 ++++++++++++++++++----------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/test/e2e/runner/test.go b/test/e2e/runner/test.go index 2237588a1..da7a4a50f 100644 --- a/test/e2e/runner/test.go +++ b/test/e2e/runner/test.go @@ -13,5 +13,5 @@ func Test(testnet *e2e.Testnet) error { return err } - return execVerbose("./build/tests", "-test.count=1", "-test.v") + return execVerbose("./build/tests", "-test.count=1") } diff --git a/test/e2e/tests/net_test.go b/test/e2e/tests/net_test.go index 71a958412..2d8f28549 100644 --- a/test/e2e/tests/net_test.go +++ b/test/e2e/tests/net_test.go @@ -7,33 +7,40 @@ import ( "github.com/stretchr/testify/require" e2e "github.com/tendermint/tendermint/test/e2e/pkg" + "github.com/tendermint/tendermint/types" ) // Tests that all nodes have peered with each other, regardless of discovery method. func TestNet_Peers(t *testing.T) { - // FIXME Skip test since nodes aren't always able to fully mesh - t.SkipNow() - testNode(t, func(ctx context.Context, t *testing.T, node e2e.Node) { client, err := node.Client() require.NoError(t, err) netInfo, err := client.NetInfo(ctx) require.NoError(t, err) - require.Equal(t, len(node.Testnet.Nodes)-1, netInfo.NPeers, - "node is not fully meshed with peers") - + expectedPeers := len(node.Testnet.Nodes) + peers := make(map[string]*e2e.Node, 0) seen := map[string]bool{} for _, n := range node.Testnet.Nodes { - seen[n.Name] = (n.Name == node.Name) // we've clearly seen ourself + // we never save light client addresses as they use RPC or ourselves + if n.Mode == e2e.ModeLight || n.Name == node.Name { + expectedPeers-- + continue + } + peers[string(types.NodeIDFromPubKey(n.NodeKey.PubKey()))] = n + seen[n.Name] = false } + + require.Equal(t, expectedPeers, netInfo.NPeers, + "node is not fully meshed with peers") + for _, peerInfo := range netInfo.Peers { - id := peerInfo.ID - peer := node.Testnet.LookupNode(string(id)) - require.NotNil(t, peer, "unknown node %v", id) + id := string(peerInfo.ID) + peer, ok := peers[id] + require.True(t, ok, "unknown node %v", id) require.Contains(t, peerInfo.URL, peer.IP.String(), "unexpected IP address for peer %v", id) - seen[string(id)] = true + seen[peer.Name] = true } for name := range seen { From 7cf09399bb834373002d06af83bb5190e7db06f2 Mon Sep 17 00:00:00 2001 From: Adolfo Olivera Date: Thu, 16 Jun 2022 15:36:58 -0300 Subject: [PATCH 18/61] Fix typo in Using Tendermint section (#8780) Modify using-tendermint.md to replace unsafe_reset_all to unsafe-reset-all . Closes #8779 . --- docs/tendermint-core/using-tendermint.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tendermint-core/using-tendermint.md b/docs/tendermint-core/using-tendermint.md index 9501c5e66..b249d2619 100644 --- a/docs/tendermint-core/using-tendermint.md +++ b/docs/tendermint-core/using-tendermint.md @@ -254,7 +254,7 @@ afford to lose all blockchain data! To reset a blockchain, stop the node and run: ```sh -tendermint unsafe_reset_all +tendermint unsafe-reset-all ``` This command will remove the data directory and reset private validator and From a4f29bfd4483d928aa9571affcf9149820d8a051 Mon Sep 17 00:00:00 2001 From: Sergio Mena Date: Thu, 16 Jun 2022 21:31:17 +0200 Subject: [PATCH 19/61] Don't check PBTS-timeliness when in replay mode (#8774) Closes #8781 Temporary fix to this issue, so that e2e tests don't fail and potentially leave other problems uncovered. --- internal/consensus/state.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/consensus/state.go b/internal/consensus/state.go index 3af775bb6..a7a0b8fed 100644 --- a/internal/consensus/state.go +++ b/internal/consensus/state.go @@ -1530,7 +1530,8 @@ func (cs *State) defaultDoPrevote(ctx context.Context, height int64, round int32 } sp := cs.state.ConsensusParams.Synchrony.SynchronyParamsOrDefaults() - if cs.Proposal.POLRound == -1 && cs.LockedRound == -1 && !cs.proposalIsTimely() { + //TODO: Remove this temporary fix when the complete solution is ready. See #8739 + if !cs.replayMode && cs.Proposal.POLRound == -1 && cs.LockedRound == -1 && !cs.proposalIsTimely() { logger.Debug("prevote step: Proposal is not timely; prevoting nil", "proposed", tmtime.Canonical(cs.Proposal.Timestamp).Format(time.RFC3339Nano), From 9e5b13725d8b1061354eb99acbd2c3c1cefe18c9 Mon Sep 17 00:00:00 2001 From: Sam Kleinman Date: Fri, 17 Jun 2022 08:02:10 -0400 Subject: [PATCH 20/61] p2p: peer store and dialing changes (#8737) --- CHANGELOG_PENDING.md | 4 + config/config.go | 8 + config/toml.go | 4 + internal/p2p/metrics.gen.go | 50 +++- internal/p2p/metrics.go | 20 +- internal/p2p/p2ptest/network.go | 1 + internal/p2p/peermanager.go | 348 ++++++++++++++++++++--- internal/p2p/peermanager_scoring_test.go | 172 ++++++++++- internal/p2p/peermanager_test.go | 74 +++-- internal/p2p/router.go | 31 +- node/node.go | 2 +- node/seed.go | 2 +- node/setup.go | 13 +- proto/tendermint/p2p/types.pb.go | 121 +++++--- proto/tendermint/p2p/types.proto | 1 + 15 files changed, 708 insertions(+), 143 deletions(-) diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 608abc06d..07e755501 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -39,6 +39,10 @@ Special thanks to external contributors on this release: - [p2p] \#7035 Remove legacy P2P routing implementation and associated configuration options. (@tychoish) - [p2p] \#7265 Peer manager reduces peer score for each failed dial attempts for peers that have not successfully dialed. (@tychoish) - [p2p] [\#7594](https://github.com/tendermint/tendermint/pull/7594) always advertise self, to enable mutual address discovery. (@altergui) + - [p2p] \#8737 Introduce "inactive" peer label to avoid re-dialing incompatible peers. (@tychoish) + - [p2p] \#8737 Increase frequency of dialing attempts to reduce latency for peer acquisition. (@tychoish) + - [p2p] \#8737 Improvements to peer scoring and sorting to gossip a greater variety of peers during PEX. (@tychoish) + - [p2p] \#8737 Track incoming and outgoing peers separately to ensure more peer slots open for incoming connections. (@tychoish) - Go API diff --git a/config/config.go b/config/config.go index d875b8569..804c5fc87 100644 --- a/config/config.go +++ b/config/config.go @@ -627,6 +627,10 @@ type P2PConfig struct { //nolint: maligned // outbound). MaxConnections uint16 `mapstructure:"max-connections"` + // MaxOutgoingConnections defines the maximum number of connected peers (inbound and + // outbound). + MaxOutgoingConnections uint16 `mapstructure:"max-outgoing-connections"` + // MaxIncomingConnectionAttempts rate limits the number of incoming connection // attempts per IP address. MaxIncomingConnectionAttempts uint `mapstructure:"max-incoming-connection-attempts"` @@ -667,6 +671,7 @@ func DefaultP2PConfig() *P2PConfig { ExternalAddress: "", UPNP: false, MaxConnections: 64, + MaxOutgoingConnections: 32, MaxIncomingConnectionAttempts: 100, FlushThrottleTimeout: 100 * time.Millisecond, // The MTU (Maximum Transmission Unit) for Ethernet is 1500 bytes. @@ -699,6 +704,9 @@ func (cfg *P2PConfig) ValidateBasic() error { if cfg.RecvRate < 0 { return errors.New("recv-rate can't be negative") } + if cfg.MaxOutgoingConnections > cfg.MaxConnections { + return errors.New("max-outgoing-connections cannot be larger than max-connections") + } return nil } diff --git a/config/toml.go b/config/toml.go index cf6635d45..0aecbc1a3 100644 --- a/config/toml.go +++ b/config/toml.go @@ -309,6 +309,10 @@ upnp = {{ .P2P.UPNP }} # Maximum number of connections (inbound and outbound). max-connections = {{ .P2P.MaxConnections }} +# Maximum number of connections reserved for outgoing +# connections. Must be less than max-connections +max-outgoing-connections = {{ .P2P.MaxOutgoingConnections }} + # Rate limits the number of incoming connection attempts per IP address. max-incoming-connection-attempts = {{ .P2P.MaxIncomingConnectionAttempts }} diff --git a/internal/p2p/metrics.gen.go b/internal/p2p/metrics.gen.go index cbfba29d9..cb215f2b6 100644 --- a/internal/p2p/metrics.gen.go +++ b/internal/p2p/metrics.gen.go @@ -14,11 +14,23 @@ func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics { labels = append(labels, labelsAndValues[i]) } return &Metrics{ - Peers: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + PeersConnected: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ Namespace: namespace, Subsystem: MetricsSubsystem, - Name: "peers", - Help: "Number of peers.", + Name: "peers_connected", + Help: "Number of peers connected.", + }, labels).With(labelsAndValues...), + PeersStored: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: MetricsSubsystem, + Name: "peers_stored", + Help: "Nomber of peers in the peer store database.", + }, labels).With(labelsAndValues...), + PeersInactivated: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: MetricsSubsystem, + Name: "peers_inactivated", + Help: "Number of inactive peers stored.", }, labels).With(labelsAndValues...), PeerReceiveBytesTotal: prometheus.NewCounterFrom(stdprometheus.CounterOpts{ Namespace: namespace, @@ -38,6 +50,30 @@ func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics { Name: "peer_pending_send_bytes", Help: "Number of bytes pending being sent to a given peer.", }, append(labels, "peer_id")).With(labelsAndValues...), + PeersConnectedSuccess: prometheus.NewCounterFrom(stdprometheus.CounterOpts{ + Namespace: namespace, + Subsystem: MetricsSubsystem, + Name: "peers_connected_success", + Help: "Number of successful connection attempts", + }, labels).With(labelsAndValues...), + PeersConnectedFailure: prometheus.NewCounterFrom(stdprometheus.CounterOpts{ + Namespace: namespace, + Subsystem: MetricsSubsystem, + Name: "peers_connected_failure", + Help: "Number of failed connection attempts", + }, labels).With(labelsAndValues...), + PeersConnectedIncoming: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: MetricsSubsystem, + Name: "peers_connected_incoming", + Help: "Number of peers connected as a result of dialing the peer.", + }, labels).With(labelsAndValues...), + PeersConnectedOutgoing: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: MetricsSubsystem, + Name: "peers_connected_outgoing", + Help: "Number of peers connected as a result of the peer dialing this node.", + }, labels).With(labelsAndValues...), RouterPeerQueueRecv: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{ Namespace: namespace, Subsystem: MetricsSubsystem, @@ -73,10 +109,16 @@ func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics { func NopMetrics() *Metrics { return &Metrics{ - Peers: discard.NewGauge(), + PeersConnected: discard.NewGauge(), + PeersStored: discard.NewGauge(), + PeersInactivated: discard.NewGauge(), PeerReceiveBytesTotal: discard.NewCounter(), PeerSendBytesTotal: discard.NewCounter(), PeerPendingSendBytes: discard.NewGauge(), + PeersConnectedSuccess: discard.NewCounter(), + PeersConnectedFailure: discard.NewCounter(), + PeersConnectedIncoming: discard.NewGauge(), + PeersConnectedOutgoing: discard.NewGauge(), RouterPeerQueueRecv: discard.NewHistogram(), RouterPeerQueueSend: discard.NewHistogram(), RouterChannelQueueSend: discard.NewHistogram(), diff --git a/internal/p2p/metrics.go b/internal/p2p/metrics.go index b45f128e5..a88aaa3f2 100644 --- a/internal/p2p/metrics.go +++ b/internal/p2p/metrics.go @@ -26,8 +26,12 @@ var ( // Metrics contains metrics exposed by this package. type Metrics struct { - // Number of peers. - Peers metrics.Gauge + // Number of peers connected. + PeersConnected metrics.Gauge + // Nomber of peers in the peer store database. + PeersStored metrics.Gauge + // Number of inactive peers stored. + PeersInactivated metrics.Gauge // Number of bytes per channel received from a given peer. PeerReceiveBytesTotal metrics.Counter `metrics_labels:"peer_id, chID, message_type"` // Number of bytes per channel sent to a given peer. @@ -35,6 +39,18 @@ type Metrics struct { // Number of bytes pending being sent to a given peer. PeerPendingSendBytes metrics.Gauge `metrics_labels:"peer_id"` + // Number of successful connection attempts + PeersConnectedSuccess metrics.Counter + // Number of failed connection attempts + PeersConnectedFailure metrics.Counter + + // Number of peers connected as a result of dialing the + // peer. + PeersConnectedIncoming metrics.Gauge + // Number of peers connected as a result of the peer dialing + // this node. + PeersConnectedOutgoing metrics.Gauge + // RouterPeerQueueRecv defines the time taken to read off of a peer's queue // before sending on the connection. //metrics:The time taken to read off of a peer's queue before sending on the connection. diff --git a/internal/p2p/p2ptest/network.go b/internal/p2p/p2ptest/network.go index 85df029d8..5c1b0a218 100644 --- a/internal/p2p/p2ptest/network.go +++ b/internal/p2p/p2ptest/network.go @@ -257,6 +257,7 @@ func (n *Network) MakeNode(ctx context.Context, t *testing.T, opts NodeOptions) RetryTimeJitter: time.Millisecond, MaxPeers: opts.MaxPeers, MaxConnected: opts.MaxConnected, + Metrics: p2p.NopMetrics(), }) require.NoError(t, err) diff --git a/internal/p2p/peermanager.go b/internal/p2p/peermanager.go index 7391de4ea..ef4011093 100644 --- a/internal/p2p/peermanager.go +++ b/internal/p2p/peermanager.go @@ -38,11 +38,18 @@ const ( PeerStatusBad PeerStatus = "bad" // peer observed as bad ) -// PeerScore is a numeric score assigned to a peer (higher is better). -type PeerScore uint8 +type peerConnectionDirection int const ( - PeerScorePersistent PeerScore = math.MaxUint8 // persistent peers + peerConnectionIncoming peerConnectionDirection = iota + 1 + peerConnectionOutgoing +) + +// PeerScore is a numeric score assigned to a peer (higher is better). +type PeerScore int16 + +const ( + PeerScorePersistent PeerScore = math.MaxInt16 // persistent peers MaxPeerScoreNotPersistent PeerScore = PeerScorePersistent - 1 ) @@ -101,6 +108,13 @@ type PeerManagerOptions struct { // outbound). 0 means no limit. MaxConnected uint16 + // MaxOutgoingConnections specifies how many outgoing + // connections a node will maintain. It must be lower than MaxConnected. If it is + // 0, then all connections can be outgoing. Once this limit is + // reached, the node will not dial peers, allowing the + // remaining peer connections to be used by incoming connections. + MaxOutgoingConnections uint16 + // MaxConnectedUpgrade is the maximum number of additional connections to // use for probing any better-scored peers to upgrade to when all connection // slots are full. 0 disables peer upgrading. @@ -145,6 +159,9 @@ type PeerManagerOptions struct { // persistentPeers provides fast PersistentPeers lookups. It is built // by optimize(). persistentPeers map[types.NodeID]bool + + // Peer Metrics + Metrics *Metrics } // Validate validates the options. @@ -193,6 +210,10 @@ func (o *PeerManagerOptions) Validate() error { } } + if o.MaxOutgoingConnections > 0 && o.MaxConnected < o.MaxOutgoingConnections { + return errors.New("cannot set MaxOutgoingConnections to a value larger than MaxConnected") + } + return nil } @@ -261,19 +282,20 @@ func (o *PeerManagerOptions) optimize() { type PeerManager struct { selfID types.NodeID options PeerManagerOptions + metrics *Metrics rand *rand.Rand dialWaker *tmsync.Waker // wakes up DialNext() on relevant peer changes evictWaker *tmsync.Waker // wakes up EvictNext() on relevant peer changes mtx sync.Mutex store *peerStore - subscriptions map[*PeerUpdates]*PeerUpdates // keyed by struct identity (address) - dialing map[types.NodeID]bool // peers being dialed (DialNext → Dialed/DialFail) - upgrading map[types.NodeID]types.NodeID // peers claimed for upgrade (DialNext → Dialed/DialFail) - connected map[types.NodeID]bool // connected peers (Dialed/Accepted → Disconnected) - ready map[types.NodeID]bool // ready peers (Ready → Disconnected) - evict map[types.NodeID]bool // peers scheduled for eviction (Connected → EvictNext) - evicting map[types.NodeID]bool // peers being evicted (EvictNext → Disconnected) + subscriptions map[*PeerUpdates]*PeerUpdates // keyed by struct identity (address) + dialing map[types.NodeID]bool // peers being dialed (DialNext → Dialed/DialFail) + upgrading map[types.NodeID]types.NodeID // peers claimed for upgrade (DialNext → Dialed/DialFail) + connected map[types.NodeID]peerConnectionDirection // connected peers (Dialed/Accepted → Disconnected) + ready map[types.NodeID]bool // ready peers (Ready → Disconnected) + evict map[types.NodeID]bool // peers scheduled for eviction (Connected → EvictNext) + evicting map[types.NodeID]bool // peers being evicted (EvictNext → Disconnected) } // NewPeerManager creates a new peer manager. @@ -298,16 +320,22 @@ func NewPeerManager(selfID types.NodeID, peerDB dbm.DB, options PeerManagerOptio rand: rand.New(rand.NewSource(time.Now().UnixNano())), // nolint:gosec dialWaker: tmsync.NewWaker(), evictWaker: tmsync.NewWaker(), + metrics: NopMetrics(), store: store, dialing: map[types.NodeID]bool{}, upgrading: map[types.NodeID]types.NodeID{}, - connected: map[types.NodeID]bool{}, + connected: map[types.NodeID]peerConnectionDirection{}, ready: map[types.NodeID]bool{}, evict: map[types.NodeID]bool{}, evicting: map[types.NodeID]bool{}, subscriptions: map[*PeerUpdates]*PeerUpdates{}, } + + if options.Metrics != nil { + peerManager.metrics = options.Metrics + } + if err = peerManager.configurePeers(); err != nil { return nil, err } @@ -368,20 +396,45 @@ func (m *PeerManager) prunePeers() error { ranked := m.store.Ranked() for i := len(ranked) - 1; i >= 0; i-- { peerID := ranked[i].ID + switch { case m.store.Size() <= int(m.options.MaxPeers): return nil case m.dialing[peerID]: - case m.connected[peerID]: + case m.isConnected(peerID): default: if err := m.store.Delete(peerID); err != nil { return err } + m.metrics.PeersStored.Add(-1) } } return nil } +func (m *PeerManager) isConnected(peerID types.NodeID) bool { + _, ok := m.connected[peerID] + return ok +} + +type connectionStats struct { + incoming uint16 + outgoing uint16 +} + +func (m *PeerManager) getConnectedInfo() connectionStats { + out := connectionStats{} + for _, direction := range m.connected { + switch direction { + case peerConnectionIncoming: + out.incoming++ + case peerConnectionOutgoing: + out.outgoing++ + } + } + return out +} + // Add adds a peer to the manager, given as an address. If the peer already // exists, the address is added to it if it isn't already present. This will push // low scoring peers out of the address book if it exceeds the maximum size. @@ -405,12 +458,17 @@ func (m *PeerManager) Add(address NodeAddress) (bool, error) { if ok { return false, nil } + if peer.Inactive { + return false, nil + } // else add the new address peer.AddressInfo[address] = &peerAddressInfo{Address: address} if err := m.store.Set(peer); err != nil { return false, err } + + m.metrics.PeersStored.Add(1) if err := m.prunePeers(); err != nil { return true, err } @@ -464,13 +522,17 @@ func (m *PeerManager) TryDialNext() (NodeAddress, error) { // We allow dialing MaxConnected+MaxConnectedUpgrade peers. Including // MaxConnectedUpgrade allows us to probe additional peers that have a // higher score than any other peers, and if successful evict it. - if m.options.MaxConnected > 0 && len(m.connected)+len(m.dialing) >= - int(m.options.MaxConnected)+int(m.options.MaxConnectedUpgrade) { + if m.options.MaxConnected > 0 && len(m.connected)+len(m.dialing) >= int(m.options.MaxConnected)+int(m.options.MaxConnectedUpgrade) { + return NodeAddress{}, nil + } + + cinfo := m.getConnectedInfo() + if m.options.MaxOutgoingConnections > 0 && cinfo.outgoing >= m.options.MaxOutgoingConnections { return NodeAddress{}, nil } for _, peer := range m.store.Ranked() { - if m.dialing[peer.ID] || m.connected[peer.ID] { + if m.dialing[peer.ID] || m.isConnected(peer.ID) { continue } @@ -503,11 +565,10 @@ func (m *PeerManager) TryDialNext() (NodeAddress, error) { // DialFailed reports a failed dial attempt. This will make the peer available // for dialing again when appropriate (possibly after a retry timeout). -// -// FIXME: This should probably delete or mark bad addresses/peers after some time. func (m *PeerManager) DialFailed(ctx context.Context, address NodeAddress) error { m.mtx.Lock() defer m.mtx.Unlock() + m.metrics.PeersConnectedFailure.Add(1) delete(m.dialing, address.NodeID) for from, to := range m.upgrading { @@ -527,6 +588,7 @@ func (m *PeerManager) DialFailed(ctx context.Context, address NodeAddress) error addressInfo.LastDialFailure = time.Now().UTC() addressInfo.DialFailures++ + if err := m.store.Set(peer); err != nil { return err } @@ -560,6 +622,8 @@ func (m *PeerManager) Dialed(address NodeAddress) error { m.mtx.Lock() defer m.mtx.Unlock() + m.metrics.PeersConnectedSuccess.Add(1) + delete(m.dialing, address.NodeID) var upgradeFromPeer types.NodeID @@ -574,12 +638,11 @@ func (m *PeerManager) Dialed(address NodeAddress) error { if address.NodeID == m.selfID { return fmt.Errorf("rejecting connection to self (%v)", address.NodeID) } - if m.connected[address.NodeID] { + if m.isConnected(address.NodeID) { return fmt.Errorf("peer %v is already connected", address.NodeID) } if m.options.MaxConnected > 0 && len(m.connected) >= int(m.options.MaxConnected) { - if upgradeFromPeer == "" || len(m.connected) >= - int(m.options.MaxConnected)+int(m.options.MaxConnectedUpgrade) { + if upgradeFromPeer == "" || len(m.connected) >= int(m.options.MaxConnected)+int(m.options.MaxConnectedUpgrade) { return fmt.Errorf("already connected to maximum number of peers") } } @@ -589,6 +652,11 @@ func (m *PeerManager) Dialed(address NodeAddress) error { return fmt.Errorf("peer %q was removed while dialing", address.NodeID) } now := time.Now().UTC() + if peer.Inactive { + m.metrics.PeersInactivated.Add(-1) + } + peer.Inactive = false + peer.LastConnected = now if addressInfo, ok := peer.AddressInfo[address]; ok { addressInfo.DialFailures = 0 @@ -611,7 +679,9 @@ func (m *PeerManager) Dialed(address NodeAddress) error { } m.evict[upgradeFromPeer] = true } - m.connected[peer.ID] = true + + m.metrics.PeersConnectedOutgoing.Add(1) + m.connected[peer.ID] = peerConnectionOutgoing m.evictWaker.Wake() return nil @@ -641,11 +711,10 @@ func (m *PeerManager) Accepted(peerID types.NodeID) error { if peerID == m.selfID { return fmt.Errorf("rejecting connection from self (%v)", peerID) } - if m.connected[peerID] { + if m.isConnected(peerID) { return fmt.Errorf("peer %q is already connected", peerID) } - if m.options.MaxConnected > 0 && - len(m.connected) >= int(m.options.MaxConnected)+int(m.options.MaxConnectedUpgrade) { + if m.options.MaxConnected > 0 && len(m.connected) >= int(m.options.MaxConnected)+int(m.options.MaxConnectedUpgrade) { return fmt.Errorf("already connected to maximum number of peers") } @@ -670,12 +739,17 @@ func (m *PeerManager) Accepted(peerID types.NodeID) error { } } + if peer.Inactive { + m.metrics.PeersInactivated.Add(-1) + } + peer.Inactive = false peer.LastConnected = time.Now().UTC() if err := m.store.Set(peer); err != nil { return err } - m.connected[peerID] = true + m.metrics.PeersConnectedIncoming.Add(1) + m.connected[peerID] = peerConnectionIncoming if upgradeFromPeer != "" { m.evict[upgradeFromPeer] = true } @@ -694,7 +768,7 @@ func (m *PeerManager) Ready(ctx context.Context, peerID types.NodeID, channels C m.mtx.Lock() defer m.mtx.Unlock() - if m.connected[peerID] { + if m.isConnected(peerID) { m.ready[peerID] = true m.broadcast(ctx, PeerUpdate{ NodeID: peerID, @@ -730,7 +804,7 @@ func (m *PeerManager) TryEvictNext() (types.NodeID, error) { // random one. for peerID := range m.evict { delete(m.evict, peerID) - if m.connected[peerID] && !m.evicting[peerID] { + if m.isConnected(peerID) && !m.evicting[peerID] { m.evicting[peerID] = true return peerID, nil } @@ -747,7 +821,7 @@ func (m *PeerManager) TryEvictNext() (types.NodeID, error) { ranked := m.store.Ranked() for i := len(ranked) - 1; i >= 0; i-- { peer := ranked[i] - if m.connected[peer.ID] && !m.evicting[peer.ID] { + if m.isConnected(peer.ID) && !m.evicting[peer.ID] { m.evicting[peer.ID] = true return peer.ID, nil } @@ -762,6 +836,13 @@ func (m *PeerManager) Disconnected(ctx context.Context, peerID types.NodeID) { m.mtx.Lock() defer m.mtx.Unlock() + switch m.connected[peerID] { + case peerConnectionIncoming: + m.metrics.PeersConnectedIncoming.Add(-1) + case peerConnectionOutgoing: + m.metrics.PeersConnectedOutgoing.Add(-1) + } + ready := m.ready[peerID] delete(m.connected, peerID) @@ -792,17 +873,34 @@ func (m *PeerManager) Errored(peerID types.NodeID, err error) { m.mtx.Lock() defer m.mtx.Unlock() - if m.connected[peerID] { + if m.isConnected(peerID) { m.evict[peerID] = true } m.evictWaker.Wake() } +// Inactivate marks a peer as inactive which means we won't attempt to +// dial this peer again. A peer can be reactivated by successfully +// dialing and connecting to the node. +func (m *PeerManager) Inactivate(peerID types.NodeID) error { + m.mtx.Lock() + defer m.mtx.Unlock() + + peer, ok := m.store.peers[peerID] + if !ok { + return nil + } + + peer.Inactive = true + m.metrics.PeersInactivated.Add(1) + return m.store.Set(*peer) +} + // Advertise returns a list of peer addresses to advertise to a peer. // -// FIXME: This is fairly naïve and only returns the addresses of the -// highest-ranked peers. +// It sorts all peers in the peer store, and assembles a list of peers +// that is most likely to include the highest priority of peers. func (m *PeerManager) Advertise(peerID types.NodeID, limit uint16) []NodeAddress { m.mtx.Lock() defer m.mtx.Unlock() @@ -815,19 +913,92 @@ func (m *PeerManager) Advertise(peerID types.NodeID, limit uint16) []NodeAddress addresses = append(addresses, m.options.SelfAddress) } - for _, peer := range m.store.Ranked() { + var numAddresses int + var totalScore int + ranked := m.store.Ranked() + seenAddresses := map[NodeAddress]struct{}{} + scores := map[types.NodeID]int{} + + // get the total number of possible addresses + for _, peer := range ranked { if peer.ID == peerID { continue } + score := int(peer.Score()) - for nodeAddr, addressInfo := range peer.AddressInfo { - if len(addresses) >= int(limit) { - return addresses + totalScore += score + scores[peer.ID] = score + for addr := range peer.AddressInfo { + if _, ok := m.options.PrivatePeers[addr.NodeID]; !ok { + numAddresses++ + } + } + } + + var attempts uint16 + var addedLastIteration bool + + // if the number of addresses is less than the number of peers + // to advertise, adjust the limit downwards + if numAddresses < int(limit) { + limit = uint16(numAddresses) + } + + // collect addresses until we have the number requested + // (limit), or we've added all known addresses, or we've tried + // at least 256 times and the last time we iterated over + // remaining addresses we added no new candidates. + for len(addresses) < int(limit) && (attempts < (limit*2) || !addedLastIteration) { + attempts++ + addedLastIteration = false + + for idx, peer := range ranked { + if peer.ID == peerID { + continue } - // only add non-private NodeIDs - if _, ok := m.options.PrivatePeers[nodeAddr.NodeID]; !ok { - addresses = append(addresses, addressInfo.Address) + if len(addresses) >= int(limit) { + break + } + + for nodeAddr, addressInfo := range peer.AddressInfo { + if len(addresses) >= int(limit) { + break + } + + // only look at each address once, by + // tracking a set of addresses seen + if _, ok := seenAddresses[addressInfo.Address]; ok { + continue + } + + // only add non-private NodeIDs + if _, ok := m.options.PrivatePeers[nodeAddr.NodeID]; !ok { + // add the peer if the total number of ranked addresses is + // will fit within the limit, or otherwise adding + // addresses based on a coin flip. + + // the coinflip is based on the score, commonly, but + // 10% of the time we'll randomly insert a "loosing" + // peer. + + // nolint:gosec // G404: Use of weak random number generator + if numAddresses <= int(limit) || rand.Intn(totalScore+1) <= scores[peer.ID]+1 || rand.Intn((idx+1)*10) <= idx+1 { + addresses = append(addresses, addressInfo.Address) + addedLastIteration = true + seenAddresses[addressInfo.Address] = struct{}{} + } + } else { + seenAddresses[addressInfo.Address] = struct{}{} + // if the number of addresses + // is the same as the limit, + // we should remove private + // addresses from the limit so + // we can still return early. + if numAddresses == int(limit) { + limit-- + } + } } } } @@ -901,8 +1072,14 @@ func (m *PeerManager) processPeerEvent(ctx context.Context, pu PeerUpdate) { switch pu.Status { case PeerStatusBad: + if m.store.peers[pu.NodeID].MutableScore == math.MinInt16 { + return + } m.store.peers[pu.NodeID].MutableScore-- case PeerStatusGood: + if m.store.peers[pu.NodeID].MutableScore == math.MaxInt16 { + return + } m.store.peers[pu.NodeID].MutableScore++ } } @@ -993,9 +1170,11 @@ func (m *PeerManager) findUpgradeCandidate(id types.NodeID, score PeerScore) typ for i := len(ranked) - 1; i >= 0; i-- { candidate := ranked[i] switch { + case candidate.ID == id: + continue case candidate.Score() >= score: return "" // no further peers can be scored lower, due to sorting - case !m.connected[candidate.ID]: + case !m.isConnected(candidate.ID): case m.evict[candidate.ID]: case m.evicting[candidate.ID]: case m.upgrading[candidate.ID] != "": @@ -1175,9 +1354,48 @@ func (s *peerStore) Ranked() []*peerInfo { s.ranked = append(s.ranked, peer) } sort.Slice(s.ranked, func(i, j int) bool { - // FIXME: If necessary, consider precomputing scores before sorting, - // to reduce the number of Score() calls. return s.ranked[i].Score() > s.ranked[j].Score() + // TODO: reevaluate more wholistic sorting, perhaps as follows: + + // // sort inactive peers after active peers + // if s.ranked[i].Inactive && !s.ranked[j].Inactive { + // return false + // } else if !s.ranked[i].Inactive && s.ranked[j].Inactive { + // return true + // } + + // iLastDialed, iLastDialSuccess := s.ranked[i].LastDialed() + // jLastDialed, jLastDialSuccess := s.ranked[j].LastDialed() + + // // sort peers who our most recent dialing attempt was + // // successful ahead of peers with recent dialing + // // failures + // switch { + // case iLastDialSuccess && jLastDialSuccess: + // // if both peers were (are?) successfully + // // connected, convey their score, but give the + // // one we dialed successfully most recently a bonus + + // iScore := s.ranked[i].Score() + // jScore := s.ranked[j].Score() + // if jLastDialed.Before(iLastDialed) { + // jScore++ + // } else { + // iScore++ + // } + + // return iScore > jScore + // case iLastDialSuccess: + // return true + // case jLastDialSuccess: + // return false + // default: + // // if both peers were not successful in their + // // most recent dialing attempt, fall back to + // // peer score. + + // return s.ranked[i].Score() > s.ranked[j].Score() + // } }) return s.ranked } @@ -1195,11 +1413,11 @@ type peerInfo struct { // These fields are ephemeral, i.e. not persisted to the database. Persistent bool - Seed bool Height int64 FixedScore PeerScore // mainly for tests MutableScore int64 // updated by router + Inactive bool } // peerInfoFromProto converts a Protobuf PeerInfo message to a peerInfo, @@ -1208,6 +1426,7 @@ func peerInfoFromProto(msg *p2pproto.PeerInfo) (*peerInfo, error) { p := &peerInfo{ ID: types.NodeID(msg.ID), AddressInfo: map[NodeAddress]*peerAddressInfo{}, + Inactive: msg.Inactive, } if msg.LastConnected != nil { p.LastConnected = *msg.LastConnected @@ -1231,6 +1450,7 @@ func (p *peerInfo) ToProto() *p2pproto.PeerInfo { msg := &p2pproto.PeerInfo{ ID: string(p.ID), LastConnected: &p.LastConnected, + Inactive: p.Inactive, } for _, addressInfo := range p.AddressInfo { msg.AddressInfo = append(msg.AddressInfo, addressInfo.ToProto()) @@ -1254,6 +1474,46 @@ func (p *peerInfo) Copy() peerInfo { return c } +// LastDialed returns when the peer was last dialed, and if that dial +// attempt was successful. If the peer was never dialed the time stamp +// is zero time. +func (p *peerInfo) LastDialed() (time.Time, bool) { + var ( + last time.Time + success bool + ) + last = last.Add(-1) // so it's after the epoch + + for _, addr := range p.AddressInfo { + if addr.LastDialFailure.Equal(addr.LastDialSuccess) { + if addr.LastDialFailure.IsZero() { + continue + } + if last.After(addr.LastDialSuccess) { + continue + } + success = true + last = addr.LastDialSuccess + } + if addr.LastDialFailure.After(last) { + success = false + last = addr.LastDialFailure + } + if addr.LastDialSuccess.After(last) || last.Equal(addr.LastDialSuccess) { + success = true + last = addr.LastDialSuccess + } + } + + // if we never modified last, then we should return it to the + // zero value + if last.Add(1).IsZero() { + return time.Time{}, success + } + + return last, success +} + // Score calculates a score for the peer. Higher-scored peers will be // preferred over lower scores. func (p *peerInfo) Score() PeerScore { @@ -1275,10 +1535,6 @@ func (p *peerInfo) Score() PeerScore { score -= int64(addr.DialFailures) } - if score <= 0 { - return 0 - } - return PeerScore(score) } diff --git a/internal/p2p/peermanager_scoring_test.go b/internal/p2p/peermanager_scoring_test.go index a45df0b72..b454da151 100644 --- a/internal/p2p/peermanager_scoring_test.go +++ b/internal/p2p/peermanager_scoring_test.go @@ -34,7 +34,7 @@ func TestPeerScoring(t *testing.T) { t.Run("Synchronous", func(t *testing.T) { // update the manager and make sure it's correct - require.EqualValues(t, 0, peerManager.Scores()[id]) + require.Zero(t, peerManager.Scores()[id]) // add a bunch of good status updates and watch things increase. for i := 1; i < 10; i++ { @@ -97,3 +97,173 @@ func TestPeerScoring(t *testing.T) { } }) } + +func makeMockPeerStore(t *testing.T, peers ...peerInfo) *peerStore { + t.Helper() + s, err := newPeerStore(dbm.NewMemDB()) + if err != nil { + t.Fatal(err) + } + for idx := range peers { + if err := s.Set(peers[idx]); err != nil { + t.Fatal(err) + } + } + return s +} + +func TestPeerRanking(t *testing.T) { + t.Run("InactiveSecond", func(t *testing.T) { + t.Skip("inactive status is not currently factored into peer rank.") + + store := makeMockPeerStore(t, + peerInfo{ID: "second", Inactive: true}, + peerInfo{ID: "first", Inactive: false}, + ) + + ranked := store.Ranked() + if len(ranked) != 2 { + t.Fatal("missing peer in ranked output") + } + if ranked[0].ID != "first" { + t.Error("inactive peer is first") + } + if ranked[1].ID != "second" { + t.Error("active peer is second") + } + }) + t.Run("ScoreOrder", func(t *testing.T) { + for _, test := range []struct { + Name string + First int64 + Second int64 + }{ + { + Name: "Mirror", + First: 100, + Second: -100, + }, + { + Name: "VeryLow", + First: 0, + Second: -100, + }, + { + Name: "High", + First: 300, + Second: 256, + }, + } { + t.Run(test.Name, func(t *testing.T) { + store := makeMockPeerStore(t, + peerInfo{ + ID: "second", + MutableScore: test.Second, + }, + peerInfo{ + ID: "first", + MutableScore: test.First, + }) + + ranked := store.Ranked() + if len(ranked) != 2 { + t.Fatal("missing peer in ranked output") + } + if ranked[0].ID != "first" { + t.Error("higher peer is first") + } + if ranked[1].ID != "second" { + t.Error("higher peer is second") + } + }) + } + }) +} + +func TestLastDialed(t *testing.T) { + t.Run("Zero", func(t *testing.T) { + p := &peerInfo{} + ts, ok := p.LastDialed() + if !ts.IsZero() { + t.Error("timestamp should be zero:", ts) + } + if ok { + t.Error("peer reported success, despite none") + } + }) + t.Run("NeverDialed", func(t *testing.T) { + p := &peerInfo{ + AddressInfo: map[NodeAddress]*peerAddressInfo{ + {NodeID: "kip"}: {}, + {NodeID: "merlin"}: {}, + }, + } + ts, ok := p.LastDialed() + if !ts.IsZero() { + t.Error("timestamp should be zero:", ts) + } + if ok { + t.Error("peer reported success, despite none") + } + }) + t.Run("Ordered", func(t *testing.T) { + base := time.Now() + for _, test := range []struct { + Name string + SuccessTime time.Time + FailTime time.Time + ExpectedSuccess bool + }{ + { + Name: "Zero", + }, + { + Name: "Success", + SuccessTime: base.Add(time.Hour), + FailTime: base, + ExpectedSuccess: true, + }, + { + Name: "Equal", + SuccessTime: base, + FailTime: base, + ExpectedSuccess: true, + }, + { + Name: "Failure", + SuccessTime: base, + FailTime: base.Add(time.Hour), + ExpectedSuccess: false, + }, + } { + t.Run(test.Name, func(t *testing.T) { + p := &peerInfo{ + AddressInfo: map[NodeAddress]*peerAddressInfo{ + {NodeID: "kip"}: {LastDialSuccess: test.SuccessTime}, + {NodeID: "merlin"}: {LastDialFailure: test.FailTime}, + }, + } + ts, ok := p.LastDialed() + if test.ExpectedSuccess && !ts.Equal(test.SuccessTime) { + if !ts.Equal(test.FailTime) { + t.Fatal("got unexpected timestamp:", ts) + } + + t.Error("last dialed time reported incorrect value:", ts) + } + if !test.ExpectedSuccess && !ts.Equal(test.FailTime) { + if !ts.Equal(test.SuccessTime) { + t.Fatal("got unexpected timestamp:", ts) + } + + t.Error("last dialed time reported incorrect value:", ts) + } + if test.ExpectedSuccess != ok { + t.Error("test reported incorrect outcome for last dialed type") + } + }) + } + + }) + +} diff --git a/internal/p2p/peermanager_test.go b/internal/p2p/peermanager_test.go index 47e8462a4..5e9c3a8a4 100644 --- a/internal/p2p/peermanager_test.go +++ b/internal/p2p/peermanager_test.go @@ -524,11 +524,11 @@ func TestPeerManager_TryDialNext_MaxConnectedUpgrade(t *testing.T) { peerManager, err := p2p.NewPeerManager(selfID, dbm.NewMemDB(), p2p.PeerManagerOptions{ PeerScores: map[types.NodeID]p2p.PeerScore{ - a.NodeID: 0, - b.NodeID: 1, - c.NodeID: 2, - d.NodeID: 3, - e.NodeID: 0, + a.NodeID: p2p.PeerScore(0), + b.NodeID: p2p.PeerScore(1), + c.NodeID: p2p.PeerScore(2), + d.NodeID: p2p.PeerScore(3), + e.NodeID: p2p.PeerScore(0), }, PersistentPeers: []types.NodeID{c.NodeID, d.NodeID}, MaxConnected: 2, @@ -581,10 +581,8 @@ func TestPeerManager_TryDialNext_MaxConnectedUpgrade(t *testing.T) { // Now, if we disconnect a, we should be allowed to dial d because we have a // free upgrade slot. + require.Error(t, peerManager.Dialed(d)) peerManager.Disconnected(ctx, a.NodeID) - dial, err = peerManager.TryDialNext() - require.NoError(t, err) - require.Equal(t, d, dial) require.NoError(t, peerManager.Dialed(d)) // However, if we disconnect b (such that only c and d are connected), we @@ -605,7 +603,7 @@ func TestPeerManager_TryDialNext_UpgradeReservesPeer(t *testing.T) { c := p2p.NodeAddress{Protocol: "memory", NodeID: types.NodeID(strings.Repeat("c", 40))} peerManager, err := p2p.NewPeerManager(selfID, dbm.NewMemDB(), p2p.PeerManagerOptions{ - PeerScores: map[types.NodeID]p2p.PeerScore{b.NodeID: 1, c.NodeID: 1}, + PeerScores: map[types.NodeID]p2p.PeerScore{b.NodeID: p2p.PeerScore(1), c.NodeID: 1}, MaxConnected: 1, MaxConnectedUpgrade: 2, }) @@ -771,7 +769,10 @@ func TestPeerManager_DialFailed_UnreservePeer(t *testing.T) { c := p2p.NodeAddress{Protocol: "memory", NodeID: types.NodeID(strings.Repeat("c", 40))} peerManager, err := p2p.NewPeerManager(selfID, dbm.NewMemDB(), p2p.PeerManagerOptions{ - PeerScores: map[types.NodeID]p2p.PeerScore{b.NodeID: 1, c.NodeID: 1}, + PeerScores: map[types.NodeID]p2p.PeerScore{ + b.NodeID: p2p.PeerScore(1), + c.NodeID: p2p.PeerScore(2), + }, MaxConnected: 1, MaxConnectedUpgrade: 2, }) @@ -887,7 +888,7 @@ func TestPeerManager_Dialed_MaxConnectedUpgrade(t *testing.T) { peerManager, err := p2p.NewPeerManager(selfID, dbm.NewMemDB(), p2p.PeerManagerOptions{ MaxConnected: 2, MaxConnectedUpgrade: 1, - PeerScores: map[types.NodeID]p2p.PeerScore{c.NodeID: 1, d.NodeID: 1}, + PeerScores: map[types.NodeID]p2p.PeerScore{c.NodeID: p2p.PeerScore(1), d.NodeID: 1}, }) require.NoError(t, err) @@ -937,7 +938,7 @@ func TestPeerManager_Dialed_Upgrade(t *testing.T) { peerManager, err := p2p.NewPeerManager(selfID, dbm.NewMemDB(), p2p.PeerManagerOptions{ MaxConnected: 1, MaxConnectedUpgrade: 2, - PeerScores: map[types.NodeID]p2p.PeerScore{b.NodeID: 1, c.NodeID: 1}, + PeerScores: map[types.NodeID]p2p.PeerScore{b.NodeID: p2p.PeerScore(1), c.NodeID: 1}, }) require.NoError(t, err) @@ -984,10 +985,10 @@ func TestPeerManager_Dialed_UpgradeEvenLower(t *testing.T) { MaxConnected: 2, MaxConnectedUpgrade: 1, PeerScores: map[types.NodeID]p2p.PeerScore{ - a.NodeID: 3, - b.NodeID: 2, - c.NodeID: 10, - d.NodeID: 1, + a.NodeID: p2p.PeerScore(3), + b.NodeID: p2p.PeerScore(2), + c.NodeID: p2p.PeerScore(10), + d.NodeID: p2p.PeerScore(1), }, }) require.NoError(t, err) @@ -1040,9 +1041,9 @@ func TestPeerManager_Dialed_UpgradeNoEvict(t *testing.T) { MaxConnected: 2, MaxConnectedUpgrade: 1, PeerScores: map[types.NodeID]p2p.PeerScore{ - a.NodeID: 1, - b.NodeID: 2, - c.NodeID: 3, + a.NodeID: p2p.PeerScore(1), + b.NodeID: p2p.PeerScore(2), + c.NodeID: p2p.PeerScore(3), }, }) require.NoError(t, err) @@ -1161,8 +1162,8 @@ func TestPeerManager_Accepted_MaxConnectedUpgrade(t *testing.T) { peerManager, err := p2p.NewPeerManager(selfID, dbm.NewMemDB(), p2p.PeerManagerOptions{ PeerScores: map[types.NodeID]p2p.PeerScore{ - c.NodeID: 1, - d.NodeID: 2, + c.NodeID: p2p.PeerScore(1), + d.NodeID: p2p.PeerScore(2), }, MaxConnected: 1, MaxConnectedUpgrade: 1, @@ -1209,8 +1210,8 @@ func TestPeerManager_Accepted_Upgrade(t *testing.T) { peerManager, err := p2p.NewPeerManager(selfID, dbm.NewMemDB(), p2p.PeerManagerOptions{ PeerScores: map[types.NodeID]p2p.PeerScore{ - b.NodeID: 1, - c.NodeID: 1, + b.NodeID: p2p.PeerScore(1), + c.NodeID: p2p.PeerScore(1), }, MaxConnected: 1, MaxConnectedUpgrade: 2, @@ -1252,8 +1253,8 @@ func TestPeerManager_Accepted_UpgradeDialing(t *testing.T) { peerManager, err := p2p.NewPeerManager(selfID, dbm.NewMemDB(), p2p.PeerManagerOptions{ PeerScores: map[types.NodeID]p2p.PeerScore{ - b.NodeID: 1, - c.NodeID: 1, + b.NodeID: p2p.PeerScore(1), + c.NodeID: p2p.PeerScore(1), }, MaxConnected: 1, MaxConnectedUpgrade: 2, @@ -1428,7 +1429,7 @@ func TestPeerManager_EvictNext_WakeOnUpgradeDialed(t *testing.T) { peerManager, err := p2p.NewPeerManager(selfID, dbm.NewMemDB(), p2p.PeerManagerOptions{ MaxConnected: 1, MaxConnectedUpgrade: 1, - PeerScores: map[types.NodeID]p2p.PeerScore{b.NodeID: 1}, + PeerScores: map[types.NodeID]p2p.PeerScore{b.NodeID: p2p.PeerScore(1)}, }) require.NoError(t, err) @@ -1469,7 +1470,9 @@ func TestPeerManager_EvictNext_WakeOnUpgradeAccepted(t *testing.T) { peerManager, err := p2p.NewPeerManager(selfID, dbm.NewMemDB(), p2p.PeerManagerOptions{ MaxConnected: 1, MaxConnectedUpgrade: 1, - PeerScores: map[types.NodeID]p2p.PeerScore{b.NodeID: 1}, + PeerScores: map[types.NodeID]p2p.PeerScore{ + b.NodeID: p2p.PeerScore(1), + }, }) require.NoError(t, err) @@ -1833,6 +1836,7 @@ func TestPeerManager_Advertise(t *testing.T) { require.NoError(t, err) require.True(t, added) + require.Len(t, peerManager.Advertise(dID, 100), 6) // d should get all addresses. require.ElementsMatch(t, []p2p.NodeAddress{ aTCP, aMem, bTCP, bMem, cTCP, cMem, @@ -1846,10 +1850,18 @@ func TestPeerManager_Advertise(t *testing.T) { // Asking for 0 addresses should return, well, 0. require.Empty(t, peerManager.Advertise(aID, 0)) - // Asking for 2 addresses should get the highest-rated ones, i.e. a. - require.ElementsMatch(t, []p2p.NodeAddress{ - aTCP, aMem, - }, peerManager.Advertise(dID, 2)) + // Asking for 2 addresses should get two addresses + // the content of the list when there are two + addrs := peerManager.Advertise(dID, 2) + require.Len(t, addrs, 2) + for _, addr := range addrs { + if dID == addr.NodeID { + t.Fatal("never advertise self") + } + if cID == addr.NodeID { + t.Fatal("should not have returned the lowest ranked peer") + } + } } func TestPeerManager_Advertise_Self(t *testing.T) { diff --git a/internal/p2p/router.go b/internal/p2p/router.go index 511fa0fb9..d7236d472 100644 --- a/internal/p2p/router.go +++ b/internal/p2p/router.go @@ -443,9 +443,12 @@ func (r *Router) filterPeersID(ctx context.Context, id types.NodeID) error { func (r *Router) dialSleep(ctx context.Context) { if r.options.DialSleep == nil { + // the connTracker (on the other side) only rate + // limits peers for dialing more than once every 10ms, + // so these numbers are safe. const ( - maxDialerInterval = 3000 - minDialerInterval = 250 + maxDialerInterval = 500 // ms + minDialerInterval = 100 // ms ) // nolint:gosec // G404: Use of weak random number generator @@ -611,7 +614,7 @@ func (r *Router) connectPeer(ctx context.Context, address NodeAddress) { case errors.Is(err, context.Canceled): return case err != nil: - r.logger.Error("failed to dial peer", "peer", address, "err", err) + r.logger.Debug("failed to dial peer", "peer", address, "err", err) if err = r.peerManager.DialFailed(ctx, address); err != nil { r.logger.Error("failed to report dial failure", "peer", address, "err", err) } @@ -633,8 +636,7 @@ func (r *Router) connectPeer(ctx context.Context, address NodeAddress) { } if err := r.runWithPeerMutex(func() error { return r.peerManager.Dialed(address) }); err != nil { - r.logger.Error("failed to dial peer", - "op", "outgoing/dialing", "peer", address.NodeID, "err", err) + r.logger.Error("failed to dial peer", "op", "outgoing/dialing", "peer", address.NodeID, "err", err) conn.Close() return } @@ -692,12 +694,13 @@ func (r *Router) dialPeer(ctx context.Context, address NodeAddress) (Connection, // Internet can't and needs a different public address. conn, err := r.transport.Dial(dialCtx, endpoint) if err != nil { - r.logger.Error("failed to dial endpoint", "peer", address.NodeID, "endpoint", endpoint, "err", err) + r.logger.Debug("failed to dial endpoint", "peer", address.NodeID, "endpoint", endpoint, "err", err) } else { r.logger.Debug("dialed peer", "peer", address.NodeID, "endpoint", endpoint) return conn, nil } } + return nil, errors.New("all endpoints failed") } @@ -724,14 +727,6 @@ func (r *Router) handshakePeer( return peerInfo, fmt.Errorf("invalid handshake NodeInfo: %w", err) } - if peerInfo.Network != nodeInfo.Network { - if err := r.peerManager.store.Delete(peerInfo.NodeID); err != nil { - return peerInfo, fmt.Errorf("problem removing peer from store from incorrect network [%s]: %w", peerInfo.Network, err) - } - - return peerInfo, fmt.Errorf("connected to peer from wrong network, %q, removed from peer store", peerInfo.Network) - } - if types.NodeIDFromPubKey(peerKey) != peerInfo.NodeID { return peerInfo, fmt.Errorf("peer's public key did not match its node ID %q (expected %q)", peerInfo.NodeID, types.NodeIDFromPubKey(peerKey)) @@ -742,6 +737,10 @@ func (r *Router) handshakePeer( } if err := nodeInfo.CompatibleWith(peerInfo); err != nil { + if err := r.peerManager.Inactivate(peerInfo.NodeID); err != nil { + return peerInfo, fmt.Errorf("problem inactivating peer %q: %w", peerInfo.ID(), err) + } + return peerInfo, ErrRejected{ err: err, id: peerInfo.ID(), @@ -761,7 +760,7 @@ func (r *Router) runWithPeerMutex(fn func() error) error { // channels. It will close the given connection and send queue when done, or if // they are closed elsewhere it will cause this method to shut down and return. func (r *Router) routePeer(ctx context.Context, peerID types.NodeID, conn Connection, channels ChannelIDSet) { - r.metrics.Peers.Add(1) + r.metrics.PeersConnected.Add(1) r.peerManager.Ready(ctx, peerID, channels) sendQueue := r.getOrMakeQueue(peerID, channels) @@ -774,7 +773,7 @@ func (r *Router) routePeer(ctx context.Context, peerID types.NodeID, conn Connec sendQueue.close() r.peerManager.Disconnected(ctx, peerID) - r.metrics.Peers.Add(-1) + r.metrics.PeersConnected.Add(-1) }() r.logger.Info("peer connected", "peer", peerID, "endpoint", conn) diff --git a/node/node.go b/node/node.go index 1bda1f0f7..77773044b 100644 --- a/node/node.go +++ b/node/node.go @@ -203,7 +203,7 @@ func makeNode( } } - peerManager, peerCloser, err := createPeerManager(cfg, dbProvider, nodeKey.ID) + peerManager, peerCloser, err := createPeerManager(cfg, dbProvider, nodeKey.ID, nodeMetrics.p2p) closers = append(closers, peerCloser) if err != nil { return nil, combineCloseError( diff --git a/node/seed.go b/node/seed.go index a0b71e411..92d55230f 100644 --- a/node/seed.go +++ b/node/seed.go @@ -67,7 +67,7 @@ func makeSeedNode( // Setup Transport and Switch. p2pMetrics := p2p.PrometheusMetrics(cfg.Instrumentation.Namespace, "chain_id", genDoc.ChainID) - peerManager, closer, err := createPeerManager(cfg, dbProvider, nodeKey.ID) + peerManager, closer, err := createPeerManager(cfg, dbProvider, nodeKey.ID, p2pMetrics) if err != nil { return nil, combineCloseError( fmt.Errorf("failed to create peer manager: %w", err), diff --git a/node/setup.go b/node/setup.go index 51a048249..345489f8d 100644 --- a/node/setup.go +++ b/node/setup.go @@ -202,6 +202,7 @@ func createPeerManager( cfg *config.Config, dbProvider config.DBProvider, nodeID types.NodeID, + metrics *p2p.Metrics, ) (*p2p.PeerManager, closer, error) { selfAddr, err := p2p.ParseNodeAddress(nodeID.AddressString(cfg.P2P.ExternalAddress)) @@ -223,18 +224,28 @@ func createPeerManager( maxConns = 64 } + var maxOutgoingConns uint16 + switch { + case cfg.P2P.MaxOutgoingConnections > 0: + maxOutgoingConns = cfg.P2P.MaxOutgoingConnections + default: + maxOutgoingConns = maxConns / 2 + } + maxUpgradeConns := uint16(4) options := p2p.PeerManagerOptions{ SelfAddress: selfAddr, MaxConnected: maxConns, + MaxOutgoingConnections: maxOutgoingConns, MaxConnectedUpgrade: maxUpgradeConns, - MaxPeers: maxUpgradeConns + 2*maxConns, + MaxPeers: maxUpgradeConns + 4*maxConns, MinRetryTime: 250 * time.Millisecond, MaxRetryTime: 30 * time.Minute, MaxRetryTimePersistent: 5 * time.Minute, RetryTimeJitter: 5 * time.Second, PrivatePeers: privatePeerIDs, + Metrics: metrics, } peers := []p2p.NodeAddress{} diff --git a/proto/tendermint/p2p/types.pb.go b/proto/tendermint/p2p/types.pb.go index bffa6884f..7965b668b 100644 --- a/proto/tendermint/p2p/types.pb.go +++ b/proto/tendermint/p2p/types.pb.go @@ -243,6 +243,7 @@ type PeerInfo struct { ID string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` AddressInfo []*PeerAddressInfo `protobuf:"bytes,2,rep,name=address_info,json=addressInfo,proto3" json:"address_info,omitempty"` LastConnected *time.Time `protobuf:"bytes,3,opt,name=last_connected,json=lastConnected,proto3,stdtime" json:"last_connected,omitempty"` + Inactive bool `protobuf:"varint,4,opt,name=inactive,proto3" json:"inactive,omitempty"` } func (m *PeerInfo) Reset() { *m = PeerInfo{} } @@ -299,6 +300,13 @@ func (m *PeerInfo) GetLastConnected() *time.Time { return nil } +func (m *PeerInfo) GetInactive() bool { + if m != nil { + return m.Inactive + } + return false +} + type PeerAddressInfo struct { Address string `protobuf:"bytes,1,opt,name=address,proto3" json:"address,omitempty"` LastDialSuccess *time.Time `protobuf:"bytes,2,opt,name=last_dial_success,json=lastDialSuccess,proto3,stdtime" json:"last_dial_success,omitempty"` @@ -378,46 +386,46 @@ func init() { func init() { proto.RegisterFile("tendermint/p2p/types.proto", fileDescriptor_c8a29e659aeca578) } var fileDescriptor_c8a29e659aeca578 = []byte{ - // 610 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0xcd, 0x4e, 0x1b, 0x3d, - 0x14, 0xcd, 0x24, 0x21, 0x09, 0x37, 0x84, 0xf0, 0x59, 0xe8, 0xd3, 0x10, 0xa9, 0x19, 0x14, 0x36, - 0xac, 0x26, 0x52, 0xaa, 0x2e, 0xba, 0x64, 0x40, 0xad, 0x22, 0x55, 0x25, 0x9a, 0xa2, 0x2e, 0xda, - 0xc5, 0x68, 0x32, 0x76, 0x82, 0xc5, 0xc4, 0xb6, 0x3c, 0x4e, 0x4b, 0xdf, 0x82, 0x37, 0xe9, 0x63, - 0x94, 0x25, 0xcb, 0xae, 0xd2, 0x6a, 0xd8, 0xf6, 0x21, 0x2a, 0xdb, 0x33, 0x40, 0xa2, 0x2e, 0xd8, - 0xf9, 0xdc, 0xe3, 0x73, 0xee, 0x8f, 0xad, 0x0b, 0x3d, 0x45, 0x18, 0x26, 0x72, 0x41, 0x99, 0x1a, - 0x8a, 0x91, 0x18, 0xaa, 0x6f, 0x82, 0x64, 0xbe, 0x90, 0x5c, 0x71, 0xb4, 0xfb, 0xc8, 0xf9, 0x62, - 0x24, 0x7a, 0xfb, 0x73, 0x3e, 0xe7, 0x86, 0x1a, 0xea, 0x93, 0xbd, 0xd5, 0xf3, 0xe6, 0x9c, 0xcf, - 0x53, 0x32, 0x34, 0x68, 0xba, 0x9c, 0x0d, 0x15, 0x5d, 0x90, 0x4c, 0xc5, 0x0b, 0x61, 0x2f, 0x0c, - 0x2e, 0xa0, 0x3b, 0xd1, 0x87, 0x84, 0xa7, 0x1f, 0x89, 0xcc, 0x28, 0x67, 0xe8, 0x00, 0x6a, 0x62, - 0x24, 0x5c, 0xe7, 0xd0, 0x39, 0xae, 0x07, 0xcd, 0x7c, 0xe5, 0xd5, 0x26, 0xa3, 0x49, 0xa8, 0x63, - 0x68, 0x1f, 0xb6, 0xa6, 0x29, 0x4f, 0xae, 0xdc, 0xaa, 0x26, 0x43, 0x0b, 0xd0, 0x1e, 0xd4, 0x62, - 0x21, 0xdc, 0x9a, 0x89, 0xe9, 0xe3, 0xe0, 0x47, 0x15, 0x5a, 0xef, 0x39, 0x26, 0x63, 0x36, 0xe3, - 0x68, 0x02, 0x7b, 0xa2, 0x48, 0x11, 0x7d, 0xb1, 0x39, 0x8c, 0x79, 0x7b, 0xe4, 0xf9, 0xeb, 0x4d, - 0xf8, 0x1b, 0xa5, 0x04, 0xf5, 0xdb, 0x95, 0x57, 0x09, 0xbb, 0x62, 0xa3, 0xc2, 0x23, 0x68, 0x32, - 0x8e, 0x49, 0x44, 0xb1, 0x29, 0x64, 0x3b, 0x80, 0x7c, 0xe5, 0x35, 0x4c, 0xc2, 0xb3, 0xb0, 0xa1, - 0xa9, 0x31, 0x46, 0x1e, 0xb4, 0x53, 0x9a, 0x29, 0xc2, 0xa2, 0x18, 0x63, 0x69, 0xaa, 0xdb, 0x0e, - 0xc1, 0x86, 0x4e, 0x30, 0x96, 0xc8, 0x85, 0x26, 0x23, 0xea, 0x2b, 0x97, 0x57, 0x6e, 0xdd, 0x90, - 0x25, 0xd4, 0x4c, 0x59, 0xe8, 0x96, 0x65, 0x0a, 0x88, 0x7a, 0xd0, 0x4a, 0x2e, 0x63, 0xc6, 0x48, - 0x9a, 0xb9, 0x8d, 0x43, 0xe7, 0x78, 0x27, 0x7c, 0xc0, 0x5a, 0xb5, 0xe0, 0x8c, 0x5e, 0x11, 0xe9, - 0x36, 0xad, 0xaa, 0x80, 0xe8, 0x35, 0x6c, 0x71, 0x75, 0x49, 0xa4, 0xdb, 0x32, 0x6d, 0xbf, 0xd8, - 0x6c, 0xbb, 0x1c, 0xd5, 0xb9, 0xbe, 0x54, 0x34, 0x6d, 0x15, 0x83, 0xcf, 0xd0, 0x59, 0x63, 0xd1, - 0x01, 0xb4, 0xd4, 0x75, 0x44, 0x19, 0x26, 0xd7, 0x66, 0x8a, 0xdb, 0x61, 0x53, 0x5d, 0x8f, 0x35, - 0x44, 0x43, 0x68, 0x4b, 0x91, 0x98, 0x76, 0x49, 0x96, 0x15, 0xa3, 0xd9, 0xcd, 0x57, 0x1e, 0x84, - 0x93, 0xd3, 0x13, 0x1b, 0x0d, 0x41, 0x8a, 0xa4, 0x38, 0x0f, 0xbe, 0x3b, 0xd0, 0x9a, 0x10, 0x22, - 0xcd, 0x33, 0xfd, 0x0f, 0x55, 0x8a, 0xad, 0x65, 0xd0, 0xc8, 0x57, 0x5e, 0x75, 0x7c, 0x16, 0x56, - 0x29, 0x46, 0x01, 0xec, 0x14, 0x8e, 0x11, 0x65, 0x33, 0xee, 0x56, 0x0f, 0x6b, 0xff, 0x7c, 0x3a, - 0x42, 0x64, 0xe1, 0xab, 0xed, 0xc2, 0x76, 0xfc, 0x08, 0xd0, 0x5b, 0xd8, 0x4d, 0xe3, 0x4c, 0x45, - 0x09, 0x67, 0x8c, 0x24, 0x8a, 0x60, 0xf3, 0x1c, 0xed, 0x51, 0xcf, 0xb7, 0xff, 0xd3, 0x2f, 0xff, - 0xa7, 0x7f, 0x51, 0xfe, 0xcf, 0xa0, 0x7e, 0xf3, 0xcb, 0x73, 0xc2, 0x8e, 0xd6, 0x9d, 0x96, 0xb2, - 0xc1, 0x1f, 0x07, 0xba, 0x1b, 0x99, 0xf4, 0xdc, 0xcb, 0x96, 0x8b, 0x81, 0x14, 0x10, 0xbd, 0x83, - 0xff, 0x4c, 0x5a, 0x4c, 0xe3, 0x34, 0xca, 0x96, 0x49, 0x52, 0x8e, 0xe5, 0x39, 0x99, 0xbb, 0x5a, - 0x7a, 0x46, 0xe3, 0xf4, 0x83, 0x15, 0xae, 0xbb, 0xcd, 0x62, 0x9a, 0x2e, 0x25, 0x79, 0x76, 0x1f, - 0x0f, 0x6e, 0x6f, 0xac, 0x10, 0x1d, 0x41, 0xe7, 0xa9, 0x51, 0x66, 0xfe, 0x60, 0x27, 0xdc, 0xc1, - 0x8f, 0x77, 0xb2, 0xe0, 0xfc, 0x36, 0xef, 0x3b, 0x77, 0x79, 0xdf, 0xf9, 0x9d, 0xf7, 0x9d, 0x9b, - 0xfb, 0x7e, 0xe5, 0xee, 0xbe, 0x5f, 0xf9, 0x79, 0xdf, 0xaf, 0x7c, 0x7a, 0x35, 0xa7, 0xea, 0x72, - 0x39, 0xf5, 0x13, 0xbe, 0x18, 0x3e, 0xd9, 0x12, 0x4f, 0x17, 0x86, 0xd9, 0x05, 0xeb, 0x1b, 0x64, - 0xda, 0x30, 0xd1, 0x97, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0x0b, 0xe9, 0x56, 0xd3, 0x5a, 0x04, - 0x00, 0x00, + // 621 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0x41, 0x4f, 0xdb, 0x30, + 0x14, 0x6e, 0xda, 0xd2, 0x96, 0x57, 0x4a, 0x99, 0x85, 0xa6, 0x50, 0x69, 0x0d, 0x2a, 0x17, 0x4e, + 0x89, 0xd4, 0x69, 0x87, 0x1d, 0x09, 0x68, 0x53, 0xa5, 0x69, 0x54, 0x1e, 0xda, 0x61, 0x3b, 0x44, + 0x69, 0xec, 0x16, 0x8b, 0xd4, 0xb6, 0x12, 0x97, 0xb1, 0x7f, 0xc1, 0xbf, 0x1a, 0xd2, 0x2e, 0x1c, + 0x77, 0xea, 0xa6, 0x70, 0xdd, 0x8f, 0x98, 0xec, 0x24, 0xd0, 0x56, 0x3b, 0x70, 0xf3, 0xf7, 0x9e, + 0xbf, 0xcf, 0xdf, 0x7b, 0xcf, 0x7a, 0xd0, 0x53, 0x94, 0x13, 0x9a, 0xcc, 0x19, 0x57, 0x9e, 0x1c, + 0x4a, 0x4f, 0x7d, 0x97, 0x34, 0x75, 0x65, 0x22, 0x94, 0x40, 0xbb, 0x4f, 0x39, 0x57, 0x0e, 0x65, + 0x6f, 0x7f, 0x26, 0x66, 0xc2, 0xa4, 0x3c, 0x7d, 0xca, 0x6f, 0xf5, 0x9c, 0x99, 0x10, 0xb3, 0x98, + 0x7a, 0x06, 0x4d, 0x16, 0x53, 0x4f, 0xb1, 0x39, 0x4d, 0x55, 0x38, 0x97, 0xf9, 0x85, 0xc1, 0x05, + 0x74, 0xc7, 0xfa, 0x10, 0x89, 0xf8, 0x33, 0x4d, 0x52, 0x26, 0x38, 0x3a, 0x80, 0x9a, 0x1c, 0x4a, + 0xdb, 0x3a, 0xb4, 0x8e, 0xeb, 0x7e, 0x33, 0x5b, 0x3a, 0xb5, 0xf1, 0x70, 0x8c, 0x75, 0x0c, 0xed, + 0xc3, 0xd6, 0x24, 0x16, 0xd1, 0x95, 0x5d, 0xd5, 0x49, 0x9c, 0x03, 0xb4, 0x07, 0xb5, 0x50, 0x4a, + 0xbb, 0x66, 0x62, 0xfa, 0x38, 0xf8, 0x51, 0x85, 0xd6, 0x47, 0x41, 0xe8, 0x88, 0x4f, 0x05, 0x1a, + 0xc3, 0x9e, 0x2c, 0x9e, 0x08, 0xae, 0xf3, 0x37, 0x8c, 0x78, 0x7b, 0xe8, 0xb8, 0xeb, 0x45, 0xb8, + 0x1b, 0x56, 0xfc, 0xfa, 0xdd, 0xd2, 0xa9, 0xe0, 0xae, 0xdc, 0x70, 0x78, 0x04, 0x4d, 0x2e, 0x08, + 0x0d, 0x18, 0x31, 0x46, 0xb6, 0x7d, 0xc8, 0x96, 0x4e, 0xc3, 0x3c, 0x78, 0x86, 0x1b, 0x3a, 0x35, + 0x22, 0xc8, 0x81, 0x76, 0xcc, 0x52, 0x45, 0x79, 0x10, 0x12, 0x92, 0x18, 0x77, 0xdb, 0x18, 0xf2, + 0xd0, 0x09, 0x21, 0x09, 0xb2, 0xa1, 0xc9, 0xa9, 0xfa, 0x26, 0x92, 0x2b, 0xbb, 0x6e, 0x92, 0x25, + 0xd4, 0x99, 0xd2, 0xe8, 0x56, 0x9e, 0x29, 0x20, 0xea, 0x41, 0x2b, 0xba, 0x0c, 0x39, 0xa7, 0x71, + 0x6a, 0x37, 0x0e, 0xad, 0xe3, 0x1d, 0xfc, 0x88, 0x35, 0x6b, 0x2e, 0x38, 0xbb, 0xa2, 0x89, 0xdd, + 0xcc, 0x59, 0x05, 0x44, 0x6f, 0x61, 0x4b, 0xa8, 0x4b, 0x9a, 0xd8, 0x2d, 0x53, 0xf6, 0xab, 0xcd, + 0xb2, 0xcb, 0x56, 0x9d, 0xeb, 0x4b, 0x45, 0xd1, 0x39, 0x63, 0xf0, 0x15, 0x3a, 0x6b, 0x59, 0x74, + 0x00, 0x2d, 0x75, 0x13, 0x30, 0x4e, 0xe8, 0x8d, 0xe9, 0xe2, 0x36, 0x6e, 0xaa, 0x9b, 0x91, 0x86, + 0xc8, 0x83, 0x76, 0x22, 0x23, 0x53, 0x2e, 0x4d, 0xd3, 0xa2, 0x35, 0xbb, 0xd9, 0xd2, 0x01, 0x3c, + 0x3e, 0x3d, 0xc9, 0xa3, 0x18, 0x12, 0x19, 0x15, 0xe7, 0xc1, 0x4f, 0x0b, 0x5a, 0x63, 0x4a, 0x13, + 0x33, 0xa6, 0x97, 0x50, 0x65, 0x24, 0x97, 0xf4, 0x1b, 0xd9, 0xd2, 0xa9, 0x8e, 0xce, 0x70, 0x95, + 0x11, 0xe4, 0xc3, 0x4e, 0xa1, 0x18, 0x30, 0x3e, 0x15, 0x76, 0xf5, 0xb0, 0xf6, 0xdf, 0xd1, 0x51, + 0x9a, 0x14, 0xba, 0x5a, 0x0e, 0xb7, 0xc3, 0x27, 0x80, 0xde, 0xc3, 0x6e, 0x1c, 0xa6, 0x2a, 0x88, + 0x04, 0xe7, 0x34, 0x52, 0x94, 0x98, 0x71, 0xb4, 0x87, 0x3d, 0x37, 0xff, 0x9f, 0x6e, 0xf9, 0x3f, + 0xdd, 0x8b, 0xf2, 0x7f, 0xfa, 0xf5, 0xdb, 0xdf, 0x8e, 0x85, 0x3b, 0x9a, 0x77, 0x5a, 0xd2, 0x74, + 0xff, 0x19, 0x0f, 0x23, 0xc5, 0xae, 0xa9, 0x19, 0x5a, 0x0b, 0x3f, 0xe2, 0xc1, 0x5f, 0x0b, 0xba, + 0x1b, 0x2e, 0xf4, 0x4c, 0xca, 0x76, 0x14, 0xcd, 0x2a, 0x20, 0xfa, 0x00, 0x2f, 0x8c, 0x25, 0xc2, + 0xc2, 0x38, 0x48, 0x17, 0x51, 0x54, 0xb6, 0xec, 0x39, 0xae, 0xba, 0x9a, 0x7a, 0xc6, 0xc2, 0xf8, + 0x53, 0x4e, 0x5c, 0x57, 0x9b, 0x86, 0x2c, 0x5e, 0x24, 0xf4, 0xd9, 0x35, 0x3e, 0xaa, 0xbd, 0xcb, + 0x89, 0xe8, 0x08, 0x3a, 0xab, 0x42, 0xa9, 0x29, 0xb5, 0x83, 0x77, 0xc8, 0xd3, 0x9d, 0xd4, 0x3f, + 0xbf, 0xcb, 0xfa, 0xd6, 0x7d, 0xd6, 0xb7, 0xfe, 0x64, 0x7d, 0xeb, 0xf6, 0xa1, 0x5f, 0xb9, 0x7f, + 0xe8, 0x57, 0x7e, 0x3d, 0xf4, 0x2b, 0x5f, 0xde, 0xcc, 0x98, 0xba, 0x5c, 0x4c, 0xdc, 0x48, 0xcc, + 0xbd, 0x95, 0x0d, 0xb2, 0xba, 0x4c, 0xcc, 0x9e, 0x58, 0xdf, 0x2e, 0x93, 0x86, 0x89, 0xbe, 0xfe, + 0x17, 0x00, 0x00, 0xff, 0xff, 0x42, 0xcb, 0x37, 0x26, 0x76, 0x04, 0x00, 0x00, } func (m *ProtocolVersion) Marshal() (dAtA []byte, err error) { @@ -600,6 +608,16 @@ func (m *PeerInfo) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.Inactive { + i-- + if m.Inactive { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x20 + } if m.LastConnected != nil { n3, err3 := github_com_gogo_protobuf_types.StdTimeMarshalTo(*m.LastConnected, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(*m.LastConnected):]) if err3 != nil { @@ -792,6 +810,9 @@ func (m *PeerInfo) Size() (n int) { l = github_com_gogo_protobuf_types.SizeOfStdTime(*m.LastConnected) n += 1 + l + sovTypes(uint64(l)) } + if m.Inactive { + n += 2 + } return n } @@ -1487,6 +1508,26 @@ func (m *PeerInfo) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Inactive", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Inactive = bool(v != 0) default: iNdEx = preIndex skippy, err := skipTypes(dAtA[iNdEx:]) diff --git a/proto/tendermint/p2p/types.proto b/proto/tendermint/p2p/types.proto index faccd59d2..a2ed6c90f 100644 --- a/proto/tendermint/p2p/types.proto +++ b/proto/tendermint/p2p/types.proto @@ -32,6 +32,7 @@ message PeerInfo { string id = 1 [(gogoproto.customname) = "ID"]; repeated PeerAddressInfo address_info = 2; google.protobuf.Timestamp last_connected = 3 [(gogoproto.stdtime) = true]; + bool inactive = 4; } message PeerAddressInfo { From 0ac03468d84147df8c23061c8cfe2f14c5695e12 Mon Sep 17 00:00:00 2001 From: Sam Kleinman Date: Fri, 17 Jun 2022 14:10:10 -0400 Subject: [PATCH 21/61] p2p: track peers stored on startup (#8787) --- internal/p2p/peermanager.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/p2p/peermanager.go b/internal/p2p/peermanager.go index ef4011093..10b95798b 100644 --- a/internal/p2p/peermanager.go +++ b/internal/p2p/peermanager.go @@ -367,6 +367,9 @@ func (m *PeerManager) configurePeers() error { } } } + + m.metrics.PeersStored.Add(float64(m.store.Size())) + return nil } From 82c1372f9e5f17a2de2eeb72aa29f092b615724e Mon Sep 17 00:00:00 2001 From: Sam Kleinman Date: Fri, 17 Jun 2022 17:22:40 -0400 Subject: [PATCH 22/61] abci+test/e2e/app: add mutex for new methods (#8577) These methods should be protected by a mutex. --- test/e2e/app/app.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/e2e/app/app.go b/test/e2e/app/app.go index 771b72643..1f66e95cc 100644 --- a/test/e2e/app/app.go +++ b/test/e2e/app/app.go @@ -342,6 +342,9 @@ func (app *Application) ApplySnapshotChunk(_ context.Context, req *abci.RequestA // total number of transaction bytes to exceed `req.MaxTxBytes`, we will not // append our special vote extension transaction. func (app *Application) PrepareProposal(_ context.Context, req *abci.RequestPrepareProposal) (*abci.ResponsePrepareProposal, error) { + app.mu.Lock() + defer app.mu.Unlock() + var sum int64 var extCount int for _, vote := range req.LocalLastCommit.Votes { @@ -423,6 +426,9 @@ func (app *Application) PrepareProposal(_ context.Context, req *abci.RequestPrep // ProcessProposal implements part of the Application interface. // It accepts any proposal that does not contain a malformed transaction. func (app *Application) ProcessProposal(_ context.Context, req *abci.RequestProcessProposal) (*abci.ResponseProcessProposal, error) { + app.mu.Lock() + defer app.mu.Unlock() + for _, tx := range req.Txs { k, v, err := parseTx(tx) if err != nil { @@ -454,6 +460,9 @@ func (app *Application) ProcessProposal(_ context.Context, req *abci.RequestProc // key/value store ("extensionSum") with the sum of all of the numbers collected // from the vote extensions. func (app *Application) ExtendVote(_ context.Context, req *abci.RequestExtendVote) (*abci.ResponseExtendVote, error) { + app.mu.Lock() + defer app.mu.Unlock() + // We ignore any requests for vote extensions that don't match our expected // next height. if req.Height != int64(app.state.Height)+1 { @@ -485,6 +494,9 @@ func (app *Application) ExtendVote(_ context.Context, req *abci.RequestExtendVot // without doing anything about them. In this case, it just makes sure that the // vote extension is a well-formed integer value. func (app *Application) VerifyVoteExtension(_ context.Context, req *abci.RequestVerifyVoteExtension) (*abci.ResponseVerifyVoteExtension, error) { + app.mu.Lock() + defer app.mu.Unlock() + // We allow vote extensions to be optional if len(req.VoteExtension) == 0 { return &abci.ResponseVerifyVoteExtension{ From 4d820ff4f5c93cf00e7618b2d3086ad23e1bb5de Mon Sep 17 00:00:00 2001 From: Sam Kleinman Date: Fri, 17 Jun 2022 18:27:38 -0400 Subject: [PATCH 23/61] p2p: peer score should not wrap around (#8790) --- internal/p2p/peermanager.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/p2p/peermanager.go b/internal/p2p/peermanager.go index 10b95798b..a5438b9ec 100644 --- a/internal/p2p/peermanager.go +++ b/internal/p2p/peermanager.go @@ -1538,6 +1538,10 @@ func (p *peerInfo) Score() PeerScore { score -= int64(addr.DialFailures) } + if score < math.MinInt16 { + score = math.MinInt16 + } + return PeerScore(score) } From e3e162ff10a169c0a16f9c5723580fa51f2668b1 Mon Sep 17 00:00:00 2001 From: Ian Jungyong Um <31336310+code0xff@users.noreply.github.com> Date: Mon, 20 Jun 2022 00:26:25 +0900 Subject: [PATCH 24/61] p2p: fix typo (#8793) Fix the typo in router --- internal/p2p/router.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/p2p/router.go b/internal/p2p/router.go index d7236d472..55dc73720 100644 --- a/internal/p2p/router.go +++ b/internal/p2p/router.go @@ -417,7 +417,7 @@ func (r *Router) routeChannel( } } -func (r *Router) numConccurentDials() int { +func (r *Router) numConcurrentDials() int { if r.options.NumConcurrentDials == nil { return runtime.NumCPU() } @@ -564,7 +564,7 @@ func (r *Router) dialPeers(ctx context.Context) { // able to add peers at a reasonable pace, though the number // is somewhat arbitrary. The action is further throttled by a // sleep after sending to the addresses channel. - for i := 0; i < r.numConccurentDials(); i++ { + for i := 0; i < r.numConcurrentDials(); i++ { wg.Add(1) go func() { defer wg.Done() From 2382b5c364e3ab4915d07fbec2bb4769fa3c4688 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jun 2022 08:54:09 +0000 Subject: [PATCH 25/61] build(deps): Bump github.com/adlio/schema from 1.3.0 to 1.3.3 (#8798) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/adlio/schema](https://github.com/adlio/schema) from 1.3.0 to 1.3.3.
Release notes

Sourced from github.com/adlio/schema's releases.

v1.3.3

Full Changelog: https://github.com/adlio/schema/compare/v1.3.1...v1.3.3

v1.3.1

What's Changed

Full Changelog: https://github.com/adlio/schema/compare/v1.3.0...v1.3.1

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/adlio/schema&package-manager=go_modules&previous-version=1.3.0&new-version=1.3.3)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- go.mod | 13 ++++---- go.sum | 102 +++++++++++++++++++++++++++++++++------------------------ 2 files changed, 66 insertions(+), 49 deletions(-) diff --git a/go.mod b/go.mod index 2ecb3ea48..227e0f5cb 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.17 require ( github.com/BurntSushi/toml v1.1.0 - github.com/adlio/schema v1.3.0 + github.com/adlio/schema v1.3.3 github.com/btcsuite/btcd v0.22.1 github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce github.com/fortytw2/leaktest v1.3.0 @@ -30,7 +30,7 @@ require ( github.com/stretchr/testify v1.7.2 github.com/tendermint/tm-db v0.6.6 golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e - golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2 + golang.org/x/net v0.0.0-20220617184016-355a448f1bc9 golang.org/x/sync v0.0.0-20220513210516-0976fa681c29 google.golang.org/grpc v1.47.0 pgregory.net/rapid v0.4.7 @@ -55,7 +55,7 @@ require ( github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24 // indirect github.com/GaijinEntertainment/go-exhaustruct/v2 v2.1.0 // indirect github.com/Masterminds/semver v1.5.0 // indirect - github.com/Microsoft/go-winio v0.5.1 // indirect + github.com/Microsoft/go-winio v0.5.2 // indirect github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect github.com/OpenPeeDeeP/depguard v1.1.0 // indirect github.com/alexkohler/prealloc v1.0.0 // indirect @@ -73,7 +73,7 @@ require ( github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/charithe/durationcheck v0.0.9 // indirect github.com/chavacava/garif v0.0.0-20220316182200-5cad0b5181d4 // indirect - github.com/containerd/continuity v0.2.1 // indirect + github.com/containerd/continuity v0.3.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect github.com/daixiang0/gci v0.3.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -123,6 +123,7 @@ require ( github.com/gostaticanalysis/comment v1.4.2 // indirect github.com/gostaticanalysis/forcetypeassert v0.1.0 // indirect github.com/gostaticanalysis/nilerr v0.1.1 // indirect + github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-version v1.4.0 // indirect @@ -167,7 +168,7 @@ require ( github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.2 // indirect - github.com/opencontainers/runc v1.0.3 // indirect + github.com/opencontainers/runc v1.1.3 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.0.2 // indirect github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d // indirect @@ -220,7 +221,7 @@ require ( go.uber.org/zap v1.21.0 // indirect golang.org/x/exp/typeparams v0.0.0-20220218215828-6cf2b201936e // indirect golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect - golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d // indirect + golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c // indirect golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/tools v0.1.11 // indirect diff --git a/go.sum b/go.sum index 5f391f394..8ee6ee311 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,5 @@ 4d63.com/gochecknoglobals v0.1.0 h1:zeZSRqj5yCg28tCkIV/z/lWbwvNm5qnKVS15PI8nhD0= 4d63.com/gochecknoglobals v0.1.0/go.mod h1:wfdC5ZjKSPr7CybKEcgJhUOgeAQW1+7WcyK8OvUilfo= -bazil.org/fuse v0.0.0-20200407214033-5883e5a4b512/go.mod h1:FbcW6z/2VytnFDhZfumh8Ss8zxHE6qpMP5sHTRe0EaM= bitbucket.org/creachadair/shell v0.0.6/go.mod h1:8Qqi/cYk7vPnsOePHroKXDJYmb5x7ENhtiFtfZq8K+M= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= @@ -70,6 +69,7 @@ github.com/Antonboom/nilnil v0.1.1/go.mod h1:L1jBqoWM7AOeTD+tSquifKSesRHs4ZdaxvZ github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw= github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0= github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2TzfVZ1pCb5vxm4BtZPUdYWe/Xo8= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -95,8 +95,8 @@ github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3Q github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/sprig v2.15.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= -github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6FgY= -github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= @@ -107,8 +107,8 @@ github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWX github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/VividCortex/gohistogram v1.0.0 h1:6+hBz+qvs0JOrrNhhmR7lFxo5sINxBCGXrdtl/UvroE= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= -github.com/adlio/schema v1.3.0 h1:eSVYLxYWbm/6ReZBCkLw4Fz7uqC+ZNoPvA39bOwi52A= -github.com/adlio/schema v1.3.0/go.mod h1:51QzxkpeFs6lRY11kPye26IaFPOV+HqEj01t5aXXKfs= +github.com/adlio/schema v1.3.3 h1:oBJn8I02PyTB466pZO1UZEn1TV5XLlifBSyMrmHl/1I= +github.com/adlio/schema v1.3.3/go.mod h1:1EsRssiv9/Ce2CMzq5DoL7RiMshhuigQxrR4DMV9fHg= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= @@ -150,7 +150,6 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bkielbasa/cyclop v1.2.0 h1:7Jmnh0yL2DjKfw28p86YTd/B4lRGcNuu12sKE35sM7A= github.com/bkielbasa/cyclop v1.2.0/go.mod h1:qOI0yy6A7dYC4Zgsa72Ppm9kONl0RoIlPbzot9mhmeI= github.com/blizzy78/varnamelen v0.8.0 h1:oqSblyuQvFsW1hbBHh1zfwrKe3kcSj0rnXkKzsQ089M= @@ -185,6 +184,8 @@ github.com/casbin/casbin/v2 v2.37.0/go.mod h1:vByNa/Fchek0KZUgG5wEsl7iFsiviAYKRt github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= +github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= @@ -196,11 +197,11 @@ github.com/charithe/durationcheck v0.0.9 h1:mPP4ucLrf/rKZiIG/a9IPXHGlh8p4CzgpyTy github.com/charithe/durationcheck v0.0.9/go.mod h1:SSbRIBVfMjCi/kEB6K65XEA83D6prSM8ap1UCpNKtgg= github.com/chavacava/garif v0.0.0-20220316182200-5cad0b5181d4 h1:tFXjAxje9thrTF4h57Ckik+scJjTWdwAtZqZPtOT48M= github.com/chavacava/garif v0.0.0-20220316182200-5cad0b5181d4/go.mod h1:W8EnPSQ8Nv4fUjc/v1/8tHFqhuOJXnRub0dTfuAQktU= -github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= +github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= +github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng= @@ -216,16 +217,14 @@ github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ= -github.com/containerd/continuity v0.2.1 h1:/EeEo2EtN3umhbbgCveyjifoMYg0pS+nMMEemaYw634= -github.com/containerd/continuity v0.2.1/go.mod h1:wCYX+dRqZdImhGucXOqTQn05AhX6EUDaGEMUzTFFpLg= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= +github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg= +github.com/containerd/continuity v0.3.0/go.mod h1:wJEAIwKOm/pBZuBd0JmeTvnLquTB1Ag8espWhkykbPM= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= @@ -246,7 +245,8 @@ github.com/creachadair/tomledit v0.0.22 h1:lRtepmrwhzDq+g1gv5ftVn5itgo7CjYbm6abK github.com/creachadair/tomledit v0.0.22/go.mod h1:cIu/4x5L855oSRejIqr+WRFh+mv9g4fWLiUFaApYn/Y= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= +github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/daixiang0/gci v0.3.3 h1:55xJKH7Gl9Vk6oQ1cMkwrDWjAkT1D+D1G9kNmRcAIY4= github.com/daixiang0/gci v0.3.3/go.mod h1:1Xr2bxnQbDxCqqulUOv8qpGqkgRw9RSCGGjEC2LjF8o= github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -266,8 +266,13 @@ github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KP github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= +github.com/docker/cli v20.10.14+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v20.10.17+incompatible h1:eO2KS7ZFeov5UJeaDmIs1NFEDRf32PaqRpvoEkKBy5M= +github.com/docker/cli v20.10.17+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/docker v20.10.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v20.10.17+incompatible h1:JYCuMrWaVNophQTOrMMoSwudOVEfcegoZZrleKc1xwE= +github.com/docker/docker v20.10.17+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= @@ -376,6 +381,7 @@ github.com/go-zookeeper/zk v1.0.2/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gofrs/uuid v4.2.0+incompatible h1:yyYWMnhkhrKwwr8gAOcOCYxOOscHgDS9yZgBrnJfGa0= @@ -394,7 +400,6 @@ github.com/golang-sql/sqlexp v0.0.0-20170517235910-f1bb20e5a188/go.mod h1:vXjM/+ github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -497,6 +502,8 @@ github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/trillian v1.3.11/go.mod h1:0tPraVHrSDkA3BO6vKX67zgLXs6SsOAbHEivX+9mPgw= github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -522,7 +529,6 @@ github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51 github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -546,14 +552,12 @@ github.com/gostaticanalysis/testutil v0.4.0/go.mod h1:bLIoPefWXrRi/ssLFWX1dx7Rep github.com/gotestyourself/gotestyourself v2.2.0+incompatible h1:AQwinXlbQR2HvPjQZOmDhRqsv5mZf+Jb1RnSLxcqZcI= github.com/gotestyourself/gotestyourself v2.2.0+incompatible/go.mod h1:zZKM6oeNM8k+FRljX1mnzVYeS8wiGgQyvST1/GafPbY= github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.2.2/go.mod h1:EaizFBKfUKtMIF5iaDEhniwNedqGo9FuLFzppDr3uwI= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.12.1/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= @@ -610,6 +614,9 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.4/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= +github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb1-client v0.0.0-20200827194710-b269163b24ab/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= @@ -702,6 +709,7 @@ github.com/ldez/tagliatelle v0.3.1/go.mod h1:8s6WJQwEYHbKZDsp/LjArytKOG8qaMrKQQ3 github.com/leonklingele/grouper v1.1.0 h1:tC2y/ygPbMFSBOs3DcyaEMKnnwH7eYKzohOtRrf0SAg= github.com/leonklingele/grouper v1.1.0/go.mod h1:uk3I3uDfi9B6PeUjsCKi6ndcf63Uy7snXgR4yDYQVDY= github.com/letsencrypt/pkcs11key/v4 v4.0.0/go.mod h1:EFUvBDay26dErnNb70Nd0/VW3tJiIbETBPTl9ATXQag= +github.com/lib/pq v0.0.0-20180327071824-d34b9ff171c2/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.9.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= @@ -771,13 +779,17 @@ github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= -github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= +github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= +github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= +github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 h1:dcztxKSvZ4Id8iPpHERQBbIJfabdt4wUm5qy3wOL2Zc= +github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -819,7 +831,6 @@ github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oasisprotocol/curve25519-voi v0.0.0-20210609091139-0a56a4bca00b h1:MKwruh+HeCSKWphkxuzvRzU4QzDkg7yiPkDVV0cDFgI= github.com/oasisprotocol/curve25519-voi v0.0.0-20210609091139-0a56a4bca00b/go.mod h1:TLJifjWF6eotcfzDjKZsDqWJ+73Uvj/N85MvVyrvynM= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/oklog/ulid/v2 v2.0.2/go.mod h1:mtBL0Qe/0HAx6/a4Z30qxVIAL1eQDweXq5lxOEiwQ68= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= @@ -850,15 +861,18 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/runc v1.0.3 h1:1hbqejyQWCJBvtKAfdO0b1FmaEf2z/bxnjqbARass5k= -github.com/opencontainers/runc v1.0.3/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= +github.com/opencontainers/runc v1.1.2/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc= +github.com/opencontainers/runc v1.1.3 h1:vIXrkId+0/J2Ymu2m7VjGvbSlAId9XNRPhn2p4b+d8w= +github.com/opencontainers/runc v1.1.3/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= +github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/openzipkin/zipkin-go v0.2.5/go.mod h1:KpXfKdgRDnnhsxw4pNIH9Md5lyFqKUa4YDFlwRYAMyE= github.com/ory/dockertest v3.3.5+incompatible h1:iLLK6SQwIhcbrG783Dghaaa3WPzGc+4Emza6EbVUUGA= github.com/ory/dockertest v3.3.5+incompatible/go.mod h1:1vX4m9wsvi00u5bseYwXaSnhNrne+V0E6LAcBILJdPs= +github.com/ory/dockertest/v3 v3.9.1 h1:v4dkG+dlu76goxMiTT2j8zV7s4oPPEppKT8K8p2f1kY= +github.com/ory/dockertest/v3 v3.9.1/go.mod h1:42Ir9hmvaAPm0Mgibk6mBPi7SFvTXxEcnztDYOJ//uM= github.com/otiai10/copy v1.2.0 h1:HvG945u96iNadPoG2/Ja2+AUJeW5YuFQMixq9yirC+k= github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw= github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= @@ -904,7 +918,6 @@ github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndr github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= @@ -918,8 +931,6 @@ github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1: github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= @@ -929,14 +940,12 @@ github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+ github.com/prometheus/common v0.34.0 h1:RBmGO9d/FVjqHT0yUGQwBJhkwKV+wPCn7KGpvfab0uE= github.com/prometheus/common v0.34.0/go.mod h1:gB3sOl7P0TvJabZpLY5uQMpUqRCPPCyRLCZYc7JZTNE= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/pseudomuto/protoc-gen-doc v1.3.2/go.mod h1:y5+P6n3iGrbKG+9O04V5ld71in3v/bX88wUwgt+U8EA= github.com/pseudomuto/protokit v0.2.0/go.mod h1:2PdH30hxVHsup8KpBTOXTBeMVhJZVio3Q8ViKSAXT0Q= github.com/quasilyte/go-ruleguard v0.3.1-0.20210203134552-1b5a410e1cc8/go.mod h1:KsAh3x0e7Fkpgs+Q9pNLS5XpFSvYCEVl5gP9Pp1xp30= @@ -985,7 +994,8 @@ github.com/sagikazarmark/crypt v0.6.0/go.mod h1:U8+INwJo3nBv1m6A/8OBXAq7Jnpspk5A github.com/sanposhiho/wastedassign/v2 v2.0.6 h1:+6/hQIHKNJAUixEj6EmOngGIisyeI+T3335lYTyxRoA= github.com/sanposhiho/wastedassign/v2 v2.0.6/go.mod h1:KyZ0MWTwxxBmfwn33zh3k1dmsbF2ud9pAAGfoLfjhtI= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= +github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= +github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/securego/gosec/v2 v2.11.0 h1:+PDkpzR41OI2jrw1q6AdXZCbsNGNGT7pQjal0H0cArI= github.com/securego/gosec/v2 v2.11.0/go.mod h1:SX8bptShuG8reGC0XS09+a4H2BoWSJi+fscA+Pulbpo= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= @@ -1029,7 +1039,6 @@ github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi9V4= github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q= github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= @@ -1041,7 +1050,6 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= github.com/spf13/viper v1.11.0/go.mod h1:djo0X/bA5+tYVoCn+C7cAYJGcVn/qYLFTG8gdUsX7Zk= github.com/spf13/viper v1.12.0 h1:CZ7eSOd3kZoaYDLbXnmzgQI5RlciuXBMA+18HwHRfZQ= @@ -1101,8 +1109,6 @@ github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce/go.mod h1:o8v6yHRoi github.com/tommy-muehle/go-mnd/v2 v2.5.0 h1:iAj0a8e6+dXSL7Liq0aXPox36FiN1dBbjA6lt9fl65s= github.com/tommy-muehle/go-mnd/v2 v2.5.0/go.mod h1:WsUAkMJMYww6l/ufffCD3m+P7LEvr8TnZn9lwVDlgzw= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= -github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ultraware/funlen v0.0.3 h1:5ylVWm8wsNwH5aWo9438pwvsK0QiqVuUrt9bn7S/iLA= github.com/ultraware/funlen v0.0.3/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA= @@ -1121,6 +1127,13 @@ github.com/vektra/mockery/v2 v2.13.1/go.mod h1:bnD1T8tExSgPD1ripLkDbr60JA9VtQeu1 github.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8/go.mod h1:dniwbG03GafCjFohMDmz6Zc6oCuiqgH6tGNyXTkHzXE= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= @@ -1141,7 +1154,6 @@ github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= gitlab.com/bosi/decorder v0.2.1 h1:ehqZe8hI4w7O4b1vgsDZw1YU1PE7iJXrQWFMsocbQ1w= gitlab.com/bosi/decorder v0.2.1/go.mod h1:6C/nhLSbF6qZbYD8bRmISBwc6vcWdNsiIBkRvjJFrH0= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= @@ -1284,7 +1296,6 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -1331,14 +1342,14 @@ golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210917221730-978cfadd31cf/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211208012354-db4efeb81f4b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2 h1:NWy5+hlRbC7HK+PmcXVUmW1IMyFce7to56IUvhUFm7Y= golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220617184016-355a448f1bc9 h1:Yqz/iviulwKwAREEeUd3nbBFn0XuyJqkoft2IlrvOhc= +golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1406,7 +1417,6 @@ golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1429,8 +1439,8 @@ golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1450,7 +1460,6 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1463,12 +1472,15 @@ golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211105183446-c75c47738b0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1481,13 +1493,15 @@ golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220403020550-483a9cbc67c0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220405210540-1e041c57c461/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d h1:Zu/JngovGLVi6t2J3nmAf3AoTDwuzw85YZ3b9o4yU7s= golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c h1:aFV+BgZ4svzjfabn8ERpuB4JI4N6/rdy1iusx77G3oU= +golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1531,6 +1545,7 @@ golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1764,7 +1779,6 @@ google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd/go.mod h1:RAyBrSAP google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= @@ -1854,6 +1868,8 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +gotest.tools/v3 v3.2.0/go.mod h1:Mcr9QNxkg0uMvy/YElmo4SpXgJKWgQvYrT7Kw5RzJ1A= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From acf97128f32c91bd8c2b8bc2b47bd5bb1cfa1c2c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jun 2022 12:04:19 +0000 Subject: [PATCH 26/61] build(deps): Bump github.com/prometheus/common from 0.34.0 to 0.35.0 (#8800) Bumps [github.com/prometheus/common](https://github.com/prometheus/common) from 0.34.0 to 0.35.0.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/prometheus/common&package-manager=go_modules&previous-version=0.34.0&new-version=0.35.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 227e0f5cb..545a86655 100644 --- a/go.mod +++ b/go.mod @@ -240,6 +240,6 @@ require ( require ( github.com/creachadair/tomledit v0.0.22 github.com/prometheus/client_model v0.2.0 - github.com/prometheus/common v0.34.0 + github.com/prometheus/common v0.35.0 github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca ) diff --git a/go.sum b/go.sum index 8ee6ee311..aa6b12633 100644 --- a/go.sum +++ b/go.sum @@ -937,8 +937,8 @@ github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB8 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.34.0 h1:RBmGO9d/FVjqHT0yUGQwBJhkwKV+wPCn7KGpvfab0uE= -github.com/prometheus/common v0.34.0/go.mod h1:gB3sOl7P0TvJabZpLY5uQMpUqRCPPCyRLCZYc7JZTNE= +github.com/prometheus/common v0.35.0 h1:Eyr+Pw2VymWejHqCugNaQXkAi6KayVNxaHeu6khmFBE= +github.com/prometheus/common v0.35.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= From 28d323995891080484617c076649b63a504c5bf4 Mon Sep 17 00:00:00 2001 From: Sam Kleinman Date: Mon, 20 Jun 2022 11:47:56 -0400 Subject: [PATCH 27/61] p2p: wake dialing thread after sleep (#8803) --- internal/p2p/peermanager.go | 9 +++++++++ internal/p2p/router.go | 5 +++++ 2 files changed, 14 insertions(+) diff --git a/internal/p2p/peermanager.go b/internal/p2p/peermanager.go index a5438b9ec..210c34e2a 100644 --- a/internal/p2p/peermanager.go +++ b/internal/p2p/peermanager.go @@ -498,6 +498,15 @@ func (m *PeerManager) HasMaxPeerCapacity() bool { return len(m.connected) >= int(m.options.MaxConnected) } +func (m *PeerManager) HasDialedMaxPeers() bool { + m.mtx.Lock() + defer m.mtx.Unlock() + + stats := m.getConnectedInfo() + + return stats.outgoing >= m.options.MaxOutgoingConnections +} + // DialNext finds an appropriate peer address to dial, and marks it as dialing. // If no peer is found, or all connection slots are full, it blocks until one // becomes available. The caller must call Dialed() or DialFailed() for the diff --git a/internal/p2p/router.go b/internal/p2p/router.go index 55dc73720..ec225f8e3 100644 --- a/internal/p2p/router.go +++ b/internal/p2p/router.go @@ -466,6 +466,11 @@ func (r *Router) dialSleep(ctx context.Context) { } r.options.DialSleep(ctx) + + if !r.peerManager.HasDialedMaxPeers() { + r.peerManager.dialWaker.Wake() + } + } // acceptPeers accepts inbound connections from peers on the given transport, From 6f168df7e46228458a43840053a99c4fa7d1da97 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Jun 2022 17:46:01 +0000 Subject: [PATCH 28/61] build(deps): Bump github.com/spf13/cobra from 1.4.0 to 1.5.0 (#8812) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/spf13/cobra](https://github.com/spf13/cobra) from 1.4.0 to 1.5.0.
Release notes

Sourced from github.com/spf13/cobra's releases.

v1.5.0

Spring 2022 Release 🌥️

Hello everyone! Welcome to another release of cobra. Completions continue to get better and better. This release adds a few really cool new features. We also continue to patch versions of our dependencies as they become available via dependabot. Happy coding!

Active help 👐🏼

Shout out to @​marckhouzam for a big value add: Active Help spf13/cobra#1482. With active help, a program can provide some inline warnings or hints for users as they hit tab. Now, your CLIs can be even more intuitive to use!

Currently active help is only supported for bash V2 and zsh. Marc wrote a whole guide on how to do this, so make sure to give it a good read to learn how you can add this to your cobra code! https://github.com/spf13/cobra/blob/master/active_help.md

Group flags 🧑🏼‍🤝‍🧑🏼

Cobra now has the ability to mark flags as required or exclusive as a group. Shout out to our newest maintainer @​johnSchnake for this! spf13/cobra#1654 Let's say you have a username flag that MUST be partnered with a password flag. Well, now, you can enforce those as being required together:

rootCmd.Flags().StringVarP(&u, "username", "u", "", "Username (required if password is set)")
rootCmd.Flags().StringVarP(&pw, "password", "p", "", "Password (required if username is set)")
rootCmd.MarkFlagsRequiredTogether("username", "password")

Flags may also be marked as "mutally exclusive" with the MarkFlagsMutuallyExclusive(string, string ... ) command API. Refer to our user guide documentation for further info!

Completions 👀

Documentation 📝

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/spf13/cobra&package-manager=go_modules&previous-version=1.4.0&new-version=1.5.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- go.mod | 4 ++-- go.sum | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 545a86655..8dd4c9288 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/rs/cors v1.8.2 github.com/rs/zerolog v1.27.0 github.com/snikch/goodman v0.0.0-20171125024755-10e37e294daa - github.com/spf13/cobra v1.4.0 + github.com/spf13/cobra v1.5.0 github.com/spf13/viper v1.12.0 github.com/stretchr/testify v1.7.2 github.com/tendermint/tm-db v0.6.6 @@ -74,7 +74,7 @@ require ( github.com/charithe/durationcheck v0.0.9 // indirect github.com/chavacava/garif v0.0.0-20220316182200-5cad0b5181d4 // indirect github.com/containerd/continuity v0.3.0 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/daixiang0/gci v0.3.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/denis-tingaikin/go-header v0.4.3 // indirect diff --git a/go.sum b/go.sum index aa6b12633..bdb93f91e 100644 --- a/go.sum +++ b/go.sum @@ -234,8 +234,9 @@ github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creachadair/atomicfile v0.2.6 h1:FgYxYvGcqREApTY8Nxg8msM6P/KVKK3ob5h9FaRUTNg= github.com/creachadair/atomicfile v0.2.6/go.mod h1:BRq8Une6ckFneYXZQ+kO7p1ZZP3I2fzVzf28JxrIkBc= github.com/creachadair/command v0.0.0-20220426235536-a748effdf6a1/go.mod h1:bAM+qFQb/KwWyCc9MLC4U1jvn3XyakqP5QRkds5T6cY= @@ -1040,8 +1041,9 @@ github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi9V4= -github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q= github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= +github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU= +github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= From cfd13825e2c4ea5b6099ae05f73524a97a1eba82 Mon Sep 17 00:00:00 2001 From: Sam Kleinman Date: Tue, 21 Jun 2022 16:44:14 -0400 Subject: [PATCH 29/61] p2p: add eviction metrics and cleanup dialing error handling (#8819) --- internal/p2p/metrics.gen.go | 7 ++ internal/p2p/metrics.go | 3 + internal/p2p/peermanager.go | 24 +++--- internal/p2p/peermanager_test.go | 132 +++++++++++-------------------- internal/p2p/router.go | 39 ++++----- 5 files changed, 84 insertions(+), 121 deletions(-) diff --git a/internal/p2p/metrics.gen.go b/internal/p2p/metrics.gen.go index cb215f2b6..9cffbc46b 100644 --- a/internal/p2p/metrics.gen.go +++ b/internal/p2p/metrics.gen.go @@ -74,6 +74,12 @@ func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics { Name: "peers_connected_outgoing", Help: "Number of peers connected as a result of the peer dialing this node.", }, labels).With(labelsAndValues...), + PeersEvicted: prometheus.NewCounterFrom(stdprometheus.CounterOpts{ + Namespace: namespace, + Subsystem: MetricsSubsystem, + Name: "peers_evicted", + Help: "Number of peers evicted by this node.", + }, labels).With(labelsAndValues...), RouterPeerQueueRecv: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{ Namespace: namespace, Subsystem: MetricsSubsystem, @@ -119,6 +125,7 @@ func NopMetrics() *Metrics { PeersConnectedFailure: discard.NewCounter(), PeersConnectedIncoming: discard.NewGauge(), PeersConnectedOutgoing: discard.NewGauge(), + PeersEvicted: discard.NewCounter(), RouterPeerQueueRecv: discard.NewHistogram(), RouterPeerQueueSend: discard.NewHistogram(), RouterChannelQueueSend: discard.NewHistogram(), diff --git a/internal/p2p/metrics.go b/internal/p2p/metrics.go index a88aaa3f2..bc233f691 100644 --- a/internal/p2p/metrics.go +++ b/internal/p2p/metrics.go @@ -51,6 +51,9 @@ type Metrics struct { // this node. PeersConnectedOutgoing metrics.Gauge + // Number of peers evicted by this node. + PeersEvicted metrics.Counter + // RouterPeerQueueRecv defines the time taken to read off of a peer's queue // before sending on the connection. //metrics:The time taken to read off of a peer's queue before sending on the connection. diff --git a/internal/p2p/peermanager.go b/internal/p2p/peermanager.go index 210c34e2a..f2f74007b 100644 --- a/internal/p2p/peermanager.go +++ b/internal/p2p/peermanager.go @@ -513,12 +513,13 @@ func (m *PeerManager) HasDialedMaxPeers() bool { // returned peer. func (m *PeerManager) DialNext(ctx context.Context) (NodeAddress, error) { for { - address, err := m.TryDialNext() - if err != nil || (address != NodeAddress{}) { - return address, err + if address := m.TryDialNext(); (address != NodeAddress{}) { + return address, nil } + select { case <-m.dialWaker.Sleep(): + continue case <-ctx.Done(): return NodeAddress{}, ctx.Err() } @@ -527,7 +528,7 @@ func (m *PeerManager) DialNext(ctx context.Context) (NodeAddress, error) { // TryDialNext is equivalent to DialNext(), but immediately returns an empty // address if no peers or connection slots are available. -func (m *PeerManager) TryDialNext() (NodeAddress, error) { +func (m *PeerManager) TryDialNext() NodeAddress { m.mtx.Lock() defer m.mtx.Unlock() @@ -535,12 +536,12 @@ func (m *PeerManager) TryDialNext() (NodeAddress, error) { // MaxConnectedUpgrade allows us to probe additional peers that have a // higher score than any other peers, and if successful evict it. if m.options.MaxConnected > 0 && len(m.connected)+len(m.dialing) >= int(m.options.MaxConnected)+int(m.options.MaxConnectedUpgrade) { - return NodeAddress{}, nil + return NodeAddress{} } cinfo := m.getConnectedInfo() if m.options.MaxOutgoingConnections > 0 && cinfo.outgoing >= m.options.MaxOutgoingConnections { - return NodeAddress{}, nil + return NodeAddress{} } for _, peer := range m.store.Ranked() { @@ -563,16 +564,16 @@ func (m *PeerManager) TryDialNext() (NodeAddress, error) { if m.options.MaxConnected > 0 && len(m.connected) >= int(m.options.MaxConnected) { upgradeFromPeer := m.findUpgradeCandidate(peer.ID, peer.Score()) if upgradeFromPeer == "" { - return NodeAddress{}, nil + return NodeAddress{} } m.upgrading[upgradeFromPeer] = peer.ID } m.dialing[peer.ID] = true - return addressInfo.Address, nil + return addressInfo.Address } } - return NodeAddress{}, nil + return NodeAddress{} } // DialFailed reports a failed dial attempt. This will make the peer available @@ -680,8 +681,7 @@ func (m *PeerManager) Dialed(address NodeAddress) error { return err } - if upgradeFromPeer != "" && m.options.MaxConnected > 0 && - len(m.connected) >= int(m.options.MaxConnected) { + if upgradeFromPeer != "" && m.options.MaxConnected > 0 && len(m.connected) >= int(m.options.MaxConnected) { // Look for an even lower-scored peer that may have appeared since we // started the upgrade. if p, ok := m.store.Get(upgradeFromPeer); ok { @@ -690,11 +690,11 @@ func (m *PeerManager) Dialed(address NodeAddress) error { } } m.evict[upgradeFromPeer] = true + m.evictWaker.Wake() } m.metrics.PeersConnectedOutgoing.Add(1) m.connected[peer.ID] = peerConnectionOutgoing - m.evictWaker.Wake() return nil } diff --git a/internal/p2p/peermanager_test.go b/internal/p2p/peermanager_test.go index 5e9c3a8a4..a1543bf18 100644 --- a/internal/p2p/peermanager_test.go +++ b/internal/p2p/peermanager_test.go @@ -384,16 +384,14 @@ func TestPeerManager_DialNext_WakeOnDialFailed(t *testing.T) { added, err := peerManager.Add(a) require.NoError(t, err) require.True(t, added) - dial, err := peerManager.TryDialNext() - require.NoError(t, err) + dial := peerManager.TryDialNext() require.Equal(t, a, dial) // Add b. We shouldn't be able to dial it, due to MaxConnected. added, err = peerManager.Add(b) require.NoError(t, err) require.True(t, added) - dial, err = peerManager.TryDialNext() - require.NoError(t, err) + dial = peerManager.TryDialNext() require.Zero(t, dial) // Spawn a goroutine to fail a's dial attempt. @@ -427,8 +425,7 @@ func TestPeerManager_DialNext_WakeOnDialFailedRetry(t *testing.T) { added, err := peerManager.Add(a) require.NoError(t, err) require.True(t, added) - dial, err := peerManager.TryDialNext() - require.NoError(t, err) + dial := peerManager.TryDialNext() require.Equal(t, a, dial) require.NoError(t, peerManager.DialFailed(ctx, dial)) failed := time.Now() @@ -458,8 +455,7 @@ func TestPeerManager_DialNext_WakeOnDisconnected(t *testing.T) { err = peerManager.Accepted(a.NodeID) require.NoError(t, err) - dial, err := peerManager.TryDialNext() - require.NoError(t, err) + dial := peerManager.TryDialNext() require.Zero(t, dial) dctx, dcancel := context.WithTimeout(ctx, 300*time.Millisecond) @@ -490,8 +486,7 @@ func TestPeerManager_TryDialNext_MaxConnected(t *testing.T) { added, err := peerManager.Add(a) require.NoError(t, err) require.True(t, added) - dial, err := peerManager.TryDialNext() - require.NoError(t, err) + dial := peerManager.TryDialNext() require.Equal(t, a, dial) require.NoError(t, peerManager.Dialed(a)) @@ -499,16 +494,14 @@ func TestPeerManager_TryDialNext_MaxConnected(t *testing.T) { added, err = peerManager.Add(b) require.NoError(t, err) require.True(t, added) - dial, err = peerManager.TryDialNext() - require.NoError(t, err) + dial = peerManager.TryDialNext() require.Equal(t, b, dial) // At this point, adding c will not allow dialing it. added, err = peerManager.Add(c) require.NoError(t, err) require.True(t, added) - dial, err = peerManager.TryDialNext() - require.NoError(t, err) + dial = peerManager.TryDialNext() require.Zero(t, dial) } @@ -540,7 +533,7 @@ func TestPeerManager_TryDialNext_MaxConnectedUpgrade(t *testing.T) { added, err := peerManager.Add(a) require.NoError(t, err) require.True(t, added) - dial, err := peerManager.TryDialNext() + dial := peerManager.TryDialNext() require.NoError(t, err) require.Equal(t, a, dial) require.NoError(t, peerManager.Dialed(a)) @@ -549,8 +542,7 @@ func TestPeerManager_TryDialNext_MaxConnectedUpgrade(t *testing.T) { added, err = peerManager.Add(b) require.NoError(t, err) require.True(t, added) - dial, err = peerManager.TryDialNext() - require.NoError(t, err) + dial = peerManager.TryDialNext() require.Equal(t, b, dial) // Even though we are at capacity, we should be allowed to dial c for an @@ -558,8 +550,7 @@ func TestPeerManager_TryDialNext_MaxConnectedUpgrade(t *testing.T) { added, err = peerManager.Add(c) require.NoError(t, err) require.True(t, added) - dial, err = peerManager.TryDialNext() - require.NoError(t, err) + dial = peerManager.TryDialNext() require.Equal(t, c, dial) // However, since we're using all upgrade slots now, we can't add and dial @@ -567,16 +558,14 @@ func TestPeerManager_TryDialNext_MaxConnectedUpgrade(t *testing.T) { added, err = peerManager.Add(d) require.NoError(t, err) require.True(t, added) - dial, err = peerManager.TryDialNext() - require.NoError(t, err) + dial = peerManager.TryDialNext() require.Zero(t, dial) // We go through with c's upgrade. require.NoError(t, peerManager.Dialed(c)) // Still can't dial d. - dial, err = peerManager.TryDialNext() - require.NoError(t, err) + dial = peerManager.TryDialNext() require.Zero(t, dial) // Now, if we disconnect a, we should be allowed to dial d because we have a @@ -592,8 +581,7 @@ func TestPeerManager_TryDialNext_MaxConnectedUpgrade(t *testing.T) { added, err = peerManager.Add(e) require.NoError(t, err) require.True(t, added) - dial, err = peerManager.TryDialNext() - require.NoError(t, err) + dial = peerManager.TryDialNext() require.Zero(t, dial) } @@ -613,8 +601,7 @@ func TestPeerManager_TryDialNext_UpgradeReservesPeer(t *testing.T) { added, err := peerManager.Add(a) require.NoError(t, err) require.True(t, added) - dial, err := peerManager.TryDialNext() - require.NoError(t, err) + dial := peerManager.TryDialNext() require.Equal(t, a, dial) require.NoError(t, peerManager.Dialed(a)) @@ -622,8 +609,7 @@ func TestPeerManager_TryDialNext_UpgradeReservesPeer(t *testing.T) { added, err = peerManager.Add(b) require.NoError(t, err) require.True(t, added) - dial, err = peerManager.TryDialNext() - require.NoError(t, err) + dial = peerManager.TryDialNext() require.Equal(t, b, dial) // Adding c and dialing it will fail, because a is the only connected @@ -631,8 +617,7 @@ func TestPeerManager_TryDialNext_UpgradeReservesPeer(t *testing.T) { added, err = peerManager.Add(c) require.NoError(t, err) require.True(t, added) - dial, err = peerManager.TryDialNext() - require.NoError(t, err) + dial = peerManager.TryDialNext() require.Empty(t, dial) } @@ -653,22 +638,19 @@ func TestPeerManager_TryDialNext_DialingConnected(t *testing.T) { added, err := peerManager.Add(a) require.NoError(t, err) require.True(t, added) - dial, err := peerManager.TryDialNext() - require.NoError(t, err) + dial := peerManager.TryDialNext() require.Equal(t, a, dial) // Adding a's TCP address will not dispense a, since it's already dialing. added, err = peerManager.Add(aTCP) require.NoError(t, err) require.True(t, added) - dial, err = peerManager.TryDialNext() - require.NoError(t, err) + dial = peerManager.TryDialNext() require.Zero(t, dial) // Marking a as dialed will still not dispense it. require.NoError(t, peerManager.Dialed(a)) - dial, err = peerManager.TryDialNext() - require.NoError(t, err) + dial = peerManager.TryDialNext() require.Zero(t, dial) // Adding b and accepting a connection from it will not dispense it either. @@ -676,8 +658,7 @@ func TestPeerManager_TryDialNext_DialingConnected(t *testing.T) { require.NoError(t, err) require.True(t, added) require.NoError(t, peerManager.Accepted(bID)) - dial, err = peerManager.TryDialNext() - require.NoError(t, err) + dial = peerManager.TryDialNext() require.Zero(t, dial) } @@ -706,16 +687,14 @@ func TestPeerManager_TryDialNext_Multiple(t *testing.T) { // All addresses should be dispensed as long as dialing them has failed. dial := []p2p.NodeAddress{} for range addresses { - address, err := peerManager.TryDialNext() - require.NoError(t, err) + address := peerManager.TryDialNext() require.NotZero(t, address) require.NoError(t, peerManager.DialFailed(ctx, address)) dial = append(dial, address) } require.ElementsMatch(t, dial, addresses) - address, err := peerManager.TryDialNext() - require.NoError(t, err) + address := peerManager.TryDialNext() require.Zero(t, address) } @@ -740,15 +719,14 @@ func TestPeerManager_DialFailed(t *testing.T) { // Dialing and then calling DialFailed with a different address (same // NodeID) should unmark as dialing and allow us to dial the other address // again, but not register the failed address. - dial, err := peerManager.TryDialNext() + dial := peerManager.TryDialNext() require.NoError(t, err) require.Equal(t, a, dial) require.NoError(t, peerManager.DialFailed(ctx, p2p.NodeAddress{ Protocol: "tcp", NodeID: aID, Hostname: "localhost"})) require.Equal(t, []p2p.NodeAddress{a}, peerManager.Addresses(aID)) - dial, err = peerManager.TryDialNext() - require.NoError(t, err) + dial = peerManager.TryDialNext() require.Equal(t, a, dial) // Calling DialFailed on same address twice should be fine. @@ -782,8 +760,7 @@ func TestPeerManager_DialFailed_UnreservePeer(t *testing.T) { added, err := peerManager.Add(a) require.NoError(t, err) require.True(t, added) - dial, err := peerManager.TryDialNext() - require.NoError(t, err) + dial := peerManager.TryDialNext() require.Equal(t, a, dial) require.NoError(t, peerManager.Dialed(a)) @@ -791,8 +768,7 @@ func TestPeerManager_DialFailed_UnreservePeer(t *testing.T) { added, err = peerManager.Add(b) require.NoError(t, err) require.True(t, added) - dial, err = peerManager.TryDialNext() - require.NoError(t, err) + dial = peerManager.TryDialNext() require.Equal(t, b, dial) // Adding c and dialing it will fail, even though it could upgrade a and we @@ -801,14 +777,12 @@ func TestPeerManager_DialFailed_UnreservePeer(t *testing.T) { added, err = peerManager.Add(c) require.NoError(t, err) require.True(t, added) - dial, err = peerManager.TryDialNext() - require.NoError(t, err) + dial = peerManager.TryDialNext() require.Empty(t, dial) // Failing b's dial will now make c available for dialing. require.NoError(t, peerManager.DialFailed(ctx, b)) - dial, err = peerManager.TryDialNext() - require.NoError(t, err) + dial = peerManager.TryDialNext() require.Equal(t, c, dial) } @@ -823,8 +797,7 @@ func TestPeerManager_Dialed_Connected(t *testing.T) { added, err := peerManager.Add(a) require.NoError(t, err) require.True(t, added) - dial, err := peerManager.TryDialNext() - require.NoError(t, err) + dial := peerManager.TryDialNext() require.Equal(t, a, dial) require.NoError(t, peerManager.Dialed(a)) @@ -834,8 +807,7 @@ func TestPeerManager_Dialed_Connected(t *testing.T) { added, err = peerManager.Add(b) require.NoError(t, err) require.True(t, added) - dial, err = peerManager.TryDialNext() - require.NoError(t, err) + dial = peerManager.TryDialNext() require.Equal(t, b, dial) require.NoError(t, peerManager.Accepted(b.NodeID)) @@ -864,8 +836,7 @@ func TestPeerManager_Dialed_MaxConnected(t *testing.T) { added, err := peerManager.Add(a) require.NoError(t, err) require.True(t, added) - dial, err := peerManager.TryDialNext() - require.NoError(t, err) + dial := peerManager.TryDialNext() require.Equal(t, a, dial) // Marking b as dialed in the meanwhile (even without TryDialNext) @@ -907,8 +878,7 @@ func TestPeerManager_Dialed_MaxConnectedUpgrade(t *testing.T) { added, err = peerManager.Add(c) require.NoError(t, err) require.True(t, added) - dial, err := peerManager.TryDialNext() - require.NoError(t, err) + dial := peerManager.TryDialNext() require.Equal(t, c, dial) require.NoError(t, peerManager.Dialed(c)) @@ -952,8 +922,7 @@ func TestPeerManager_Dialed_Upgrade(t *testing.T) { added, err = peerManager.Add(b) require.NoError(t, err) require.True(t, added) - dial, err := peerManager.TryDialNext() - require.NoError(t, err) + dial := peerManager.TryDialNext() require.Equal(t, b, dial) require.NoError(t, peerManager.Dialed(b)) @@ -962,8 +931,7 @@ func TestPeerManager_Dialed_Upgrade(t *testing.T) { added, err = peerManager.Add(c) require.NoError(t, err) require.True(t, added) - dial, err = peerManager.TryDialNext() - require.NoError(t, err) + dial = peerManager.TryDialNext() require.Empty(t, dial) // a should now be evicted. @@ -1009,8 +977,7 @@ func TestPeerManager_Dialed_UpgradeEvenLower(t *testing.T) { added, err = peerManager.Add(c) require.NoError(t, err) require.True(t, added) - dial, err := peerManager.TryDialNext() - require.NoError(t, err) + dial := peerManager.TryDialNext() require.Equal(t, c, dial) // In the meanwhile, a disconnects and d connects. d is even lower-scored @@ -1063,7 +1030,7 @@ func TestPeerManager_Dialed_UpgradeNoEvict(t *testing.T) { added, err = peerManager.Add(c) require.NoError(t, err) require.True(t, added) - dial, err := peerManager.TryDialNext() + dial := peerManager.TryDialNext() require.NoError(t, err) require.Equal(t, c, dial) @@ -1109,8 +1076,7 @@ func TestPeerManager_Accepted(t *testing.T) { added, err = peerManager.Add(c) require.NoError(t, err) require.True(t, added) - dial, err := peerManager.TryDialNext() - require.NoError(t, err) + dial := peerManager.TryDialNext() require.Equal(t, c, dial) require.NoError(t, peerManager.Accepted(c.NodeID)) require.Error(t, peerManager.Dialed(c)) @@ -1119,8 +1085,7 @@ func TestPeerManager_Accepted(t *testing.T) { added, err = peerManager.Add(d) require.NoError(t, err) require.True(t, added) - dial, err = peerManager.TryDialNext() - require.NoError(t, err) + dial = peerManager.TryDialNext() require.Equal(t, d, dial) require.NoError(t, peerManager.Dialed(d)) require.Error(t, peerManager.Accepted(d.NodeID)) @@ -1271,8 +1236,7 @@ func TestPeerManager_Accepted_UpgradeDialing(t *testing.T) { added, err = peerManager.Add(b) require.NoError(t, err) require.True(t, added) - dial, err := peerManager.TryDialNext() - require.NoError(t, err) + dial := peerManager.TryDialNext() require.Equal(t, b, dial) // a has already been claimed as an upgrade of a, so accepting @@ -1446,8 +1410,7 @@ func TestPeerManager_EvictNext_WakeOnUpgradeDialed(t *testing.T) { added, err := peerManager.Add(b) require.NoError(t, err) require.True(t, added) - dial, err := peerManager.TryDialNext() - require.NoError(t, err) + dial := peerManager.TryDialNext() require.Equal(t, b, dial) require.NoError(t, peerManager.Dialed(b)) }() @@ -1581,13 +1544,11 @@ func TestPeerManager_Disconnected(t *testing.T) { // Disconnecting a dialing peer does not unmark it as dialing, to avoid // dialing it multiple times in parallel. - dial, err := peerManager.TryDialNext() - require.NoError(t, err) + dial := peerManager.TryDialNext() require.Equal(t, a, dial) peerManager.Disconnected(ctx, a.NodeID) - dial, err = peerManager.TryDialNext() - require.NoError(t, err) + dial = peerManager.TryDialNext() require.Zero(t, dial) } @@ -1660,8 +1621,7 @@ func TestPeerManager_Subscribe(t *testing.T) { require.Equal(t, p2p.PeerUpdate{NodeID: a.NodeID, Status: p2p.PeerStatusDown}, <-sub.Updates()) // Outbound connection with peer error and eviction. - dial, err := peerManager.TryDialNext() - require.NoError(t, err) + dial := peerManager.TryDialNext() require.Equal(t, a, dial) require.Empty(t, sub.Updates()) @@ -1684,8 +1644,7 @@ func TestPeerManager_Subscribe(t *testing.T) { require.Equal(t, p2p.PeerUpdate{NodeID: a.NodeID, Status: p2p.PeerStatusDown}, <-sub.Updates()) // Outbound connection with dial failure. - dial, err = peerManager.TryDialNext() - require.NoError(t, err) + dial = peerManager.TryDialNext() require.Equal(t, a, dial) require.Empty(t, sub.Updates()) @@ -1790,8 +1749,7 @@ func TestPeerManager_Close(t *testing.T) { added, err := peerManager.Add(a) require.NoError(t, err) require.True(t, added) - dial, err := peerManager.TryDialNext() - require.NoError(t, err) + dial := peerManager.TryDialNext() require.Equal(t, a, dial) require.NoError(t, peerManager.DialFailed(ctx, a)) } diff --git a/internal/p2p/router.go b/internal/p2p/router.go index ec225f8e3..ff90e8c21 100644 --- a/internal/p2p/router.go +++ b/internal/p2p/router.go @@ -310,11 +310,7 @@ func (r *Router) routeChannel( ) { for { select { - case envelope, ok := <-outCh: - if !ok { - return - } - + case envelope := <-outCh: // Mark the envelope with the channel ID to allow sendPeer() to pass // it on to Transport.SendMessage(). envelope.ChannelID = chID @@ -391,20 +387,22 @@ func (r *Router) routeChannel( } } - case peerError, ok := <-errCh: - if !ok { - return - } - - shouldEvict := peerError.Fatal || r.peerManager.HasMaxPeerCapacity() + case peerError := <-errCh: + maxPeerCapacity := r.peerManager.HasMaxPeerCapacity() r.logger.Error("peer error", "peer", peerError.NodeID, "err", peerError.Err, - "evicting", shouldEvict, + "disconnecting", peerError.Fatal || maxPeerCapacity, ) - if shouldEvict { + + if peerError.Fatal || maxPeerCapacity { + // if the error is fatal or all peer + // slots are in use, we can error + // (disconnect) from the peer. r.peerManager.Errored(peerError.NodeID, peerError.Err) } else { + // this just decrements the peer + // score. r.peerManager.processPeerEvent(ctx, PeerUpdate{ NodeID: peerError.NodeID, Status: PeerStatusBad, @@ -466,11 +464,6 @@ func (r *Router) dialSleep(ctx context.Context) { } r.options.DialSleep(ctx) - - if !r.peerManager.HasDialedMaxPeers() { - r.peerManager.dialWaker.Wake() - } - } // acceptPeers accepts inbound connections from peers on the given transport, @@ -591,9 +584,8 @@ LOOP: switch { case errors.Is(err, context.Canceled): break LOOP - case err != nil: - r.logger.Error("failed to find next peer to dial", "err", err) - break LOOP + case address == NodeAddress{}: + continue LOOP } select { @@ -603,7 +595,7 @@ LOOP: // create connections too quickly. r.dialSleep(ctx) - continue + continue LOOP case <-ctx.Done(): close(addresses) break LOOP @@ -642,6 +634,7 @@ func (r *Router) connectPeer(ctx context.Context, address NodeAddress) { if err := r.runWithPeerMutex(func() error { return r.peerManager.Dialed(address) }); err != nil { r.logger.Error("failed to dial peer", "op", "outgoing/dialing", "peer", address.NodeID, "err", err) + r.peerManager.dialWaker.Wake() conn.Close() return } @@ -937,6 +930,8 @@ func (r *Router) evictPeers(ctx context.Context) { queue, ok := r.peerQueues[peerID] r.peerMtx.RUnlock() + r.metrics.PeersEvicted.Add(1) + if ok { queue.close() } From 8860e027a888e67936c096c7c5dc81806f6665bf Mon Sep 17 00:00:00 2001 From: William Banfield <4561443+williambanfield@users.noreply.github.com> Date: Tue, 21 Jun 2022 20:51:09 -0400 Subject: [PATCH 30/61] p2p: more dial routines (#8827) The dial routines perform network i/o, which is a blocking call into the kernel. These routines are completely unable to do anything else while the dial occurs, so for most of their lifecycle they are sitting idle waiting for the tcp stack to hand them data. We should increase this value by _a lot_ to enable more concurrent dials. This is unlikely to cause CPU starvation because these routines sit idle most of the time. The current value causes dials to occur _way_ too slowly. Below is a graph demonstrating the before and after of this change in a testnetwork with many dead peers. You can observe that the rate that we connect to new, valid peers, is _much_ higher than previously. Change was deployed around the 31 minute mark on the graph. ![image](https://user-images.githubusercontent.com/4561443/174919007-50e4453a-edd8-41d0-97ee-dea8853d57f7.png) --- internal/p2p/router.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/p2p/router.go b/internal/p2p/router.go index ff90e8c21..7ad5529fb 100644 --- a/internal/p2p/router.go +++ b/internal/p2p/router.go @@ -417,7 +417,7 @@ func (r *Router) routeChannel( func (r *Router) numConcurrentDials() int { if r.options.NumConcurrentDials == nil { - return runtime.NumCPU() + return runtime.NumCPU() * 32 } return r.options.NumConcurrentDials() From 2e11760fbe96dacdf8c4e37b7a179ed777a9ebfa Mon Sep 17 00:00:00 2001 From: Ian Jungyong Um <31336310+code0xff@users.noreply.github.com> Date: Wed, 22 Jun 2022 16:30:11 +0900 Subject: [PATCH 31/61] p2p: fix typo (#8836) --- internal/p2p/router.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/p2p/router.go b/internal/p2p/router.go index 7ad5529fb..e3adc77ee 100644 --- a/internal/p2p/router.go +++ b/internal/p2p/router.go @@ -68,7 +68,7 @@ type RouterOptions struct { // seconds between submitting each peer to be dialed. DialSleep func(context.Context) - // NumConcrruentDials controls how many parallel go routines + // NumConcurrentDials controls how many parallel go routines // are used to dial peers. This defaults to the value of // runtime.NumCPU. NumConcurrentDials func() int From 52b2efb8274879468af9d032afd844b300b40f6e Mon Sep 17 00:00:00 2001 From: Sam Kleinman Date: Thu, 23 Jun 2022 08:40:36 -0400 Subject: [PATCH 32/61] e2e: report peer heights in error message (#8843) --- test/e2e/runner/rpc.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/runner/rpc.go b/test/e2e/runner/rpc.go index 5f5c22149..c08805f40 100644 --- a/test/e2e/runner/rpc.go +++ b/test/e2e/runner/rpc.go @@ -23,7 +23,7 @@ func waitForHeight(ctx context.Context, testnet *e2e.Testnet, height int64) (*ty clients = map[string]*rpchttp.HTTP{} lastHeight int64 lastIncrease = time.Now() - nodesAtHeight = map[string]struct{}{} + nodesAtHeight = map[string]int64{} numRunningNodes int ) if height == 0 { @@ -85,7 +85,7 @@ func waitForHeight(ctx context.Context, testnet *e2e.Testnet, height int64) (*ty // add this node to the set of target // height nodes - nodesAtHeight[node.Name] = struct{}{} + nodesAtHeight[node.Name] = result.SyncInfo.LatestBlockHeight // if not all of the nodes that we // have clients for have reached the From 436a38f8768f32c2abf3668e586019dcc3defb04 Mon Sep 17 00:00:00 2001 From: Sam Kleinman Date: Thu, 23 Jun 2022 10:03:10 -0400 Subject: [PATCH 33/61] p2p: track peers by address (#8841) --- internal/p2p/peermanager.go | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/internal/p2p/peermanager.go b/internal/p2p/peermanager.go index f2f74007b..b30c5dbfb 100644 --- a/internal/p2p/peermanager.go +++ b/internal/p2p/peermanager.go @@ -554,6 +554,10 @@ func (m *PeerManager) TryDialNext() NodeAddress { continue } + if id, ok := m.store.Resolve(addressInfo.Address); ok && (m.isConnected(id) || m.dialing[id]) { + continue + } + // We now have an eligible address to dial. If we're full but have // upgrade capacity (as checked above), we find a lower-scored peer // we can replace and mark it as upgrading so noone else claims it. @@ -1235,6 +1239,7 @@ func (m *PeerManager) retryDelay(failures uint32, persistent bool) time.Duration type peerStore struct { db dbm.DB peers map[types.NodeID]*peerInfo + index map[NodeAddress]types.NodeID ranked []*peerInfo // cache for Ranked(), nil invalidates cache } @@ -1254,6 +1259,7 @@ func newPeerStore(db dbm.DB) (*peerStore, error) { // loadPeers loads all peers from the database into memory. func (s *peerStore) loadPeers() error { peers := map[types.NodeID]*peerInfo{} + addrs := map[NodeAddress]types.NodeID{} start, end := keyPeerInfoRange() iter, err := s.db.Iterator(start, end) @@ -1273,11 +1279,18 @@ func (s *peerStore) loadPeers() error { return fmt.Errorf("invalid peer data: %w", err) } peers[peer.ID] = peer + for addr := range peer.AddressInfo { + // TODO maybe check to see if we've seen this + // addr before for a different peer, there + // could be duplicates. + addrs[addr] = peer.ID + } } if iter.Error() != nil { return iter.Error() } s.peers = peers + s.index = addrs s.ranked = nil // invalidate cache if populated return nil } @@ -1289,6 +1302,12 @@ func (s *peerStore) Get(id types.NodeID) (peerInfo, bool) { return peer.Copy(), ok } +// Resolve returns the peer ID for a given node address if known. +func (s *peerStore) Resolve(addr NodeAddress) (types.NodeID, bool) { + id, ok := s.index[addr] + return id, ok +} + // Set stores peer data. The input data will be copied, and can safely be reused // by the caller. func (s *peerStore) Set(peer peerInfo) error { @@ -1317,20 +1336,29 @@ func (s *peerStore) Set(peer peerInfo) error { // update the existing pointer address. *current = peer } + for addr := range peer.AddressInfo { + s.index[addr] = peer.ID + } return nil } // Delete deletes a peer, or does nothing if it does not exist. func (s *peerStore) Delete(id types.NodeID) error { - if _, ok := s.peers[id]; !ok { + peer, ok := s.peers[id] + if !ok { return nil } - if err := s.db.Delete(keyPeerInfo(id)); err != nil { - return err + for _, addr := range peer.AddressInfo { + delete(s.index, addr.Address) } delete(s.peers, id) s.ranked = nil + + if err := s.db.Delete(keyPeerInfo(id)); err != nil { + return err + } + return nil } From fb209136f85083e1d491ac2589fff41dd80518cb Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Thu, 23 Jun 2022 19:11:21 +0200 Subject: [PATCH 34/61] e2e: add tolerance to peer discovery test (#8849) --- test/e2e/tests/net_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/e2e/tests/net_test.go b/test/e2e/tests/net_test.go index 2d8f28549..939c136c7 100644 --- a/test/e2e/tests/net_test.go +++ b/test/e2e/tests/net_test.go @@ -18,7 +18,9 @@ func TestNet_Peers(t *testing.T) { netInfo, err := client.NetInfo(ctx) require.NoError(t, err) - expectedPeers := len(node.Testnet.Nodes) + // FIXME: https://github.com/tendermint/tendermint/issues/8848 + // We should be able to assert that we can discover all peers in a network + expectedPeers := len(node.Testnet.Nodes) - 1 // includes extra tolerance peers := make(map[string]*e2e.Node, 0) seen := map[string]bool{} for _, n := range node.Testnet.Nodes { @@ -31,7 +33,7 @@ func TestNet_Peers(t *testing.T) { seen[n.Name] = false } - require.Equal(t, expectedPeers, netInfo.NPeers, + require.GreaterOrEqual(t, netInfo.NPeers, expectedPeers, "node is not fully meshed with peers") for _, peerInfo := range netInfo.Peers { From 5f5e74798bcc6045bb454a082405145a3230ab1a Mon Sep 17 00:00:00 2001 From: William Banfield <4561443+williambanfield@users.noreply.github.com> Date: Thu, 23 Jun 2022 18:33:21 -0400 Subject: [PATCH 35/61] p2p: set empty timeouts to small values. (#8847) These timeouts default to 'do not time out' if they are not set. This times up resources, potentially indefinitely. If node on the other side of the the handshake is up but unresponsive, the[ handshake call](https://github.com/tendermint/tendermint/blob/edec79448aa1d62b84683b1b22e12e145dbdda7c/internal/p2p/router.go#L720) will _never_ return. These are proposed values that have not been validated. I intend to validate them in a production setting. --- internal/p2p/mocks/connection.go | 20 +++++++++++--------- internal/p2p/router.go | 8 +------- internal/p2p/router_test.go | 14 +++++++------- internal/p2p/transport.go | 3 ++- internal/p2p/transport_mconn.go | 18 +++++++++++++++--- internal/p2p/transport_memory.go | 8 ++++++++ internal/p2p/transport_test.go | 12 ++++++------ node/node.go | 4 +++- 8 files changed, 53 insertions(+), 34 deletions(-) diff --git a/internal/p2p/mocks/connection.go b/internal/p2p/mocks/connection.go index 766bbf657..20727f8a6 100644 --- a/internal/p2p/mocks/connection.go +++ b/internal/p2p/mocks/connection.go @@ -13,6 +13,8 @@ import ( p2p "github.com/tendermint/tendermint/internal/p2p" + time "time" + types "github.com/tendermint/tendermint/types" ) @@ -35,20 +37,20 @@ func (_m *Connection) Close() error { return r0 } -// Handshake provides a mock function with given fields: _a0, _a1, _a2 -func (_m *Connection) Handshake(_a0 context.Context, _a1 types.NodeInfo, _a2 crypto.PrivKey) (types.NodeInfo, crypto.PubKey, error) { - ret := _m.Called(_a0, _a1, _a2) +// Handshake provides a mock function with given fields: _a0, _a1, _a2, _a3 +func (_m *Connection) Handshake(_a0 context.Context, _a1 time.Duration, _a2 types.NodeInfo, _a3 crypto.PrivKey) (types.NodeInfo, crypto.PubKey, error) { + ret := _m.Called(_a0, _a1, _a2, _a3) var r0 types.NodeInfo - if rf, ok := ret.Get(0).(func(context.Context, types.NodeInfo, crypto.PrivKey) types.NodeInfo); ok { - r0 = rf(_a0, _a1, _a2) + if rf, ok := ret.Get(0).(func(context.Context, time.Duration, types.NodeInfo, crypto.PrivKey) types.NodeInfo); ok { + r0 = rf(_a0, _a1, _a2, _a3) } else { r0 = ret.Get(0).(types.NodeInfo) } var r1 crypto.PubKey - if rf, ok := ret.Get(1).(func(context.Context, types.NodeInfo, crypto.PrivKey) crypto.PubKey); ok { - r1 = rf(_a0, _a1, _a2) + if rf, ok := ret.Get(1).(func(context.Context, time.Duration, types.NodeInfo, crypto.PrivKey) crypto.PubKey); ok { + r1 = rf(_a0, _a1, _a2, _a3) } else { if ret.Get(1) != nil { r1 = ret.Get(1).(crypto.PubKey) @@ -56,8 +58,8 @@ func (_m *Connection) Handshake(_a0 context.Context, _a1 types.NodeInfo, _a2 cry } var r2 error - if rf, ok := ret.Get(2).(func(context.Context, types.NodeInfo, crypto.PrivKey) error); ok { - r2 = rf(_a0, _a1, _a2) + if rf, ok := ret.Get(2).(func(context.Context, time.Duration, types.NodeInfo, crypto.PrivKey) error); ok { + r2 = rf(_a0, _a1, _a2, _a3) } else { r2 = ret.Error(2) } diff --git a/internal/p2p/router.go b/internal/p2p/router.go index e3adc77ee..d5edd42be 100644 --- a/internal/p2p/router.go +++ b/internal/p2p/router.go @@ -710,14 +710,8 @@ func (r *Router) handshakePeer( expectID types.NodeID, ) (types.NodeInfo, error) { - if r.options.HandshakeTimeout > 0 { - var cancel context.CancelFunc - ctx, cancel = context.WithTimeout(ctx, r.options.HandshakeTimeout) - defer cancel() - } - nodeInfo := r.nodeInfoProducer() - peerInfo, peerKey, err := conn.Handshake(ctx, *nodeInfo, r.privKey) + peerInfo, peerKey, err := conn.Handshake(ctx, r.options.HandshakeTimeout, *nodeInfo, r.privKey) if err != nil { return peerInfo, err } diff --git a/internal/p2p/router_test.go b/internal/p2p/router_test.go index 86af19385..8a58146fd 100644 --- a/internal/p2p/router_test.go +++ b/internal/p2p/router_test.go @@ -385,7 +385,7 @@ func TestRouter_AcceptPeers(t *testing.T) { connCtx, connCancel := context.WithCancel(context.Background()) mockConnection := &mocks.Connection{} mockConnection.On("String").Maybe().Return("mock") - mockConnection.On("Handshake", mock.Anything, selfInfo, selfKey). + mockConnection.On("Handshake", mock.Anything, mock.Anything, selfInfo, selfKey). Return(tc.peerInfo, tc.peerKey, nil) mockConnection.On("Close").Run(func(_ mock.Arguments) { connCancel() }).Return(nil).Maybe() mockConnection.On("RemoteEndpoint").Return(p2p.Endpoint{}) @@ -500,7 +500,7 @@ func TestRouter_AcceptPeers_HeadOfLineBlocking(t *testing.T) { mockConnection := &mocks.Connection{} mockConnection.On("String").Maybe().Return("mock") - mockConnection.On("Handshake", mock.Anything, selfInfo, selfKey). + mockConnection.On("Handshake", mock.Anything, mock.Anything, selfInfo, selfKey). WaitUntil(closeCh).Return(types.NodeInfo{}, nil, io.EOF) mockConnection.On("Close").Return(nil) mockConnection.On("RemoteEndpoint").Return(p2p.Endpoint{}) @@ -588,7 +588,7 @@ func TestRouter_DialPeers(t *testing.T) { mockConnection := &mocks.Connection{} mockConnection.On("String").Maybe().Return("mock") if tc.dialErr == nil { - mockConnection.On("Handshake", mock.Anything, selfInfo, selfKey). + mockConnection.On("Handshake", mock.Anything, mock.Anything, selfInfo, selfKey). Return(tc.peerInfo, tc.peerKey, nil) mockConnection.On("Close").Run(func(_ mock.Arguments) { connCancel() }).Return(nil).Maybe() } @@ -674,7 +674,7 @@ func TestRouter_DialPeers_Parallel(t *testing.T) { mockConnection := &mocks.Connection{} mockConnection.On("String").Maybe().Return("mock") - mockConnection.On("Handshake", mock.Anything, selfInfo, selfKey). + mockConnection.On("Handshake", mock.Anything, mock.Anything, selfInfo, selfKey). WaitUntil(closeCh).Return(types.NodeInfo{}, nil, io.EOF) mockConnection.On("Close").Return(nil) @@ -757,7 +757,7 @@ func TestRouter_EvictPeers(t *testing.T) { mockConnection := &mocks.Connection{} mockConnection.On("String").Maybe().Return("mock") - mockConnection.On("Handshake", mock.Anything, selfInfo, selfKey). + mockConnection.On("Handshake", mock.Anything, mock.Anything, selfInfo, selfKey). Return(peerInfo, peerKey.PubKey(), nil) mockConnection.On("ReceiveMessage", mock.Anything).WaitUntil(closeCh).Return(chID, nil, io.EOF) mockConnection.On("RemoteEndpoint").Return(p2p.Endpoint{}) @@ -826,7 +826,7 @@ func TestRouter_ChannelCompatability(t *testing.T) { mockConnection := &mocks.Connection{} mockConnection.On("String").Maybe().Return("mock") - mockConnection.On("Handshake", mock.Anything, selfInfo, selfKey). + mockConnection.On("Handshake", mock.Anything, mock.Anything, selfInfo, selfKey). Return(incompatiblePeer, peerKey.PubKey(), nil) mockConnection.On("RemoteEndpoint").Return(p2p.Endpoint{}) mockConnection.On("Close").Return(nil) @@ -877,7 +877,7 @@ func TestRouter_DontSendOnInvalidChannel(t *testing.T) { mockConnection := &mocks.Connection{} mockConnection.On("String").Maybe().Return("mock") - mockConnection.On("Handshake", mock.Anything, selfInfo, selfKey). + mockConnection.On("Handshake", mock.Anything, mock.Anything, selfInfo, selfKey). Return(peer, peerKey.PubKey(), nil) mockConnection.On("RemoteEndpoint").Return(p2p.Endpoint{}) mockConnection.On("Close").Return(nil) diff --git a/internal/p2p/transport.go b/internal/p2p/transport.go index 7a965260a..e644a11ae 100644 --- a/internal/p2p/transport.go +++ b/internal/p2p/transport.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net" + "time" "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/types" @@ -81,7 +82,7 @@ type Connection interface { // FIXME: The handshake should really be the Router's responsibility, but // that requires the connection interface to be byte-oriented rather than // message-oriented (see comment above). - Handshake(context.Context, types.NodeInfo, crypto.PrivKey) (types.NodeInfo, crypto.PubKey, error) + Handshake(context.Context, time.Duration, types.NodeInfo, crypto.PrivKey) (types.NodeInfo, crypto.PubKey, error) // ReceiveMessage returns the next message received on the connection, // blocking until one is available. Returns io.EOF if closed. diff --git a/internal/p2p/transport_mconn.go b/internal/p2p/transport_mconn.go index 7bf17d1a0..13a65b973 100644 --- a/internal/p2p/transport_mconn.go +++ b/internal/p2p/transport_mconn.go @@ -9,6 +9,7 @@ import ( "net" "strconv" "sync" + "time" "golang.org/x/net/netutil" @@ -274,6 +275,7 @@ func newMConnConnection( // Handshake implements Connection. func (c *mConnConnection) Handshake( ctx context.Context, + timeout time.Duration, nodeInfo types.NodeInfo, privKey crypto.PrivKey, ) (types.NodeInfo, crypto.PubKey, error) { @@ -283,6 +285,12 @@ func (c *mConnConnection) Handshake( peerKey crypto.PubKey errCh = make(chan error, 1) ) + handshakeCtx := ctx + if timeout > 0 { + var cancel context.CancelFunc + handshakeCtx, cancel = context.WithTimeout(ctx, timeout) + defer cancel() + } // To handle context cancellation, we need to do the handshake in a // goroutine and abort the blocking network calls by closing the connection // when the context is canceled. @@ -295,17 +303,17 @@ func (c *mConnConnection) Handshake( } }() var err error - mconn, peerInfo, peerKey, err = c.handshake(ctx, nodeInfo, privKey) + mconn, peerInfo, peerKey, err = c.handshake(handshakeCtx, nodeInfo, privKey) select { case errCh <- err: - case <-ctx.Done(): + case <-handshakeCtx.Done(): } }() select { - case <-ctx.Done(): + case <-handshakeCtx.Done(): _ = c.Close() return types.NodeInfo{}, nil, ctx.Err() @@ -314,6 +322,10 @@ func (c *mConnConnection) Handshake( return types.NodeInfo{}, nil, err } c.mconn = mconn + // Start must not use the handshakeCtx. The handshakeCtx may have a + // timeout set that is intended to terminate only the handshake procedure. + // The context passed to Start controls the entire lifecycle of the + // mconn. if err = c.mconn.Start(ctx); err != nil { return types.NodeInfo{}, nil, err } diff --git a/internal/p2p/transport_memory.go b/internal/p2p/transport_memory.go index 3eb4c5b51..c321bc174 100644 --- a/internal/p2p/transport_memory.go +++ b/internal/p2p/transport_memory.go @@ -7,6 +7,7 @@ import ( "io" "net" "sync" + "time" "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/libs/log" @@ -273,9 +274,16 @@ func (c *MemoryConnection) RemoteEndpoint() Endpoint { // Handshake implements Connection. func (c *MemoryConnection) Handshake( ctx context.Context, + timeout time.Duration, nodeInfo types.NodeInfo, privKey crypto.PrivKey, ) (types.NodeInfo, crypto.PubKey, error) { + if timeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, timeout) + defer cancel() + } + select { case c.sendCh <- memoryMessage{nodeInfo: &nodeInfo, pubKey: privKey.PubKey()}: c.logger.Debug("sent handshake", "nodeInfo", nodeInfo) diff --git a/internal/p2p/transport_test.go b/internal/p2p/transport_test.go index b4edf9bc9..d58c23955 100644 --- a/internal/p2p/transport_test.go +++ b/internal/p2p/transport_test.go @@ -296,7 +296,7 @@ func TestConnection_Handshake(t *testing.T) { errCh := make(chan error, 1) go func() { // Must use assert due to goroutine. - peerInfo, peerKey, err := ba.Handshake(ctx, bInfo, bKey) + peerInfo, peerKey, err := ba.Handshake(ctx, 0, bInfo, bKey) if err == nil { assert.Equal(t, aInfo, peerInfo) assert.Equal(t, aKey.PubKey(), peerKey) @@ -307,7 +307,7 @@ func TestConnection_Handshake(t *testing.T) { } }() - peerInfo, peerKey, err := ab.Handshake(ctx, aInfo, aKey) + peerInfo, peerKey, err := ab.Handshake(ctx, 0, aInfo, aKey) require.NoError(t, err) require.Equal(t, bInfo, peerInfo) require.Equal(t, bKey.PubKey(), peerKey) @@ -328,7 +328,7 @@ func TestConnection_HandshakeCancel(t *testing.T) { ab, ba := dialAccept(ctx, t, a, b) timeoutCtx, cancel := context.WithTimeout(ctx, 1*time.Minute) cancel() - _, _, err := ab.Handshake(timeoutCtx, types.NodeInfo{}, ed25519.GenPrivKey()) + _, _, err := ab.Handshake(timeoutCtx, 0, types.NodeInfo{}, ed25519.GenPrivKey()) require.Error(t, err) require.Equal(t, context.Canceled, err) _ = ab.Close() @@ -338,7 +338,7 @@ func TestConnection_HandshakeCancel(t *testing.T) { ab, ba = dialAccept(ctx, t, a, b) timeoutCtx, cancel = context.WithTimeout(ctx, 200*time.Millisecond) defer cancel() - _, _, err = ab.Handshake(timeoutCtx, types.NodeInfo{}, ed25519.GenPrivKey()) + _, _, err = ab.Handshake(timeoutCtx, 0, types.NodeInfo{}, ed25519.GenPrivKey()) require.Error(t, err) require.Equal(t, context.DeadlineExceeded, err) _ = ab.Close() @@ -642,13 +642,13 @@ func dialAcceptHandshake(ctx context.Context, t *testing.T, a, b p2p.Transport) go func() { privKey := ed25519.GenPrivKey() nodeInfo := types.NodeInfo{NodeID: types.NodeIDFromPubKey(privKey.PubKey())} - _, _, err := ba.Handshake(ctx, nodeInfo, privKey) + _, _, err := ba.Handshake(ctx, 0, nodeInfo, privKey) errCh <- err }() privKey := ed25519.GenPrivKey() nodeInfo := types.NodeInfo{NodeID: types.NodeIDFromPubKey(privKey.PubKey())} - _, _, err := ab.Handshake(ctx, nodeInfo, privKey) + _, _, err := ab.Handshake(ctx, 0, nodeInfo, privKey) require.NoError(t, err) timer := time.NewTimer(2 * time.Second) diff --git a/node/node.go b/node/node.go index 77773044b..187a7a1d6 100644 --- a/node/node.go +++ b/node/node.go @@ -715,7 +715,9 @@ func loadStateFromDBOrGenesisDocProvider(stateStore sm.Store, genDoc *types.Gene func getRouterConfig(conf *config.Config, appClient abciclient.Client) p2p.RouterOptions { opts := p2p.RouterOptions{ - QueueType: conf.P2P.QueueType, + QueueType: conf.P2P.QueueType, + HandshakeTimeout: conf.P2P.HandshakeTimeout, + DialTimeout: conf.P2P.DialTimeout, } if conf.FilterPeers && appClient != nil { From 6b5053046ac2390d513dc8c8b832cf92b6b3d8f4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 24 Jun 2022 13:12:17 +0000 Subject: [PATCH 36/61] build(deps): Bump github.com/stretchr/testify from 1.7.2 to 1.7.5 (#8864) Bumps [github.com/stretchr/testify](https://github.com/stretchr/testify) from 1.7.2 to 1.7.5.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/stretchr/testify&package-manager=go_modules&previous-version=1.7.2&new-version=1.7.5)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- go.mod | 4 ++-- go.sum | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 8dd4c9288..fde7586ad 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/snikch/goodman v0.0.0-20171125024755-10e37e294daa github.com/spf13/cobra v1.5.0 github.com/spf13/viper v1.12.0 - github.com/stretchr/testify v1.7.2 + github.com/stretchr/testify v1.7.5 github.com/tendermint/tm-db v0.6.6 golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e golang.org/x/net v0.0.0-20220617184016-355a448f1bc9 @@ -199,7 +199,7 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/ssgreg/nlreturn/v2 v2.2.1 // indirect github.com/stbenjam/no-sprintf-host-port v0.1.1 // indirect - github.com/stretchr/objx v0.1.1 // indirect + github.com/stretchr/objx v0.4.0 // indirect github.com/subosito/gotenv v1.4.0 // indirect github.com/sylvia7788/contextcheck v1.0.4 // indirect github.com/tdakkota/asciicheck v0.1.1 // indirect diff --git a/go.sum b/go.sum index bdb93f91e..e055c300e 100644 --- a/go.sum +++ b/go.sum @@ -1064,8 +1064,9 @@ github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3 github.com/streadway/amqp v1.0.0/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/handy v0.0.0-20200128134331-0f66f006fb2e/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v0.0.0-20170130113145-4d4bfba8f1d1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.1.4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -1075,8 +1076,9 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +github.com/stretchr/testify v1.7.5 h1:s5PTfem8p8EbKQOctVV53k6jCJt3UX4IEJzwh+C324Q= +github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/subosito/gotenv v1.3.0/go.mod h1:YzJjq/33h7nrwdY+iHMhEOEEbW0ovIz0tB6t6PwAXzs= github.com/subosito/gotenv v1.4.0 h1:yAzM1+SmVcz5R4tXGsNMu1jUl2aOJXoiWUCEwwnGrvs= From 409e057d73a854ea207ebb459733c2c7779b407a Mon Sep 17 00:00:00 2001 From: William Banfield <4561443+williambanfield@users.noreply.github.com> Date: Fri, 24 Jun 2022 12:10:27 -0400 Subject: [PATCH 37/61] fix light client select statement (#8871) --- light/client.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/light/client.go b/light/client.go index dfb4fccf7..f38e9d59d 100644 --- a/light/client.go +++ b/light/client.go @@ -1034,7 +1034,12 @@ func (c *Client) findNewPrimary(ctx context.Context, height int64, remove bool) // process all the responses as they come in for i := 0; i < cap(witnessResponsesC); i++ { - response := <-witnessResponsesC + var response witnessResponse + select { + case response = <-witnessResponsesC: + case <-ctx.Done(): + return nil, ctx.Err() + } switch response.err { // success! We have found a new primary case nil: @@ -1063,10 +1068,6 @@ func (c *Client) findNewPrimary(ctx context.Context, height int64, remove bool) // return the light block that new primary responded with return response.lb, nil - // catch canceled contexts or deadlines - case context.Canceled, context.DeadlineExceeded: - return nil, response.err - // process benign errors by logging them only case provider.ErrNoResponse, provider.ErrLightBlockNotFound, provider.ErrHeightTooHigh: lastError = response.err From c4d24eed7d0b8902c41d790e1de446baa0256e6b Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Fri, 24 Jun 2022 18:31:30 +0200 Subject: [PATCH 38/61] e2e: disable another network test (#8862) Follow up on: https://github.com/tendermint/tendermint/pull/8849 --- test/e2e/tests/net_test.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/test/e2e/tests/net_test.go b/test/e2e/tests/net_test.go index 939c136c7..43cab77e8 100644 --- a/test/e2e/tests/net_test.go +++ b/test/e2e/tests/net_test.go @@ -20,7 +20,7 @@ func TestNet_Peers(t *testing.T) { // FIXME: https://github.com/tendermint/tendermint/issues/8848 // We should be able to assert that we can discover all peers in a network - expectedPeers := len(node.Testnet.Nodes) - 1 // includes extra tolerance + expectedPeers := len(node.Testnet.Nodes) peers := make(map[string]*e2e.Node, 0) seen := map[string]bool{} for _, n := range node.Testnet.Nodes { @@ -33,7 +33,7 @@ func TestNet_Peers(t *testing.T) { seen[n.Name] = false } - require.GreaterOrEqual(t, netInfo.NPeers, expectedPeers, + require.GreaterOrEqual(t, netInfo.NPeers, expectedPeers-1, "node is not fully meshed with peers") for _, peerInfo := range netInfo.Peers { @@ -45,8 +45,10 @@ func TestNet_Peers(t *testing.T) { seen[peer.Name] = true } - for name := range seen { - require.True(t, seen[name], "node %v not peered with %v", node.Name, name) - } + // FIXME: https://github.com/tendermint/tendermint/issues/8848 + // We should be able to assert that we can discover all peers in a network + // for name := range seen { + // require.True(t, seen[name], "node %v not peered with %v", node.Name, name) + // } }) } From 52b6dc19badf50938ae2b2a1d2e22813614e5ad5 Mon Sep 17 00:00:00 2001 From: Sam Kleinman Date: Fri, 24 Jun 2022 13:57:49 -0400 Subject: [PATCH 39/61] p2p: remove dial sleep and provide disconnect cooldown (#8839) Alternative proposal for #8826 --- internal/p2p/p2ptest/network.go | 2 +- internal/p2p/peermanager.go | 34 +++++++++++++++++++++++---- internal/p2p/router.go | 41 +-------------------------------- internal/p2p/router_test.go | 1 - node/setup.go | 23 +++++++++--------- 5 files changed, 44 insertions(+), 57 deletions(-) diff --git a/internal/p2p/p2ptest/network.go b/internal/p2p/p2ptest/network.go index 5c1b0a218..c040a08a8 100644 --- a/internal/p2p/p2ptest/network.go +++ b/internal/p2p/p2ptest/network.go @@ -269,7 +269,7 @@ func (n *Network) MakeNode(ctx context.Context, t *testing.T, opts NodeOptions) func() *types.NodeInfo { return &nodeInfo }, transport, ep, - p2p.RouterOptions{DialSleep: func(_ context.Context) {}}, + p2p.RouterOptions{}, ) require.NoError(t, err) diff --git a/internal/p2p/peermanager.go b/internal/p2p/peermanager.go index b30c5dbfb..ac2e28225 100644 --- a/internal/p2p/peermanager.go +++ b/internal/p2p/peermanager.go @@ -144,6 +144,10 @@ type PeerManagerOptions struct { // retry times, to avoid thundering herds. 0 disables jitter. RetryTimeJitter time.Duration + // DisconnectCooldownPeriod is the amount of time after we + // disconnect from a peer before we'll consider dialing a new peer + DisconnectCooldownPeriod time.Duration + // PeerScores sets fixed scores for specific peers. It is mainly used // for testing. A score of 0 is ignored. PeerScores map[types.NodeID]PeerScore @@ -549,6 +553,10 @@ func (m *PeerManager) TryDialNext() NodeAddress { continue } + if !peer.LastDisconnected.IsZero() && time.Since(peer.LastDisconnected) < m.options.DisconnectCooldownPeriod { + continue + } + for _, addressInfo := range peer.AddressInfo { if time.Since(addressInfo.LastDialFailure) < m.retryDelay(addressInfo.DialFailures, peer.Persistent) { continue @@ -867,6 +875,22 @@ func (m *PeerManager) Disconnected(ctx context.Context, peerID types.NodeID) { delete(m.evicting, peerID) delete(m.ready, peerID) + if peer, ok := m.store.Get(peerID); ok { + peer.LastDisconnected = time.Now() + _ = m.store.Set(peer) + // launch a thread to ping the dialWaker when the + // disconnected peer can be dialed again. + go func() { + timer := time.NewTimer(m.options.DisconnectCooldownPeriod) + defer timer.Stop() + select { + case <-timer.C: + m.dialWaker.Wake() + case <-ctx.Done(): + } + }() + } + if ready { m.broadcast(ctx, PeerUpdate{ NodeID: peerID, @@ -1447,9 +1471,10 @@ func (s *peerStore) Size() int { // peerInfo contains peer information stored in a peerStore. type peerInfo struct { - ID types.NodeID - AddressInfo map[NodeAddress]*peerAddressInfo - LastConnected time.Time + ID types.NodeID + AddressInfo map[NodeAddress]*peerAddressInfo + LastConnected time.Time + LastDisconnected time.Time // These fields are ephemeral, i.e. not persisted to the database. Persistent bool @@ -1489,8 +1514,8 @@ func peerInfoFromProto(msg *p2pproto.PeerInfo) (*peerInfo, error) { func (p *peerInfo) ToProto() *p2pproto.PeerInfo { msg := &p2pproto.PeerInfo{ ID: string(p.ID), - LastConnected: &p.LastConnected, Inactive: p.Inactive, + LastConnected: &p.LastConnected, } for _, addressInfo := range p.AddressInfo { msg.AddressInfo = append(msg.AddressInfo, addressInfo.ToProto()) @@ -1498,6 +1523,7 @@ func (p *peerInfo) ToProto() *p2pproto.PeerInfo { if msg.LastConnected.IsZero() { msg.LastConnected = nil } + return msg } diff --git a/internal/p2p/router.go b/internal/p2p/router.go index d5edd42be..8b7db9a03 100644 --- a/internal/p2p/router.go +++ b/internal/p2p/router.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "io" - "math/rand" "net" "runtime" "sync" @@ -62,13 +61,7 @@ type RouterOptions struct { // return an error to reject the peer. FilterPeerByID func(context.Context, types.NodeID) error - // DialSleep controls the amount of time that the router - // sleeps between dialing peers. If not set, a default value - // is used that sleeps for a (random) amount of time up to 3 - // seconds between submitting each peer to be dialed. - DialSleep func(context.Context) - - // NumConcurrentDials controls how many parallel go routines + // NumConcrruentDials controls how many parallel go routines // are used to dial peers. This defaults to the value of // runtime.NumCPU. NumConcurrentDials func() int @@ -439,33 +432,6 @@ func (r *Router) filterPeersID(ctx context.Context, id types.NodeID) error { return r.options.FilterPeerByID(ctx, id) } -func (r *Router) dialSleep(ctx context.Context) { - if r.options.DialSleep == nil { - // the connTracker (on the other side) only rate - // limits peers for dialing more than once every 10ms, - // so these numbers are safe. - const ( - maxDialerInterval = 500 // ms - minDialerInterval = 100 // ms - ) - - // nolint:gosec // G404: Use of weak random number generator - dur := time.Duration(rand.Int63n(maxDialerInterval-minDialerInterval+1) + minDialerInterval) - - timer := time.NewTimer(dur * time.Millisecond) - defer timer.Stop() - - select { - case <-ctx.Done(): - case <-timer.C: - } - - return - } - - r.options.DialSleep(ctx) -} - // acceptPeers accepts inbound connections from peers on the given transport, // and spawns goroutines that route messages to/from them. func (r *Router) acceptPeers(ctx context.Context, transport Transport) { @@ -590,11 +556,6 @@ LOOP: select { case addresses <- address: - // this jitters the frequency that we call - // DialNext and prevents us from attempting to - // create connections too quickly. - - r.dialSleep(ctx) continue LOOP case <-ctx.Done(): close(addresses) diff --git a/internal/p2p/router_test.go b/internal/p2p/router_test.go index 8a58146fd..92f56f768 100644 --- a/internal/p2p/router_test.go +++ b/internal/p2p/router_test.go @@ -715,7 +715,6 @@ func TestRouter_DialPeers_Parallel(t *testing.T) { mockTransport, nil, p2p.RouterOptions{ - DialSleep: func(_ context.Context) {}, NumConcurrentDials: func() int { ncpu := runtime.NumCPU() if ncpu <= 3 { diff --git a/node/setup.go b/node/setup.go index 345489f8d..df9d19195 100644 --- a/node/setup.go +++ b/node/setup.go @@ -235,17 +235,18 @@ func createPeerManager( maxUpgradeConns := uint16(4) options := p2p.PeerManagerOptions{ - SelfAddress: selfAddr, - MaxConnected: maxConns, - MaxOutgoingConnections: maxOutgoingConns, - MaxConnectedUpgrade: maxUpgradeConns, - MaxPeers: maxUpgradeConns + 4*maxConns, - MinRetryTime: 250 * time.Millisecond, - MaxRetryTime: 30 * time.Minute, - MaxRetryTimePersistent: 5 * time.Minute, - RetryTimeJitter: 5 * time.Second, - PrivatePeers: privatePeerIDs, - Metrics: metrics, + SelfAddress: selfAddr, + MaxConnected: maxConns, + MaxOutgoingConnections: maxOutgoingConns, + MaxConnectedUpgrade: maxUpgradeConns, + DisconnectCooldownPeriod: 2 * time.Second, + MaxPeers: maxUpgradeConns + 4*maxConns, + MinRetryTime: 250 * time.Millisecond, + MaxRetryTime: 30 * time.Minute, + MaxRetryTimePersistent: 5 * time.Minute, + RetryTimeJitter: 5 * time.Second, + PrivatePeers: privatePeerIDs, + Metrics: metrics, } peers := []p2p.NodeAddress{} From 27ff2f46b8b5d54c986228a84edd053b2809c7aa Mon Sep 17 00:00:00 2001 From: Sergio Mena Date: Fri, 24 Jun 2022 20:53:31 +0200 Subject: [PATCH 40/61] Add @sergio-mena and @jmalicevic to list of spec reviewers (#8870) Co-authored-by: Callum Waters --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 35a27cfe7..950ed1dc2 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -10,4 +10,4 @@ * @ebuchman @cmwaters @tychoish @williambanfield @creachadair @sergio-mena @jmalicevic @thanethomson @ancazamfir # Spec related changes can be approved by the protocol design team -/spec @josef-widder @milosevic @cason +/spec @josef-widder @milosevic @cason @sergio-mena @jmalicevic From 463cff456b36ba7a64b6dd269163bbb661eb21da Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jun 2022 09:13:25 -0400 Subject: [PATCH 41/61] build(deps): Bump bufbuild/buf-setup-action from 1.5.0 to 1.6.0 (#8883) Bumps [bufbuild/buf-setup-action](https://github.com/bufbuild/buf-setup-action) from 1.5.0 to 1.6.0. - [Release notes](https://github.com/bufbuild/buf-setup-action/releases) - [Commits](https://github.com/bufbuild/buf-setup-action/compare/v1.5.0...v1.6.0) --- updated-dependencies: - dependency-name: bufbuild/buf-setup-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/proto-lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/proto-lint.yml b/.github/workflows/proto-lint.yml index f8fff89cc..377e43ca6 100644 --- a/.github/workflows/proto-lint.yml +++ b/.github/workflows/proto-lint.yml @@ -15,7 +15,7 @@ jobs: timeout-minutes: 5 steps: - uses: actions/checkout@v3 - - uses: bufbuild/buf-setup-action@v1.5.0 + - uses: bufbuild/buf-setup-action@v1.6.0 - uses: bufbuild/buf-lint-action@v1 with: input: 'proto' From 373b262f35cc6cf3ea3e7d7e51d5412f960d2177 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jun 2022 13:15:29 +0000 Subject: [PATCH 42/61] build(deps): Bump styfle/cancel-workflow-action from 0.9.1 to 0.10.0 (#8881) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [styfle/cancel-workflow-action](https://github.com/styfle/cancel-workflow-action) from 0.9.1 to 0.10.0.
Release notes

Sourced from styfle/cancel-workflow-action's releases.

0.10.0

Changes

  • Feat(all):support for considering all workflows with one term: #165
  • Chore: rebuild: 74a81dc1a9321342ebc12fa8670cc91600c8c494
  • Chore: update main.yml: #78
  • Bump @​vercel/ncc from 0.28.6 to 0.29.1: #106
  • Bump @​vercel/ncc from 0.29.1 to 0.29.2: #109
  • Bump @​vercel/ncc from 0.29.2 to 0.30.0: #112
  • Bump husky from 7.0.1 to 7.0.2: #110
  • Bump prettier from 2.3.2 to 2.4.0: #116
  • Bump @​vercel/ncc from 0.30.0 to 0.31.1: #115
  • Bump typescript from 4.3.5 to 4.4.3: #114
  • Bump prettier from 2.4.0 to 2.4.1: #117
  • Bump @​actions/github from 4.0.0 to 5.0.0: #89
  • Bump @​actions/core from 1.3.0 to 1.6.0: #118
  • Bump typescript from 4.4.3 to 4.4.4: #119
  • Bump husky from 7.0.2 to 7.0.4: #120
  • Bump typescript from 4.4.4 to 4.5.2: #124
  • Bump @​vercel/ncc from 0.31.1 to 0.32.0: #123
  • Bump prettier from 2.4.1 to 2.5.0: #125
  • Bump prettier from 2.5.0 to 2.5.1: #126
  • Bump @​vercel/ncc from 0.32.0 to 0.33.0: #127
  • Bump typescript from 4.5.2 to 4.5.3: #128
  • Bump @​vercel/ncc from 0.33.0 to 0.33.1: #130
  • Bump typescript from 4.5.3 to 4.5.4: #129
  • Bump typescript from 4.5.4 to 4.5.5: #131
  • Bump node-fetch from 2.6.5 to 2.6.7: #132
  • Bump @​vercel/ncc from 0.33.1 to 0.33.3: #138
  • Bump actions/setup-node from 2 to 3.0.0: #140
  • Bump actions/checkout from 2 to 3: #141
  • Bump typescript from 4.5.5 to 4.6.2: #142
  • Bump prettier from 2.5.1 to 2.6.0: #143
  • Bump prettier from 2.6.0 to 2.6.1: #145
  • Bump actions/setup-node from 3.0.0 to 3.1.0: #146
  • Bump typescript from 4.6.2 to 4.6.3: #144
  • Bump prettier from 2.6.1 to 2.6.2: #147
  • Bump @​actions/github from 5.0.0 to 5.0.1: #148
  • Bump actions/setup-node from 3.1.0 to 3.1.1: #149
  • Bump @​vercel/ncc from 0.33.3 to 0.33.4: #151
  • Bump @​actions/core from 1.6.0 to 1.7.0: #153
  • Bump typescript from 4.6.3 to 4.6.4: #154
  • Bump husky from 7.0.4 to 8.0.1: #155
  • Bump @​actions/core from 1.7.0 to 1.8.0: #156
  • Bump actions/setup-node from 3.1.1 to 3.2.0: #159
  • Bump @​actions/github from 5.0.1 to 5.0.3: #157
  • Bump @​actions/core from 1.8.0 to 1.8.2: #158
  • Bump typescript from 4.6.4 to 4.7.2: #160
  • Bump @​vercel/ncc from 0.33.4 to 0.34.0: #161
  • Bump typescript from 4.7.2 to 4.7.3: #163

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=styfle/cancel-workflow-action&package-manager=github_actions&previous-version=0.9.1&new-version=0.10.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- .github/workflows/janitor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/janitor.yml b/.github/workflows/janitor.yml index e6bc45ec1..ceb21941d 100644 --- a/.github/workflows/janitor.yml +++ b/.github/workflows/janitor.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 3 steps: - - uses: styfle/cancel-workflow-action@0.9.1 + - uses: styfle/cancel-workflow-action@0.10.0 with: workflow_id: 1041851,1401230,2837803 access_token: ${{ github.token }} From 013b46a6c37b53efffdad27ed1426329972d1166 Mon Sep 17 00:00:00 2001 From: Sam Kleinman Date: Mon, 27 Jun 2022 17:44:50 -0400 Subject: [PATCH 43/61] libs/strings: move to internal (#8890) I think we were leaving this library public because the SDK dependend upon it, but the function the SDK was using was one that we'd removed because *we* weren't using it any more, and I made a PR agasint the SDK to clean that up. ref: https://github.com/cosmos/cosmos-sdk/pull/12368 --- internal/inspect/inspect.go | 2 +- {libs => internal/libs}/strings/string.go | 0 {libs => internal/libs}/strings/string_test.go | 0 internal/rpc/core/env.go | 2 +- node/setup.go | 2 +- types/node_info.go | 2 +- types/params.go | 2 +- 7 files changed, 5 insertions(+), 5 deletions(-) rename {libs => internal/libs}/strings/string.go (100%) rename {libs => internal/libs}/strings/string_test.go (100%) diff --git a/internal/inspect/inspect.go b/internal/inspect/inspect.go index 6381ea888..573b63f40 100644 --- a/internal/inspect/inspect.go +++ b/internal/inspect/inspect.go @@ -10,13 +10,13 @@ import ( "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/internal/eventbus" "github.com/tendermint/tendermint/internal/inspect/rpc" + tmstrings "github.com/tendermint/tendermint/internal/libs/strings" rpccore "github.com/tendermint/tendermint/internal/rpc/core" "github.com/tendermint/tendermint/internal/state" "github.com/tendermint/tendermint/internal/state/indexer" "github.com/tendermint/tendermint/internal/state/indexer/sink" "github.com/tendermint/tendermint/internal/store" "github.com/tendermint/tendermint/libs/log" - tmstrings "github.com/tendermint/tendermint/libs/strings" "github.com/tendermint/tendermint/types" "golang.org/x/sync/errgroup" diff --git a/libs/strings/string.go b/internal/libs/strings/string.go similarity index 100% rename from libs/strings/string.go rename to internal/libs/strings/string.go diff --git a/libs/strings/string_test.go b/internal/libs/strings/string_test.go similarity index 100% rename from libs/strings/string_test.go rename to internal/libs/strings/string_test.go diff --git a/internal/rpc/core/env.go b/internal/rpc/core/env.go index 124525f26..f1ba58ee3 100644 --- a/internal/rpc/core/env.go +++ b/internal/rpc/core/env.go @@ -18,6 +18,7 @@ import ( "github.com/tendermint/tendermint/internal/consensus" "github.com/tendermint/tendermint/internal/eventbus" "github.com/tendermint/tendermint/internal/eventlog" + "github.com/tendermint/tendermint/internal/libs/strings" "github.com/tendermint/tendermint/internal/mempool" "github.com/tendermint/tendermint/internal/p2p" tmpubsub "github.com/tendermint/tendermint/internal/pubsub" @@ -26,7 +27,6 @@ import ( "github.com/tendermint/tendermint/internal/state/indexer" "github.com/tendermint/tendermint/internal/statesync" "github.com/tendermint/tendermint/libs/log" - "github.com/tendermint/tendermint/libs/strings" "github.com/tendermint/tendermint/rpc/coretypes" rpcserver "github.com/tendermint/tendermint/rpc/jsonrpc/server" "github.com/tendermint/tendermint/types" diff --git a/node/setup.go b/node/setup.go index df9d19195..38ad58518 100644 --- a/node/setup.go +++ b/node/setup.go @@ -17,6 +17,7 @@ import ( "github.com/tendermint/tendermint/internal/consensus" "github.com/tendermint/tendermint/internal/eventbus" "github.com/tendermint/tendermint/internal/evidence" + tmstrings "github.com/tendermint/tendermint/internal/libs/strings" "github.com/tendermint/tendermint/internal/mempool" "github.com/tendermint/tendermint/internal/p2p" "github.com/tendermint/tendermint/internal/p2p/conn" @@ -28,7 +29,6 @@ import ( "github.com/tendermint/tendermint/libs/log" tmnet "github.com/tendermint/tendermint/libs/net" "github.com/tendermint/tendermint/libs/service" - tmstrings "github.com/tendermint/tendermint/libs/strings" "github.com/tendermint/tendermint/privval" tmgrpc "github.com/tendermint/tendermint/privval/grpc" "github.com/tendermint/tendermint/types" diff --git a/types/node_info.go b/types/node_info.go index fd47816e2..bc614f55e 100644 --- a/types/node_info.go +++ b/types/node_info.go @@ -7,8 +7,8 @@ import ( "strconv" "strings" + tmstrings "github.com/tendermint/tendermint/internal/libs/strings" "github.com/tendermint/tendermint/libs/bytes" - tmstrings "github.com/tendermint/tendermint/libs/strings" tmp2p "github.com/tendermint/tendermint/proto/tendermint/p2p" ) diff --git a/types/params.go b/types/params.go index 28c969e46..754b50322 100644 --- a/types/params.go +++ b/types/params.go @@ -9,7 +9,7 @@ import ( "github.com/tendermint/tendermint/crypto/ed25519" "github.com/tendermint/tendermint/crypto/secp256k1" "github.com/tendermint/tendermint/crypto/sr25519" - tmstrings "github.com/tendermint/tendermint/libs/strings" + tmstrings "github.com/tendermint/tendermint/internal/libs/strings" tmproto "github.com/tendermint/tendermint/proto/tendermint/types" ) From 37f9d59969b03d49dc1ff4191b4a5ddac2bc8d13 Mon Sep 17 00:00:00 2001 From: Sam Kleinman Date: Tue, 28 Jun 2022 10:40:16 -0400 Subject: [PATCH 44/61] log: do not pre-process log results (#8895) I was digging around in the zero log functions, and the following functions using the `Fields()` method directly in zerolog, - https://github.com/rs/zerolog/blob/v1.27.0/event.go#L161 - https://github.com/rs/zerolog/blob/e9344a8c507b5f25a4962ff022526be0ddab8e72/fields.go#L15 Have meaningfully equivalent semantics and our pre-processing of values is getting us much (except forcing zerolog to always sort our keys, and nooping in the case when you miss the last field.) With this change also, we can pass maps (or, pratically a single map) to the logger, which might be a less wacky seeming interface. --- libs/log/default.go | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/libs/log/default.go b/libs/log/default.go index 706977659..557ba6551 100644 --- a/libs/log/default.go +++ b/libs/log/default.go @@ -61,20 +61,20 @@ func NewDefaultLogger(format, level string) (Logger, error) { } func (l defaultLogger) Info(msg string, keyVals ...interface{}) { - l.Logger.Info().Fields(getLogFields(keyVals...)).Msg(msg) + l.Logger.Info().Fields(keyVals).Msg(msg) } func (l defaultLogger) Error(msg string, keyVals ...interface{}) { - l.Logger.Error().Fields(getLogFields(keyVals...)).Msg(msg) + l.Logger.Error().Fields(keyVals).Msg(msg) } func (l defaultLogger) Debug(msg string, keyVals ...interface{}) { - l.Logger.Debug().Fields(getLogFields(keyVals...)).Msg(msg) + l.Logger.Debug().Fields(keyVals).Msg(msg) } func (l defaultLogger) With(keyVals ...interface{}) Logger { return &defaultLogger{ - Logger: l.Logger.With().Fields(getLogFields(keyVals...)).Logger(), + Logger: l.Logger.With().Fields(keyVals).Logger(), } } @@ -99,16 +99,3 @@ func OverrideWithNewLogger(logger Logger, format, level string) error { ol.Logger = nl.Logger return nil } - -func getLogFields(keyVals ...interface{}) map[string]interface{} { - if len(keyVals)%2 != 0 { - return nil - } - - fields := make(map[string]interface{}, len(keyVals)) - for i := 0; i < len(keyVals); i += 2 { - fields[fmt.Sprint(keyVals[i])] = keyVals[i+1] - } - - return fields -} From 3bec1668c61ee8d0afcb870836e34905463c1399 Mon Sep 17 00:00:00 2001 From: Thane Thomson Date: Wed, 29 Jun 2022 08:02:05 -0400 Subject: [PATCH 45/61] e2e: Extract Docker-specific functionality (#8754) * e2e: Extract Docker-specific functionality Extract Docker-specific functionality and put it behind an interface that should hopefully, without too much modification, allow us to implement a Digital Ocean-based infrastructure provider. Signed-off-by: Thane Thomson * Thread contexts through all potentially long-running functions Signed-off-by: Thane Thomson * Drop the "API" from interface/struct/var naming Signed-off-by: Thane Thomson * Simplify function returns Signed-off-by: Thane Thomson * Rename GenerateConfig to Setup to make it more generic Signed-off-by: Thane Thomson * Consolidate all infra functions into a single interface Signed-off-by: Thane Thomson * Localize linter directives Signed-off-by: Thane Thomson * Look up and use complete node in ShowNodeLogs and TailNodeLogs calls Signed-off-by: Thane Thomson * Restructure infra provider API into a separate package Signed-off-by: Thane Thomson * Rename interface again Signed-off-by: Thane Thomson * Rename exec functions for readability Signed-off-by: Thane Thomson * Relocate staticcheck lint directive Signed-off-by: Thane Thomson * Remove staticcheck lint directive Signed-off-by: Thane Thomson * Make testnet infra struct private Signed-off-by: Thane Thomson * Only pass testnetDir to Cleanup function Signed-off-by: Thane Thomson --- test/e2e/generator/main.go | 3 +- test/e2e/pkg/exec/exec.go | 34 +++++++ test/e2e/pkg/infra/docker/compose.go | 69 +++++++++++++ test/e2e/pkg/infra/docker/exec.go | 27 ++++++ test/e2e/pkg/infra/docker/infra.go | 140 +++++++++++++++++++++++++++ test/e2e/pkg/infra/infra.go | 84 ++++++++++++++++ test/e2e/pkg/testnet.go | 5 +- test/e2e/runner/cleanup.go | 58 ++--------- test/e2e/runner/exec.go | 50 ---------- test/e2e/runner/main.go | 75 +++++++++----- test/e2e/runner/perturb.go | 24 ++--- test/e2e/runner/setup.go | 84 +++------------- test/e2e/runner/start.go | 13 +-- test/e2e/runner/test.go | 6 +- 14 files changed, 454 insertions(+), 218 deletions(-) create mode 100644 test/e2e/pkg/exec/exec.go create mode 100644 test/e2e/pkg/infra/docker/compose.go create mode 100644 test/e2e/pkg/infra/docker/exec.go create mode 100644 test/e2e/pkg/infra/docker/infra.go create mode 100644 test/e2e/pkg/infra/infra.go delete mode 100644 test/e2e/runner/exec.go diff --git a/test/e2e/generator/main.go b/test/e2e/generator/main.go index bec78d89c..da32b2831 100644 --- a/test/e2e/generator/main.go +++ b/test/e2e/generator/main.go @@ -1,4 +1,3 @@ -//nolint: gosec package main import ( @@ -77,6 +76,8 @@ func (cli *CLI) generate() error { return err } + // nolint: gosec + // G404: Use of weak random number generator (math/rand instead of crypto/rand) manifests, err := Generate(rand.New(rand.NewSource(randomSeed)), cli.opts) if err != nil { return err diff --git a/test/e2e/pkg/exec/exec.go b/test/e2e/pkg/exec/exec.go new file mode 100644 index 000000000..9dcd79384 --- /dev/null +++ b/test/e2e/pkg/exec/exec.go @@ -0,0 +1,34 @@ +package exec + +import ( + "context" + "fmt" + "os" + osexec "os/exec" +) + +// Command executes a shell command. +func Command(ctx context.Context, args ...string) error { + // nolint: gosec + // G204: Subprocess launched with a potential tainted input or cmd arguments + cmd := osexec.CommandContext(ctx, args[0], args[1:]...) + out, err := cmd.CombinedOutput() + switch err := err.(type) { + case nil: + return nil + case *osexec.ExitError: + return fmt.Errorf("failed to run %q:\n%v", args, string(out)) + default: + return err + } +} + +// CommandVerbose executes a shell command while displaying its output. +func CommandVerbose(ctx context.Context, args ...string) error { + // nolint: gosec + // G204: Subprocess launched with a potential tainted input or cmd arguments + cmd := osexec.CommandContext(ctx, args[0], args[1:]...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} diff --git a/test/e2e/pkg/infra/docker/compose.go b/test/e2e/pkg/infra/docker/compose.go new file mode 100644 index 000000000..3ea5845ee --- /dev/null +++ b/test/e2e/pkg/infra/docker/compose.go @@ -0,0 +1,69 @@ +package docker + +import ( + "bytes" + "text/template" + + e2e "github.com/tendermint/tendermint/test/e2e/pkg" +) + +// makeDockerCompose generates a Docker Compose config for a testnet. +func makeDockerCompose(testnet *e2e.Testnet) ([]byte, error) { + // Must use version 2 Docker Compose format, to support IPv6. + tmpl, err := template.New("docker-compose").Funcs(template.FuncMap{ + "addUint32": func(x, y uint32) uint32 { + return x + y + }, + "isBuiltin": func(protocol e2e.Protocol, mode e2e.Mode) bool { + return mode == e2e.ModeLight || protocol == e2e.ProtocolBuiltin + }, + }).Parse(`version: '2.4' + +networks: + {{ .Name }}: + labels: + e2e: true + driver: bridge +{{- if .IPv6 }} + enable_ipv6: true +{{- end }} + ipam: + driver: default + config: + - subnet: {{ .IP }} + +services: +{{- range .Nodes }} + {{ .Name }}: + labels: + e2e: true + container_name: {{ .Name }} + image: tendermint/e2e-node +{{- if isBuiltin $.ABCIProtocol .Mode }} + entrypoint: /usr/bin/entrypoint-builtin +{{- else if .LogLevel }} + command: start --log-level {{ .LogLevel }} +{{- end }} + init: true + ports: + - 26656 + - {{ if .ProxyPort }}{{ addUint32 .ProxyPort 1000 }}:{{ end }}26660 + - {{ if .ProxyPort }}{{ .ProxyPort }}:{{ end }}26657 + - 6060 + volumes: + - ./{{ .Name }}:/tendermint + networks: + {{ $.Name }}: + ipv{{ if $.IPv6 }}6{{ else }}4{{ end}}_address: {{ .IP }} + +{{end}}`) + if err != nil { + return nil, err + } + var buf bytes.Buffer + err = tmpl.Execute(&buf, testnet) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} diff --git a/test/e2e/pkg/infra/docker/exec.go b/test/e2e/pkg/infra/docker/exec.go new file mode 100644 index 000000000..de0033e32 --- /dev/null +++ b/test/e2e/pkg/infra/docker/exec.go @@ -0,0 +1,27 @@ +package docker + +import ( + "context" + "path/filepath" + + "github.com/tendermint/tendermint/test/e2e/pkg/exec" +) + +// execCompose runs a Docker Compose command for a testnet. +func execCompose(ctx context.Context, dir string, args ...string) error { + return exec.Command(ctx, append( + []string{"docker-compose", "--ansi=never", "-f", filepath.Join(dir, "docker-compose.yml")}, + args...)...) +} + +// execComposeVerbose runs a Docker Compose command for a testnet and displays its output. +func execComposeVerbose(ctx context.Context, dir string, args ...string) error { + return exec.CommandVerbose(ctx, append( + []string{"docker-compose", "--ansi=never", "-f", filepath.Join(dir, "docker-compose.yml")}, + args...)...) +} + +// execDocker runs a Docker command. +func execDocker(ctx context.Context, args ...string) error { + return exec.Command(ctx, append([]string{"docker"}, args...)...) +} diff --git a/test/e2e/pkg/infra/docker/infra.go b/test/e2e/pkg/infra/docker/infra.go new file mode 100644 index 000000000..9827be241 --- /dev/null +++ b/test/e2e/pkg/infra/docker/infra.go @@ -0,0 +1,140 @@ +package docker + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "github.com/tendermint/tendermint/libs/log" + e2e "github.com/tendermint/tendermint/test/e2e/pkg" + "github.com/tendermint/tendermint/test/e2e/pkg/exec" + "github.com/tendermint/tendermint/test/e2e/pkg/infra" +) + +// testnetInfra provides an API for provisioning and manipulating +// infrastructure for a Docker-based testnet. +type testnetInfra struct { + logger log.Logger + testnet *e2e.Testnet +} + +var _ infra.TestnetInfra = &testnetInfra{} + +// NewTestnetInfra constructs an infrastructure provider that allows for Docker-based +// testnet infrastructure. +func NewTestnetInfra(logger log.Logger, testnet *e2e.Testnet) infra.TestnetInfra { + return &testnetInfra{ + logger: logger, + testnet: testnet, + } +} + +func (ti *testnetInfra) Setup(ctx context.Context) error { + compose, err := makeDockerCompose(ti.testnet) + if err != nil { + return err + } + // nolint: gosec + // G306: Expect WriteFile permissions to be 0600 or less + err = os.WriteFile(filepath.Join(ti.testnet.Dir, "docker-compose.yml"), compose, 0644) + if err != nil { + return err + } + return nil +} + +func (ti *testnetInfra) StartNode(ctx context.Context, node *e2e.Node) error { + return execCompose(ctx, ti.testnet.Dir, "up", "-d", node.Name) +} + +func (ti *testnetInfra) DisconnectNode(ctx context.Context, node *e2e.Node) error { + return execDocker(ctx, "network", "disconnect", ti.testnet.Name+"_"+ti.testnet.Name, node.Name) +} + +func (ti *testnetInfra) ConnectNode(ctx context.Context, node *e2e.Node) error { + return execDocker(ctx, "network", "connect", ti.testnet.Name+"_"+ti.testnet.Name, node.Name) +} + +func (ti *testnetInfra) KillNodeProcess(ctx context.Context, node *e2e.Node) error { + return execCompose(ctx, ti.testnet.Dir, "kill", "-s", "SIGKILL", node.Name) +} + +func (ti *testnetInfra) StartNodeProcess(ctx context.Context, node *e2e.Node) error { + return execCompose(ctx, ti.testnet.Dir, "start", node.Name) +} + +func (ti *testnetInfra) PauseNodeProcess(ctx context.Context, node *e2e.Node) error { + return execCompose(ctx, ti.testnet.Dir, "pause", node.Name) +} + +func (ti *testnetInfra) UnpauseNodeProcess(ctx context.Context, node *e2e.Node) error { + return execCompose(ctx, ti.testnet.Dir, "unpause", node.Name) +} + +func (ti *testnetInfra) TerminateNodeProcess(ctx context.Context, node *e2e.Node) error { + return execCompose(ctx, ti.testnet.Dir, "kill", "-s", "SIGTERM", node.Name) +} + +func (ti *testnetInfra) Stop(ctx context.Context) error { + return execCompose(ctx, ti.testnet.Dir, "down") +} + +func (ti *testnetInfra) Pause(ctx context.Context) error { + return execCompose(ctx, ti.testnet.Dir, "pause") +} + +func (ti *testnetInfra) Unpause(ctx context.Context) error { + return execCompose(ctx, ti.testnet.Dir, "unpause") +} + +func (ti *testnetInfra) ShowLogs(ctx context.Context) error { + return execComposeVerbose(ctx, ti.testnet.Dir, "logs", "--no-color") +} + +func (ti *testnetInfra) ShowNodeLogs(ctx context.Context, node *e2e.Node) error { + return execComposeVerbose(ctx, ti.testnet.Dir, "logs", "--no-color", node.Name) +} + +func (ti *testnetInfra) TailLogs(ctx context.Context) error { + return execComposeVerbose(ctx, ti.testnet.Dir, "logs", "--follow") +} + +func (ti *testnetInfra) TailNodeLogs(ctx context.Context, node *e2e.Node) error { + return execComposeVerbose(ctx, ti.testnet.Dir, "logs", "--follow", node.Name) +} + +func (ti *testnetInfra) Cleanup(ctx context.Context) error { + ti.logger.Info("Removing Docker containers and networks") + + // GNU xargs requires the -r flag to not run when input is empty, macOS + // does this by default. Ugly, but works. + xargsR := `$(if [[ $OSTYPE == "linux-gnu"* ]]; then echo -n "-r"; fi)` + + err := exec.Command(ctx, "bash", "-c", fmt.Sprintf( + "docker container ls -qa --filter label=e2e | xargs %v docker container rm -f", xargsR)) + if err != nil { + return err + } + + err = exec.Command(ctx, "bash", "-c", fmt.Sprintf( + "docker network ls -q --filter label=e2e | xargs %v docker network rm", xargsR)) + if err != nil { + return err + } + + // On Linux, some local files in the volume will be owned by root since Tendermint + // runs as root inside the container, so we need to clean them up from within a + // container running as root too. + absDir, err := filepath.Abs(ti.testnet.Dir) + if err != nil { + return err + } + err = execDocker(ctx, "run", "--rm", "--entrypoint", "", "-v", fmt.Sprintf("%v:/network", absDir), + "tendermint/e2e-node", "sh", "-c", "rm -rf /network/*/") + if err != nil { + return err + } + + return nil +} diff --git a/test/e2e/pkg/infra/infra.go b/test/e2e/pkg/infra/infra.go new file mode 100644 index 000000000..2fa7c5ad9 --- /dev/null +++ b/test/e2e/pkg/infra/infra.go @@ -0,0 +1,84 @@ +package infra + +import ( + "context" + + e2e "github.com/tendermint/tendermint/test/e2e/pkg" +) + +// TestnetInfra provides an API for manipulating the infrastructure of a +// specific testnet. +type TestnetInfra interface { + // + // Overarching testnet infrastructure management. + // + + // Setup generates any necessary configuration for the infrastructure + // provider during testnet setup. + Setup(ctx context.Context) error + + // Stop will stop all running processes throughout the testnet without + // destroying any infrastructure. + Stop(ctx context.Context) error + + // Pause will pause all processes in the testnet. + Pause(ctx context.Context) error + + // Unpause will resume a paused testnet. + Unpause(ctx context.Context) error + + // ShowLogs prints all logs for the whole testnet to stdout. + ShowLogs(ctx context.Context) error + + // TailLogs tails the logs for all nodes in the testnet, if this is + // supported by the infrastructure provider. + TailLogs(ctx context.Context) error + + // Cleanup stops and destroys all running testnet infrastructure and + // deletes any generated files. + Cleanup(ctx context.Context) error + + // + // Node management, including node infrastructure. + // + + // StartNode provisions infrastructure for the given node and starts it. + StartNode(ctx context.Context, node *e2e.Node) error + + // DisconnectNode modifies the specified node's network configuration such + // that it becomes bidirectionally disconnected from the network (it cannot + // see other nodes, and other nodes cannot see it). + DisconnectNode(ctx context.Context, node *e2e.Node) error + + // ConnectNode modifies the specified node's network configuration such + // that it can become bidirectionally connected. + ConnectNode(ctx context.Context, node *e2e.Node) error + + // ShowNodeLogs prints all logs for the node with the give ID to stdout. + ShowNodeLogs(ctx context.Context, node *e2e.Node) error + + // TailNodeLogs tails the logs for a single node, if this is supported by + // the infrastructure provider. + TailNodeLogs(ctx context.Context, node *e2e.Node) error + + // + // Node process management. + // + + // KillNodeProcess sends SIGKILL to a node's process. + KillNodeProcess(ctx context.Context, node *e2e.Node) error + + // StartNodeProcess will start a stopped node's process. Assumes that the + // node's infrastructure has previously been provisioned using + // ProvisionNode. + StartNodeProcess(ctx context.Context, node *e2e.Node) error + + // PauseNodeProcess sends a signal to the node's process to pause it. + PauseNodeProcess(ctx context.Context, node *e2e.Node) error + + // UnpauseNodeProcess resumes a paused node's process. + UnpauseNodeProcess(ctx context.Context, node *e2e.Node) error + + // TerminateNodeProcess sends SIGTERM to a node's process. + TerminateNodeProcess(ctx context.Context, node *e2e.Node) error +} diff --git a/test/e2e/pkg/testnet.go b/test/e2e/pkg/testnet.go index a5c8dffd6..2041796c4 100644 --- a/test/e2e/pkg/testnet.go +++ b/test/e2e/pkg/testnet.go @@ -1,4 +1,3 @@ -//nolint: gosec package e2e import ( @@ -467,7 +466,7 @@ func (n Node) AddressRPC() string { // Client returns an RPC client for a node. func (n Node) Client() (*rpchttp.HTTP, error) { - return rpchttp.New(fmt.Sprintf("http://127.0.0.1:%v", n.ProxyPort)) + return rpchttp.New(fmt.Sprintf("http://%s", n.AddressRPC())) } // Stateless returns true if the node is either a seed node or a light node @@ -481,6 +480,8 @@ type keyGenerator struct { } func newKeyGenerator(seed int64) *keyGenerator { + // nolint: gosec + // G404: Use of weak random number generator (math/rand instead of crypto/rand) return &keyGenerator{ random: rand.New(rand.NewSource(seed)), } diff --git a/test/e2e/runner/cleanup.go b/test/e2e/runner/cleanup.go index b08e39f6d..25a1008e6 100644 --- a/test/e2e/runner/cleanup.go +++ b/test/e2e/runner/cleanup.go @@ -1,70 +1,32 @@ package main import ( + "context" "errors" "fmt" "os" - "path/filepath" "github.com/tendermint/tendermint/libs/log" - e2e "github.com/tendermint/tendermint/test/e2e/pkg" + "github.com/tendermint/tendermint/test/e2e/pkg/infra" ) -// Cleanup removes the Docker Compose containers and testnet directory. -func Cleanup(logger log.Logger, testnet *e2e.Testnet) error { - err := cleanupDocker(logger) - if err != nil { - return err +// Cleanup destroys all infrastructure and removes all generated testnet files. +func Cleanup(ctx context.Context, logger log.Logger, testnetDir string, ti infra.TestnetInfra) error { + if testnetDir == "" { + return errors.New("no testnet directory set") } - return cleanupDir(logger, testnet.Dir) -} -// cleanupDocker removes all E2E resources (with label e2e=True), regardless -// of testnet. -func cleanupDocker(logger log.Logger) error { - logger.Info("Removing Docker containers and networks") - - // GNU xargs requires the -r flag to not run when input is empty, macOS - // does this by default. Ugly, but works. - xargsR := `$(if [[ $OSTYPE == "linux-gnu"* ]]; then echo -n "-r"; fi)` - - err := exec("bash", "-c", fmt.Sprintf( - "docker container ls -qa --filter label=e2e | xargs %v docker container rm -f", xargsR)) - if err != nil { + if err := ti.Cleanup(ctx); err != nil { return err } - return exec("bash", "-c", fmt.Sprintf( - "docker network ls -q --filter label=e2e | xargs %v docker network rm", xargsR)) -} - -// cleanupDir cleans up a testnet directory -func cleanupDir(logger log.Logger, dir string) error { - if dir == "" { - return errors.New("no directory set") - } - - _, err := os.Stat(dir) + _, err := os.Stat(testnetDir) if os.IsNotExist(err) { return nil } else if err != nil { return err } - logger.Info(fmt.Sprintf("Removing testnet directory %q", dir)) - - // On Linux, some local files in the volume will be owned by root since Tendermint - // runs as root inside the container, so we need to clean them up from within a - // container running as root too. - absDir, err := filepath.Abs(dir) - if err != nil { - return err - } - err = execDocker("run", "--rm", "--entrypoint", "", "-v", fmt.Sprintf("%v:/network", absDir), - "tendermint/e2e-node", "sh", "-c", "rm -rf /network/*/") - if err != nil { - return err - } - - return os.RemoveAll(dir) + logger.Info(fmt.Sprintf("Removing testnet directory %q", testnetDir)) + return os.RemoveAll(testnetDir) } diff --git a/test/e2e/runner/exec.go b/test/e2e/runner/exec.go deleted file mode 100644 index f2bc5163c..000000000 --- a/test/e2e/runner/exec.go +++ /dev/null @@ -1,50 +0,0 @@ -//nolint: gosec -package main - -import ( - "fmt" - "os" - osexec "os/exec" - "path/filepath" -) - -// execute executes a shell command. -func exec(args ...string) error { - cmd := osexec.Command(args[0], args[1:]...) - out, err := cmd.CombinedOutput() - switch err := err.(type) { - case nil: - return nil - case *osexec.ExitError: - return fmt.Errorf("failed to run %q:\n%v", args, string(out)) - default: - return err - } -} - -// execVerbose executes a shell command while displaying its output. -func execVerbose(args ...string) error { - cmd := osexec.Command(args[0], args[1:]...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - return cmd.Run() -} - -// execCompose runs a Docker Compose command for a testnet. -func execCompose(dir string, args ...string) error { - return exec(append( - []string{"docker-compose", "--ansi=never", "-f", filepath.Join(dir, "docker-compose.yml")}, - args...)...) -} - -// execComposeVerbose runs a Docker Compose command for a testnet and displays its output. -func execComposeVerbose(dir string, args ...string) error { - return execVerbose(append( - []string{"docker-compose", "--ansi=never", "-f", filepath.Join(dir, "docker-compose.yml")}, - args...)...) -} - -// execDocker runs a Docker command. -func execDocker(args ...string) error { - return exec(append([]string{"docker"}, args...)...) -} diff --git a/test/e2e/runner/main.go b/test/e2e/runner/main.go index c4a73d33f..9a24f1141 100644 --- a/test/e2e/runner/main.go +++ b/test/e2e/runner/main.go @@ -13,6 +13,8 @@ import ( "github.com/tendermint/tendermint/libs/log" e2e "github.com/tendermint/tendermint/test/e2e/pkg" + "github.com/tendermint/tendermint/test/e2e/pkg/infra" + "github.com/tendermint/tendermint/test/e2e/pkg/infra/docker" ) const randomSeed = 2308084734268 @@ -33,6 +35,7 @@ func main() { type CLI struct { root *cobra.Command testnet *e2e.Testnet + infra infra.TestnetInfra preserve bool } @@ -53,12 +56,23 @@ func NewCLI(logger log.Logger) *CLI { if err != nil { return err } + providerID, err := cmd.Flags().GetString("provider") + if err != nil { + return err + } + switch providerID { + case "docker": + cli.infra = docker.NewTestnetInfra(logger, testnet) + logger.Info("Using Docker-based infrastructure provider") + default: + return fmt.Errorf("unrecognized infrastructure provider ID: %s", providerID) + } cli.testnet = testnet return nil }, RunE: func(cmd *cobra.Command, args []string) (err error) { - if err = Cleanup(logger, cli.testnet); err != nil { + if err = Cleanup(cmd.Context(), logger, cli.testnet.Dir, cli.infra); err != nil { return err } defer func() { @@ -67,11 +81,11 @@ func NewCLI(logger log.Logger) *CLI { } else if err != nil { logger.Info("Preserving testnet that encountered error", "err", err) - } else if err := Cleanup(logger, cli.testnet); err != nil { + } else if err := Cleanup(cmd.Context(), logger, cli.testnet.Dir, cli.infra); err != nil { logger.Error("error cleaning up testnet contents", "err", err) } }() - if err = Setup(logger, cli.testnet); err != nil { + if err = Setup(cmd.Context(), logger, cli.testnet, cli.infra); err != nil { return err } @@ -87,7 +101,7 @@ func NewCLI(logger log.Logger) *CLI { chLoadResult <- Load(lctx, logger, r, cli.testnet) }() startAt := time.Now() - if err = Start(ctx, logger, cli.testnet); err != nil { + if err = Start(ctx, logger, cli.testnet, cli.infra); err != nil { return err } @@ -96,7 +110,7 @@ func NewCLI(logger log.Logger) *CLI { } if cli.testnet.HasPerturbations() { - if err = Perturb(ctx, logger, cli.testnet); err != nil { + if err = Perturb(ctx, logger, cli.testnet, cli.infra); err != nil { return err } if err = Wait(ctx, logger, cli.testnet, 5); err != nil { // allow some txs to go through @@ -134,7 +148,7 @@ func NewCLI(logger log.Logger) *CLI { if err = Wait(ctx, logger, cli.testnet, 5); err != nil { // wait for network to settle before tests return err } - if err := Test(cli.testnet); err != nil { + if err := Test(ctx, cli.testnet); err != nil { return err } return nil @@ -144,6 +158,8 @@ func NewCLI(logger log.Logger) *CLI { cli.root.PersistentFlags().StringP("file", "f", "", "Testnet TOML manifest") _ = cli.root.MarkPersistentFlagRequired("file") + cli.root.PersistentFlags().String("provider", "docker", "Which infrastructure provider to use") + cli.root.Flags().BoolVarP(&cli.preserve, "preserve", "p", false, "Preserves the running of the test net after tests are completed") @@ -156,7 +172,7 @@ func NewCLI(logger log.Logger) *CLI { Use: "setup", Short: "Generates the testnet directory and configuration", RunE: func(cmd *cobra.Command, args []string) error { - return Setup(logger, cli.testnet) + return Setup(cmd.Context(), logger, cli.testnet, cli.infra) }, }) @@ -166,12 +182,12 @@ func NewCLI(logger log.Logger) *CLI { RunE: func(cmd *cobra.Command, args []string) error { _, err := os.Stat(cli.testnet.Dir) if os.IsNotExist(err) { - err = Setup(logger, cli.testnet) + err = Setup(cmd.Context(), logger, cli.testnet, cli.infra) } if err != nil { return err } - return Start(cmd.Context(), logger, cli.testnet) + return Start(cmd.Context(), logger, cli.testnet, cli.infra) }, }) @@ -179,7 +195,7 @@ func NewCLI(logger log.Logger) *CLI { Use: "perturb", Short: "Perturbs the Docker testnet, e.g. by restarting or disconnecting nodes", RunE: func(cmd *cobra.Command, args []string) error { - return Perturb(cmd.Context(), logger, cli.testnet) + return Perturb(cmd.Context(), logger, cli.testnet, cli.infra) }, }) @@ -196,7 +212,7 @@ func NewCLI(logger log.Logger) *CLI { Short: "Stops the Docker testnet", RunE: func(cmd *cobra.Command, args []string) error { logger.Info("Stopping testnet") - return execCompose(cli.testnet.Dir, "down") + return cli.infra.Stop(cmd.Context()) }, }) @@ -205,7 +221,7 @@ func NewCLI(logger log.Logger) *CLI { Short: "Pauses the Docker testnet", RunE: func(cmd *cobra.Command, args []string) error { logger.Info("Pausing testnet") - return execCompose(cli.testnet.Dir, "pause") + return cli.infra.Pause(cmd.Context()) }, }) @@ -214,7 +230,7 @@ func NewCLI(logger log.Logger) *CLI { Short: "Resumes the Docker testnet", RunE: func(cmd *cobra.Command, args []string) error { logger.Info("Resuming testnet") - return execCompose(cli.testnet.Dir, "unpause") + return cli.infra.Unpause(cmd.Context()) }, }) @@ -259,7 +275,7 @@ func NewCLI(logger log.Logger) *CLI { Use: "test", Short: "Runs test cases against a running testnet", RunE: func(cmd *cobra.Command, args []string) error { - return Test(cli.testnet) + return Test(cmd.Context(), cli.testnet) }, }) @@ -267,17 +283,24 @@ func NewCLI(logger log.Logger) *CLI { Use: "cleanup", Short: "Removes the testnet directory", RunE: func(cmd *cobra.Command, args []string) error { - return Cleanup(logger, cli.testnet) + return Cleanup(cmd.Context(), logger, cli.testnet.Dir, cli.infra) }, }) cli.root.AddCommand(&cobra.Command{ Use: "logs [node]", - Short: "Shows the testnet or a specefic node's logs", + Short: "Shows the testnet or a specific node's logs", Example: "runner logs validator03", Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - return execComposeVerbose(cli.testnet.Dir, append([]string{"logs", "--no-color"}, args...)...) + if len(args) > 0 { + node := cli.testnet.LookupNode(args[0]) + if node == nil { + return fmt.Errorf("no such node: %s", args[0]) + } + return cli.infra.ShowNodeLogs(cmd.Context(), node) + } + return cli.infra.ShowLogs(cmd.Context()) }, }) @@ -287,9 +310,13 @@ func NewCLI(logger log.Logger) *CLI { Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { if len(args) == 1 { - return execComposeVerbose(cli.testnet.Dir, "logs", "--follow", args[0]) + node := cli.testnet.LookupNode(args[0]) + if node == nil { + return fmt.Errorf("no such node: %s", args[0]) + } + return cli.infra.TailNodeLogs(cmd.Context(), node) } - return execComposeVerbose(cli.testnet.Dir, "logs", "--follow") + return cli.infra.TailLogs(cmd.Context()) }, }) @@ -302,20 +329,20 @@ func NewCLI(logger log.Logger) *CLI { Min Block Interval Max Block Interval over a 100 block sampling period. - + Does not run any perbutations. `, RunE: func(cmd *cobra.Command, args []string) error { - if err := Cleanup(logger, cli.testnet); err != nil { + if err := Cleanup(cmd.Context(), logger, cli.testnet.Dir, cli.infra); err != nil { return err } defer func() { - if err := Cleanup(logger, cli.testnet); err != nil { + if err := Cleanup(cmd.Context(), logger, cli.testnet.Dir, cli.infra); err != nil { logger.Error("error cleaning up testnet contents", "err", err) } }() - if err := Setup(logger, cli.testnet); err != nil { + if err := Setup(cmd.Context(), logger, cli.testnet, cli.infra); err != nil { return err } @@ -331,7 +358,7 @@ Does not run any perbutations. chLoadResult <- Load(lctx, logger, r, cli.testnet) }() - if err := Start(ctx, logger, cli.testnet); err != nil { + if err := Start(ctx, logger, cli.testnet, cli.infra); err != nil { return err } diff --git a/test/e2e/runner/perturb.go b/test/e2e/runner/perturb.go index acabf7f34..76a209ea2 100644 --- a/test/e2e/runner/perturb.go +++ b/test/e2e/runner/perturb.go @@ -8,10 +8,11 @@ import ( "github.com/tendermint/tendermint/libs/log" rpctypes "github.com/tendermint/tendermint/rpc/coretypes" e2e "github.com/tendermint/tendermint/test/e2e/pkg" + "github.com/tendermint/tendermint/test/e2e/pkg/infra" ) // Perturbs a running testnet. -func Perturb(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error { +func Perturb(ctx context.Context, logger log.Logger, testnet *e2e.Testnet, ti infra.TestnetInfra) error { timer := time.NewTimer(0) // first tick fires immediately; reset below defer timer.Stop() @@ -21,7 +22,7 @@ func Perturb(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error case <-ctx.Done(): return ctx.Err() case <-timer.C: - _, err := PerturbNode(ctx, logger, node, perturbation) + _, err := PerturbNode(ctx, logger, node, perturbation, ti) if err != nil { return err } @@ -36,46 +37,45 @@ func Perturb(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error // PerturbNode perturbs a node with a given perturbation, returning its status // after recovering. -func PerturbNode(ctx context.Context, logger log.Logger, node *e2e.Node, perturbation e2e.Perturbation) (*rpctypes.ResultStatus, error) { - testnet := node.Testnet +func PerturbNode(ctx context.Context, logger log.Logger, node *e2e.Node, perturbation e2e.Perturbation, ti infra.TestnetInfra) (*rpctypes.ResultStatus, error) { switch perturbation { case e2e.PerturbationDisconnect: logger.Info(fmt.Sprintf("Disconnecting node %v...", node.Name)) - if err := execDocker("network", "disconnect", testnet.Name+"_"+testnet.Name, node.Name); err != nil { + if err := ti.DisconnectNode(ctx, node); err != nil { return nil, err } time.Sleep(10 * time.Second) - if err := execDocker("network", "connect", testnet.Name+"_"+testnet.Name, node.Name); err != nil { + if err := ti.ConnectNode(ctx, node); err != nil { return nil, err } case e2e.PerturbationKill: logger.Info(fmt.Sprintf("Killing node %v...", node.Name)) - if err := execCompose(testnet.Dir, "kill", "-s", "SIGKILL", node.Name); err != nil { + if err := ti.KillNodeProcess(ctx, node); err != nil { return nil, err } time.Sleep(10 * time.Second) - if err := execCompose(testnet.Dir, "start", node.Name); err != nil { + if err := ti.StartNodeProcess(ctx, node); err != nil { return nil, err } case e2e.PerturbationPause: logger.Info(fmt.Sprintf("Pausing node %v...", node.Name)) - if err := execCompose(testnet.Dir, "pause", node.Name); err != nil { + if err := ti.PauseNodeProcess(ctx, node); err != nil { return nil, err } time.Sleep(10 * time.Second) - if err := execCompose(testnet.Dir, "unpause", node.Name); err != nil { + if err := ti.UnpauseNodeProcess(ctx, node); err != nil { return nil, err } case e2e.PerturbationRestart: logger.Info(fmt.Sprintf("Restarting node %v...", node.Name)) - if err := execCompose(testnet.Dir, "kill", "-s", "SIGTERM", node.Name); err != nil { + if err := ti.TerminateNodeProcess(ctx, node); err != nil { return nil, err } time.Sleep(10 * time.Second) - if err := execCompose(testnet.Dir, "start", node.Name); err != nil { + if err := ti.StartNodeProcess(ctx, node); err != nil { return nil, err } diff --git a/test/e2e/runner/setup.go b/test/e2e/runner/setup.go index 9e6e5cc18..24477c9e0 100644 --- a/test/e2e/runner/setup.go +++ b/test/e2e/runner/setup.go @@ -1,8 +1,8 @@ -// nolint: gosec package main import ( "bytes" + "context" "encoding/base64" "encoding/json" "errors" @@ -12,7 +12,6 @@ import ( "regexp" "sort" "strings" - "text/template" "time" "github.com/BurntSushi/toml" @@ -22,6 +21,7 @@ import ( "github.com/tendermint/tendermint/libs/log" "github.com/tendermint/tendermint/privval" e2e "github.com/tendermint/tendermint/test/e2e/pkg" + "github.com/tendermint/tendermint/test/e2e/pkg/infra" "github.com/tendermint/tendermint/types" ) @@ -39,7 +39,7 @@ const ( ) // Setup sets up the testnet configuration. -func Setup(logger log.Logger, testnet *e2e.Testnet) error { +func Setup(ctx context.Context, logger log.Logger, testnet *e2e.Testnet, ti infra.TestnetInfra) error { logger.Info(fmt.Sprintf("Generating testnet files in %q", testnet.Dir)) err := os.MkdirAll(testnet.Dir, os.ModePerm) @@ -47,15 +47,6 @@ func Setup(logger log.Logger, testnet *e2e.Testnet) error { return err } - compose, err := MakeDockerCompose(testnet) - if err != nil { - return err - } - err = os.WriteFile(filepath.Join(testnet.Dir, "docker-compose.yml"), compose, 0644) - if err != nil { - return err - } - genesis, err := MakeGenesis(testnet) if err != nil { return err @@ -92,6 +83,8 @@ func Setup(logger log.Logger, testnet *e2e.Testnet) error { if err != nil { return err } + // nolint: gosec + // G306: Expect WriteFile permissions to be 0600 or less err = os.WriteFile(filepath.Join(nodeDir, "config", "app.toml"), appCfg, 0644) if err != nil { return err @@ -131,70 +124,13 @@ func Setup(logger log.Logger, testnet *e2e.Testnet) error { } } + if err := ti.Setup(ctx); err != nil { + return err + } + return nil } -// MakeDockerCompose generates a Docker Compose config for a testnet. -func MakeDockerCompose(testnet *e2e.Testnet) ([]byte, error) { - // Must use version 2 Docker Compose format, to support IPv6. - tmpl, err := template.New("docker-compose").Funcs(template.FuncMap{ - "addUint32": func(x, y uint32) uint32 { - return x + y - }, - "isBuiltin": func(protocol e2e.Protocol, mode e2e.Mode) bool { - return mode == e2e.ModeLight || protocol == e2e.ProtocolBuiltin - }, - }).Parse(`version: '2.4' - -networks: - {{ .Name }}: - labels: - e2e: true - driver: bridge -{{- if .IPv6 }} - enable_ipv6: true -{{- end }} - ipam: - driver: default - config: - - subnet: {{ .IP }} - -services: -{{- range .Nodes }} - {{ .Name }}: - labels: - e2e: true - container_name: {{ .Name }} - image: tendermint/e2e-node -{{- if isBuiltin $.ABCIProtocol .Mode }} - entrypoint: /usr/bin/entrypoint-builtin -{{- else if .LogLevel }} - command: start --log-level {{ .LogLevel }} -{{- end }} - init: true - ports: - - 26656 - - {{ if .ProxyPort }}{{ addUint32 .ProxyPort 1000 }}:{{ end }}26660 - - {{ if .ProxyPort }}{{ .ProxyPort }}:{{ end }}26657 - - 6060 - volumes: - - ./{{ .Name }}:/tendermint - networks: - {{ $.Name }}: - ipv{{ if $.IPv6 }}6{{ else }}4{{ end}}_address: {{ .IP }} - -{{end}}`) - if err != nil { - return nil, err - } - var buf bytes.Buffer - err = tmpl.Execute(&buf, testnet) - if err != nil { - return nil, err - } - return buf.Bytes(), nil -} - // MakeGenesis generates a genesis document. func MakeGenesis(testnet *e2e.Testnet) (types.GenesisDoc, error) { genesis := types.GenesisDoc{ @@ -421,5 +357,7 @@ func UpdateConfigStateSync(node *e2e.Node, height int64, hash []byte) error { } bz = regexp.MustCompile(`(?m)^trust-height =.*`).ReplaceAll(bz, []byte(fmt.Sprintf(`trust-height = %v`, height))) bz = regexp.MustCompile(`(?m)^trust-hash =.*`).ReplaceAll(bz, []byte(fmt.Sprintf(`trust-hash = "%X"`, hash))) + // nolint: gosec + // G306: Expect WriteFile permissions to be 0600 or less return os.WriteFile(cfgPath, bz, 0644) } diff --git a/test/e2e/runner/start.go b/test/e2e/runner/start.go index be9661df3..5d5c2e7a9 100644 --- a/test/e2e/runner/start.go +++ b/test/e2e/runner/start.go @@ -8,9 +8,10 @@ import ( "github.com/tendermint/tendermint/libs/log" e2e "github.com/tendermint/tendermint/test/e2e/pkg" + "github.com/tendermint/tendermint/test/e2e/pkg/infra" ) -func Start(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error { +func Start(ctx context.Context, logger log.Logger, testnet *e2e.Testnet, ti infra.TestnetInfra) error { if len(testnet.Nodes) == 0 { return fmt.Errorf("no nodes in testnet") } @@ -44,7 +45,7 @@ func Start(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error { for len(nodeQueue) > 0 && nodeQueue[0].StartAt == 0 { node := nodeQueue[0] nodeQueue = nodeQueue[1:] - if err := execCompose(testnet.Dir, "up", "-d", node.Name); err != nil { + if err := ti.StartNode(ctx, node); err != nil { return err } @@ -58,7 +59,7 @@ func Start(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error { return err } node.HasStarted = true - logger.Info(fmt.Sprintf("Node %v up on http://127.0.0.1:%v", node.Name, node.ProxyPort)) + logger.Info(fmt.Sprintf("Node %v up on http://%v:%v", node.IP, node.Name, node.ProxyPort)) } networkHeight := testnet.InitialHeight @@ -106,7 +107,7 @@ func Start(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error { } } - if err := execCompose(testnet.Dir, "up", "-d", node.Name); err != nil { + if err := ti.StartNode(ctx, node); err != nil { return err } @@ -128,8 +129,8 @@ func Start(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error { } else { lastNodeHeight = status.SyncInfo.LatestBlockHeight } - logger.Info(fmt.Sprintf("Node %v up on http://127.0.0.1:%v at height %v", - node.Name, node.ProxyPort, lastNodeHeight)) + logger.Info(fmt.Sprintf("Node %v up on http://%v:%v at height %v", + node.IP, node.Name, node.ProxyPort, lastNodeHeight)) } return nil diff --git a/test/e2e/runner/test.go b/test/e2e/runner/test.go index da7a4a50f..7766d6c8d 100644 --- a/test/e2e/runner/test.go +++ b/test/e2e/runner/test.go @@ -1,17 +1,19 @@ package main import ( + "context" "os" e2e "github.com/tendermint/tendermint/test/e2e/pkg" + "github.com/tendermint/tendermint/test/e2e/pkg/exec" ) // Test runs test cases under tests/ -func Test(testnet *e2e.Testnet) error { +func Test(ctx context.Context, testnet *e2e.Testnet) error { err := os.Setenv("E2E_MANIFEST", testnet.File) if err != nil { return err } - return execVerbose("./build/tests", "-test.count=1") + return exec.CommandVerbose(ctx, "./build/tests", "-test.count=1") } From 60881f1d06cf69bfc1fb2a27c18d0e4061fc19d7 Mon Sep 17 00:00:00 2001 From: Sam Kleinman Date: Thu, 30 Jun 2022 09:01:02 -0400 Subject: [PATCH 46/61] p2p: stop mconn channel sends without timeout (#8906) --- internal/p2p/conn/connection.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/p2p/conn/connection.go b/internal/p2p/conn/connection.go index c8fc21188..8f8453e71 100644 --- a/internal/p2p/conn/connection.go +++ b/internal/p2p/conn/connection.go @@ -100,7 +100,8 @@ type MConnection struct { // used to ensure FlushStop and OnStop // are safe to call concurrently. - stopMtx sync.Mutex + stopMtx sync.Mutex + stopSignal <-chan struct{} cancel context.CancelFunc @@ -207,6 +208,7 @@ func (c *MConnection) OnStart(ctx context.Context) error { c.quitSendRoutine = make(chan struct{}) c.doneSendRoutine = make(chan struct{}) c.quitRecvRoutine = make(chan struct{}) + c.stopSignal = ctx.Done() c.setRecvLastMsgAt(time.Now()) go c.sendRoutine(ctx) go c.recvRoutine(ctx) @@ -681,6 +683,8 @@ func (ch *channel) sendBytes(bytes []byte) bool { return true case <-time.After(defaultSendTimeout): return false + case <-ch.conn.stopSignal: + return false } } From 5c26db733b0b09e35990e70645e093cab0a3ce45 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 30 Jun 2022 16:34:43 +0000 Subject: [PATCH 47/61] build(deps): Bump github.com/stretchr/testify from 1.7.5 to 1.8.0 (#8907) Bumps [github.com/stretchr/testify](https://github.com/stretchr/testify) from 1.7.5 to 1.8.0.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/stretchr/testify&package-manager=go_modules&previous-version=1.7.5&new-version=1.8.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index fde7586ad..bae573a4a 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/snikch/goodman v0.0.0-20171125024755-10e37e294daa github.com/spf13/cobra v1.5.0 github.com/spf13/viper v1.12.0 - github.com/stretchr/testify v1.7.5 + github.com/stretchr/testify v1.8.0 github.com/tendermint/tm-db v0.6.6 golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e golang.org/x/net v0.0.0-20220617184016-355a448f1bc9 diff --git a/go.sum b/go.sum index e055c300e..f7e52298e 100644 --- a/go.sum +++ b/go.sum @@ -1077,8 +1077,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= -github.com/stretchr/testify v1.7.5 h1:s5PTfem8p8EbKQOctVV53k6jCJt3UX4IEJzwh+C324Q= -github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/subosito/gotenv v1.3.0/go.mod h1:YzJjq/33h7nrwdY+iHMhEOEEbW0ovIz0tB6t6PwAXzs= github.com/subosito/gotenv v1.4.0 h1:yAzM1+SmVcz5R4tXGsNMu1jUl2aOJXoiWUCEwwnGrvs= From 47cb30fc1d647183d500b0ecc31248df5dfc678c Mon Sep 17 00:00:00 2001 From: William Banfield <4561443+williambanfield@users.noreply.github.com> Date: Thu, 30 Jun 2022 16:51:16 -0400 Subject: [PATCH 48/61] p2p: set outgoing connections to around 20% of total connections (#8913) --- config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.go b/config/config.go index 804c5fc87..230fed98c 100644 --- a/config/config.go +++ b/config/config.go @@ -671,7 +671,7 @@ func DefaultP2PConfig() *P2PConfig { ExternalAddress: "", UPNP: false, MaxConnections: 64, - MaxOutgoingConnections: 32, + MaxOutgoingConnections: 12, MaxIncomingConnectionAttempts: 100, FlushThrottleTimeout: 100 * time.Millisecond, // The MTU (Maximum Transmission Unit) for Ethernet is 1500 bytes. From 5274f80de437795c4782b52f73dcec76616eefe2 Mon Sep 17 00:00:00 2001 From: William Banfield <4561443+williambanfield@users.noreply.github.com> Date: Thu, 30 Jun 2022 17:48:10 -0400 Subject: [PATCH 49/61] p2p: fix flakey test due to disconnect cooldown (#8917) This test was made flakey by #8839. The cooldown period means that the node in the test will not try to reconnect as quickly as the test expects. This change makes the cooldown shorter in the test so that the node quickly reconnects. --- internal/p2p/p2ptest/network.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/internal/p2p/p2ptest/network.go b/internal/p2p/p2ptest/network.go index c040a08a8..813344915 100644 --- a/internal/p2p/p2ptest/network.go +++ b/internal/p2p/p2ptest/network.go @@ -252,12 +252,13 @@ func (n *Network) MakeNode(ctx context.Context, t *testing.T, opts NodeOptions) require.NotNil(t, ep, "transport not listening an endpoint") peerManager, err := p2p.NewPeerManager(nodeID, dbm.NewMemDB(), p2p.PeerManagerOptions{ - MinRetryTime: 10 * time.Millisecond, - MaxRetryTime: 100 * time.Millisecond, - RetryTimeJitter: time.Millisecond, - MaxPeers: opts.MaxPeers, - MaxConnected: opts.MaxConnected, - Metrics: p2p.NopMetrics(), + MinRetryTime: 10 * time.Millisecond, + DisconnectCooldownPeriod: 10 * time.Millisecond, + MaxRetryTime: 100 * time.Millisecond, + RetryTimeJitter: time.Millisecond, + MaxPeers: opts.MaxPeers, + MaxConnected: opts.MaxConnected, + Metrics: p2p.NopMetrics(), }) require.NoError(t, err) From 921530c352d64229d0e78f6e1e14fe186d00b0db Mon Sep 17 00:00:00 2001 From: William Banfield <4561443+williambanfield@users.noreply.github.com> Date: Thu, 30 Jun 2022 18:02:59 -0400 Subject: [PATCH 50/61] p2p: use correct context error (#8916) handshakeCtx is the internal context carrying the timeout. Its error should be used for the error return. --- internal/p2p/transport_mconn.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/p2p/transport_mconn.go b/internal/p2p/transport_mconn.go index 13a65b973..523fbf89f 100644 --- a/internal/p2p/transport_mconn.go +++ b/internal/p2p/transport_mconn.go @@ -315,7 +315,7 @@ func (c *mConnConnection) Handshake( select { case <-handshakeCtx.Done(): _ = c.Close() - return types.NodeInfo{}, nil, ctx.Err() + return types.NodeInfo{}, nil, handshakeCtx.Err() case err := <-errCh: if err != nil { From 1d96faa35a216553fcc8d1be7d85a90450ca5be0 Mon Sep 17 00:00:00 2001 From: Ian Jungyong Um <31336310+code0xff@users.noreply.github.com> Date: Fri, 1 Jul 2022 21:44:39 +0900 Subject: [PATCH 51/61] p2p: fix typo (#8922) Fix minor typos in peermanager --- internal/p2p/peermanager.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/p2p/peermanager.go b/internal/p2p/peermanager.go index ac2e28225..8230b262e 100644 --- a/internal/p2p/peermanager.go +++ b/internal/p2p/peermanager.go @@ -221,11 +221,11 @@ func (o *PeerManagerOptions) Validate() error { return nil } -// isPersistentPeer checks if a peer is in PersistentPeers. It will panic +// isPersistent checks if a peer is in PersistentPeers. It will panic // if called before optimize(). func (o *PeerManagerOptions) isPersistent(id types.NodeID) bool { if o.persistentPeers == nil { - panic("isPersistentPeer() called before optimize()") + panic("isPersistent() called before optimize()") } return o.persistentPeers[id] } From 6d8079559baaf7428e16e504134ceb4e1e9d37db Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 1 Jul 2022 14:17:45 +0000 Subject: [PATCH 52/61] build(deps): Bump github.com/vektra/mockery/v2 from 2.13.1 to 2.14.0 (#8924) Bumps [github.com/vektra/mockery/v2](https://github.com/vektra/mockery) from 2.13.1 to 2.14.0.
Release notes

Sourced from github.com/vektra/mockery/v2's releases.

v2.14.0

Changelog

  • 8582bd6 Add test for getLocalizedPath
  • 686b90c Apply PR comments
  • de0cade Merge pull request #474 from RSid/add-flag-documentation
  • 1fa7d2f Merge pull request #480 from vektra/LandonTClipp-patch-1
  • 4d1f925 Merge pull request #484 from LouisBrunner/fix_generics_with_expecter
  • 519a84f Merge pull request #486 from abhinavnair/replace-ioutil
  • 2ca0b83 Merge pull request #488 from vektra/getLocalizedPath_test
  • cc82d49 Replace deprecated ioutil pkg with os & io
  • a420307 Update README.md
  • 4e4a96b Update issue template
  • fa182fe add documentation to readme
  • e4954a2 fix: add support for with-expecter when using generics
  • ca9ddd4 update issue template
Commits
  • 4d1f925 Merge pull request #484 from LouisBrunner/fix_generics_with_expecter
  • de0cade Merge pull request #474 from RSid/add-flag-documentation
  • 2ca0b83 Merge pull request #488 from vektra/getLocalizedPath_test
  • 8582bd6 Add test for getLocalizedPath
  • 519a84f Merge pull request #486 from abhinavnair/replace-ioutil
  • cc82d49 Replace deprecated ioutil pkg with os & io
  • e4954a2 fix: add support for with-expecter when using generics
  • ca9ddd4 update issue template
  • 686b90c Apply PR comments
  • 1fa7d2f Merge pull request #480 from vektra/LandonTClipp-patch-1
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/vektra/mockery/v2&package-manager=go_modules&previous-version=2.13.1&new-version=2.14.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index bae573a4a..691efb09f 100644 --- a/go.mod +++ b/go.mod @@ -42,7 +42,7 @@ require ( github.com/creachadair/taskgroup v0.3.2 github.com/golangci/golangci-lint v1.46.0 github.com/google/go-cmp v0.5.8 - github.com/vektra/mockery/v2 v2.13.1 + github.com/vektra/mockery/v2 v2.14.0 gotest.tools v2.2.0+incompatible ) diff --git a/go.sum b/go.sum index f7e52298e..ca88c3e8f 100644 --- a/go.sum +++ b/go.sum @@ -1126,8 +1126,8 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC github.com/valyala/fasthttp v1.30.0/go.mod h1:2rsYD01CKFrjjsvFxx75KlEUNpWNBY9JWD3K/7o2Cus= github.com/valyala/quicktemplate v1.7.0/go.mod h1:sqKJnoaOF88V07vkO+9FL8fb9uZg/VPSJnLYn+LmLk8= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= -github.com/vektra/mockery/v2 v2.13.1 h1:Lqs7aZiC7TwZO76fJ/4Zsb3NaO4F7cuuz0mZLYeNwtQ= -github.com/vektra/mockery/v2 v2.13.1/go.mod h1:bnD1T8tExSgPD1ripLkDbr60JA9VtQeu12P3wgLZd7M= +github.com/vektra/mockery/v2 v2.14.0 h1:KZ1p5Hrn8tiY+LErRMr14HHle6khxo+JKOXLBW/yfqs= +github.com/vektra/mockery/v2 v2.14.0/go.mod h1:bnD1T8tExSgPD1ripLkDbr60JA9VtQeu12P3wgLZd7M= github.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8/go.mod h1:dniwbG03GafCjFohMDmz6Zc6oCuiqgH6tGNyXTkHzXE= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= From 331860c2a8c406fa89575165322b963925fb38c5 Mon Sep 17 00:00:00 2001 From: Sergio Mena Date: Mon, 4 Jul 2022 16:25:48 +0200 Subject: [PATCH 53/61] Restore `Commit` to the ABCI++ spec, and other late modifications (#8796) * Added VoteExtensionsEnableHeight * Fix reference to `modified` * Removed old pseudo-code, now included in spec * Removed markdown warnings in abci++_basic_concepts_002_draft.md * Restored `Commit` in the Methods section * Addressed remaining markdown warnings * Revisited intro and basic concepts section * Extra pass at all spec sections to recover Commit, and other ABCI++ spec modifications * Fixed links * make proto-gen * Remove _primes_ from spec notation * Update proto/tendermint/abci/types.proto Co-authored-by: Callum Waters * Update spec/abci++/abci++_tmint_expected_behavior_002_draft.md Co-authored-by: Callum Waters * Addressed @cmwaters' comments * Addressed @angbrav's and @mpoke's comments on spec * make proto-gen * Fix MD anchor reference * Clarify throughout the spec when `ProcessProposal` and `VerifyVoteExtension` are called * Update spec/abci++/abci++_app_requirements_002_draft.md Co-authored-by: M. J. Fromberger * Update spec/abci++/abci++_app_requirements_002_draft.md Co-authored-by: M. J. Fromberger * Update spec/abci++/abci++_app_requirements_002_draft.md Co-authored-by: William Banfield <4561443+williambanfield@users.noreply.github.com> * Update spec/abci++/abci++_basic_concepts_002_draft.md Co-authored-by: William Banfield <4561443+williambanfield@users.noreply.github.com> * Update spec/abci++/abci++_basic_concepts_002_draft.md Co-authored-by: M. J. Fromberger * Update spec/abci++/abci++_basic_concepts_002_draft.md Co-authored-by: William Banfield <4561443+williambanfield@users.noreply.github.com> * Update spec/abci++/abci++_methods_002_draft.md Co-authored-by: M. J. Fromberger * Update spec/abci++/abci++_tmint_expected_behavior_002_draft.md Co-authored-by: William Banfield <4561443+williambanfield@users.noreply.github.com> * Addresed comments * Renamed 'draft' files * Adatped links to new filenames * Fixed links and minor cosmetic changes * Renamed 'byzantine_validators' to 'misbehavior' in ABCI++ spec and protobufs * make proto-gen * Renamed 'byzantine_validators' to 'misbehavior' in the code * Fixed link * Update spec/abci++/abci++_basic_concepts.md Co-authored-by: Daniel * Update spec/abci++/abci++_basic_concepts.md Co-authored-by: Daniel * Update spec/abci++/abci++_basic_concepts.md Co-authored-by: Daniel * Update spec/abci++/abci++_basic_concepts.md Co-authored-by: Daniel * Update spec/abci++/abci++_basic_concepts.md Co-authored-by: Daniel * Update spec/abci++/abci++_basic_concepts.md Co-authored-by: Daniel * Update spec/abci++/abci++_basic_concepts.md Co-authored-by: Daniel * Update spec/abci++/abci++_basic_concepts.md Co-authored-by: Daniel * Update spec/abci++/abci++_basic_concepts.md Co-authored-by: Daniel * Update spec/abci++/abci++_basic_concepts.md Co-authored-by: Daniel * Update spec/abci++/abci++_basic_concepts.md Co-authored-by: Daniel * Update spec/abci++/abci++_basic_concepts.md Co-authored-by: Daniel * Update spec/abci++/abci++_basic_concepts.md Co-authored-by: Daniel * Update spec/abci++/abci++_basic_concepts.md Co-authored-by: Daniel * Update spec/abci++/abci++_basic_concepts.md Co-authored-by: Daniel * Update spec/abci++/abci++_basic_concepts.md Co-authored-by: Daniel * Update spec/abci++/abci++_basic_concepts.md Co-authored-by: Daniel * Update spec/abci++/abci++_basic_concepts.md Co-authored-by: Daniel * Update spec/abci++/abci++_basic_concepts.md Co-authored-by: Daniel * Update spec/abci++/abci++_basic_concepts.md Co-authored-by: Daniel * Update spec/abci++/abci++_basic_concepts.md Co-authored-by: Daniel * Update spec/abci++/abci++_basic_concepts.md Co-authored-by: Daniel * Update spec/abci++/abci++_methods.md Co-authored-by: Daniel * Update spec/abci++/abci++_methods.md Co-authored-by: Daniel * Update spec/abci++/abci++_methods.md Co-authored-by: Daniel * Update spec/abci++/abci++_methods.md Co-authored-by: Daniel * Update spec/abci++/abci++_methods.md Co-authored-by: Daniel * Update spec/abci++/abci++_methods.md Co-authored-by: Daniel * Update spec/abci++/abci++_methods.md Co-authored-by: Daniel * Update spec/abci++/abci++_methods.md Co-authored-by: Daniel * Update spec/abci++/abci++_methods.md Co-authored-by: Daniel * Update spec/abci++/abci++_methods.md Co-authored-by: Daniel * Update spec/abci++/abci++_methods.md Co-authored-by: Daniel * Update spec/abci++/abci++_methods.md Co-authored-by: Daniel * Update spec/abci++/abci++_methods.md Co-authored-by: Daniel * Update spec/abci++/abci++_methods.md Co-authored-by: Daniel * Update spec/abci++/abci++_methods.md Co-authored-by: Daniel * Update spec/abci++/abci++_methods.md Co-authored-by: Daniel * Update spec/abci++/abci++_methods.md Co-authored-by: Daniel * Update spec/abci++/abci++_methods.md Co-authored-by: Daniel * Update spec/abci++/abci++_methods.md Co-authored-by: Daniel * Update spec/abci++/abci++_methods.md Co-authored-by: Daniel * Update spec/abci++/abci++_methods.md Co-authored-by: Daniel * Update spec/abci++/abci++_methods.md Co-authored-by: Daniel * Update spec/abci++/abci++_methods.md Co-authored-by: Daniel * Update spec/abci++/abci++_methods.md Co-authored-by: Daniel * Update spec/abci++/abci++_methods.md Co-authored-by: Daniel * Update spec/abci++/abci++_methods.md Co-authored-by: Daniel * Update spec/abci++/abci++_methods.md Co-authored-by: Daniel * Update spec/abci++/abci++_methods.md Co-authored-by: Daniel * Addressed @cason's comments * Clarified conditions for `ProcessProposal` call at proposer Co-authored-by: Callum Waters Co-authored-by: M. J. Fromberger Co-authored-by: William Banfield <4561443+williambanfield@users.noreply.github.com> Co-authored-by: Daniel --- UPGRADING.md | 4 +- abci/example/kvstore/kvstore.go | 2 +- abci/types/types.pb.go | 495 +++++++++--------- docs/rfc/rfc-013-abci++.md | 2 +- .../proposer-based-timestamps-runbook.md | 2 +- internal/state/execution.go | 60 +-- internal/state/execution_test.go | 16 +- internal/state/helpers_test.go | 8 +- proto/tendermint/abci/types.proto | 10 +- spec/abci++/README.md | 8 +- ...02_draft.md => abci++_app_requirements.md} | 89 ++-- spec/abci++/abci++_basic_concepts.md | 468 +++++++++++++++++ .../abci++/abci++_basic_concepts_002_draft.md | 404 -------------- ...r_002_draft.md => abci++_client_server.md} | 10 +- ...methods_002_draft.md => abci++_methods.md} | 476 ++++++++++------- ...t.md => abci++_tmint_expected_behavior.md} | 107 ++-- spec/abci++/v0.md | 156 ------ spec/abci++/v1.md | 162 ------ spec/abci++/v2.md | 180 ------- spec/abci++/v3.md | 201 ------- spec/abci++/v4.md | 199 ------- 21 files changed, 1164 insertions(+), 1895 deletions(-) rename spec/abci++/{abci++_app_requirements_002_draft.md => abci++_app_requirements.md} (93%) create mode 100644 spec/abci++/abci++_basic_concepts.md delete mode 100644 spec/abci++/abci++_basic_concepts_002_draft.md rename spec/abci++/{abci++_client_server_002_draft.md => abci++_client_server.md} (91%) rename spec/abci++/{abci++_methods_002_draft.md => abci++_methods.md} (71%) rename spec/abci++/{abci++_tmint_expected_behavior_002_draft.md => abci++_tmint_expected_behavior.md} (66%) delete mode 100644 spec/abci++/v0.md delete mode 100644 spec/abci++/v1.md delete mode 100644 spec/abci++/v2.md delete mode 100644 spec/abci++/v3.md delete mode 100644 spec/abci++/v4.md diff --git a/UPGRADING.md b/UPGRADING.md index 13582e75b..f9122a5de 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -17,9 +17,9 @@ by Tendermint itself. Right now, we return a regular error when this happens. #### ABCI++ For information on how ABCI++ works, see the -[Specification](https://github.com/tendermint/tendermint/blob/master/spec/abci%2B%2B/README.md). +[Specification](spec/abci%2B%2B/README.md). In particular, the simplest way to upgrade your application is described -[here](https://github.com/tendermint/tendermint/blob/master/spec/abci%2B%2B/abci++_tmint_expected_behavior_002_draft.md#adapting-existing-applications-that-use-abci). +[here](spec/abci%2B%2B/abci++_tmint_expected_behavior.md#adapting-existing-applications-that-use-abci). #### Moving the `app_hash` parameter diff --git a/abci/example/kvstore/kvstore.go b/abci/example/kvstore/kvstore.go index bbb2fbe34..c9a2a148c 100644 --- a/abci/example/kvstore/kvstore.go +++ b/abci/example/kvstore/kvstore.go @@ -175,7 +175,7 @@ func (app *Application) FinalizeBlock(_ context.Context, req *types.RequestFinal app.ValUpdates = make([]types.ValidatorUpdate, 0) // Punish validators who committed equivocation. - for _, ev := range req.ByzantineValidators { + for _, ev := range req.Misbehavior { if ev.Type == types.MisbehaviorType_DUPLICATE_VOTE { addr := string(ev.Validator.Address) if pubKey, ok := app.valAddrToPubKeyMap[addr]; ok { diff --git a/abci/types/types.pb.go b/abci/types/types.pb.go index 946cfa6af..9ce88f045 100644 --- a/abci/types/types.pb.go +++ b/abci/types/types.pb.go @@ -1120,12 +1120,12 @@ type RequestPrepareProposal struct { MaxTxBytes int64 `protobuf:"varint,1,opt,name=max_tx_bytes,json=maxTxBytes,proto3" json:"max_tx_bytes,omitempty"` // txs is an array of transactions that will be included in a block, // sent to the app for possible modifications. - Txs [][]byte `protobuf:"bytes,2,rep,name=txs,proto3" json:"txs,omitempty"` - LocalLastCommit ExtendedCommitInfo `protobuf:"bytes,3,opt,name=local_last_commit,json=localLastCommit,proto3" json:"local_last_commit"` - ByzantineValidators []Misbehavior `protobuf:"bytes,4,rep,name=byzantine_validators,json=byzantineValidators,proto3" json:"byzantine_validators"` - Height int64 `protobuf:"varint,5,opt,name=height,proto3" json:"height,omitempty"` - Time time.Time `protobuf:"bytes,6,opt,name=time,proto3,stdtime" json:"time"` - NextValidatorsHash []byte `protobuf:"bytes,7,opt,name=next_validators_hash,json=nextValidatorsHash,proto3" json:"next_validators_hash,omitempty"` + Txs [][]byte `protobuf:"bytes,2,rep,name=txs,proto3" json:"txs,omitempty"` + LocalLastCommit ExtendedCommitInfo `protobuf:"bytes,3,opt,name=local_last_commit,json=localLastCommit,proto3" json:"local_last_commit"` + Misbehavior []Misbehavior `protobuf:"bytes,4,rep,name=misbehavior,proto3" json:"misbehavior"` + Height int64 `protobuf:"varint,5,opt,name=height,proto3" json:"height,omitempty"` + Time time.Time `protobuf:"bytes,6,opt,name=time,proto3,stdtime" json:"time"` + NextValidatorsHash []byte `protobuf:"bytes,7,opt,name=next_validators_hash,json=nextValidatorsHash,proto3" json:"next_validators_hash,omitempty"` // address of the public key of the validator proposing the block. ProposerAddress []byte `protobuf:"bytes,8,opt,name=proposer_address,json=proposerAddress,proto3" json:"proposer_address,omitempty"` } @@ -1184,9 +1184,9 @@ func (m *RequestPrepareProposal) GetLocalLastCommit() ExtendedCommitInfo { return ExtendedCommitInfo{} } -func (m *RequestPrepareProposal) GetByzantineValidators() []Misbehavior { +func (m *RequestPrepareProposal) GetMisbehavior() []Misbehavior { if m != nil { - return m.ByzantineValidators + return m.Misbehavior } return nil } @@ -1220,9 +1220,9 @@ func (m *RequestPrepareProposal) GetProposerAddress() []byte { } type RequestProcessProposal struct { - Txs [][]byte `protobuf:"bytes,1,rep,name=txs,proto3" json:"txs,omitempty"` - ProposedLastCommit CommitInfo `protobuf:"bytes,2,opt,name=proposed_last_commit,json=proposedLastCommit,proto3" json:"proposed_last_commit"` - ByzantineValidators []Misbehavior `protobuf:"bytes,3,rep,name=byzantine_validators,json=byzantineValidators,proto3" json:"byzantine_validators"` + Txs [][]byte `protobuf:"bytes,1,rep,name=txs,proto3" json:"txs,omitempty"` + ProposedLastCommit CommitInfo `protobuf:"bytes,2,opt,name=proposed_last_commit,json=proposedLastCommit,proto3" json:"proposed_last_commit"` + Misbehavior []Misbehavior `protobuf:"bytes,3,rep,name=misbehavior,proto3" json:"misbehavior"` // hash is the merkle root hash of the fields of the proposed block. Hash []byte `protobuf:"bytes,4,opt,name=hash,proto3" json:"hash,omitempty"` Height int64 `protobuf:"varint,5,opt,name=height,proto3" json:"height,omitempty"` @@ -1279,9 +1279,9 @@ func (m *RequestProcessProposal) GetProposedLastCommit() CommitInfo { return CommitInfo{} } -func (m *RequestProcessProposal) GetByzantineValidators() []Misbehavior { +func (m *RequestProcessProposal) GetMisbehavior() []Misbehavior { if m != nil { - return m.ByzantineValidators + return m.Misbehavior } return nil } @@ -1444,10 +1444,10 @@ func (m *RequestVerifyVoteExtension) GetVoteExtension() []byte { } type RequestFinalizeBlock struct { - Txs [][]byte `protobuf:"bytes,1,rep,name=txs,proto3" json:"txs,omitempty"` - DecidedLastCommit CommitInfo `protobuf:"bytes,2,opt,name=decided_last_commit,json=decidedLastCommit,proto3" json:"decided_last_commit"` - ByzantineValidators []Misbehavior `protobuf:"bytes,3,rep,name=byzantine_validators,json=byzantineValidators,proto3" json:"byzantine_validators"` - // hash is the merkle root hash of the fields of the proposed block. + Txs [][]byte `protobuf:"bytes,1,rep,name=txs,proto3" json:"txs,omitempty"` + DecidedLastCommit CommitInfo `protobuf:"bytes,2,opt,name=decided_last_commit,json=decidedLastCommit,proto3" json:"decided_last_commit"` + Misbehavior []Misbehavior `protobuf:"bytes,3,rep,name=misbehavior,proto3" json:"misbehavior"` + // hash is the merkle root hash of the fields of the decided block. Hash []byte `protobuf:"bytes,4,opt,name=hash,proto3" json:"hash,omitempty"` Height int64 `protobuf:"varint,5,opt,name=height,proto3" json:"height,omitempty"` Time time.Time `protobuf:"bytes,6,opt,name=time,proto3,stdtime" json:"time"` @@ -1503,9 +1503,9 @@ func (m *RequestFinalizeBlock) GetDecidedLastCommit() CommitInfo { return CommitInfo{} } -func (m *RequestFinalizeBlock) GetByzantineValidators() []Misbehavior { +func (m *RequestFinalizeBlock) GetMisbehavior() []Misbehavior { if m != nil { - return m.ByzantineValidators + return m.Misbehavior } return nil } @@ -2381,7 +2381,6 @@ func (m *ResponseDeliverTx) GetCodespace() string { } type ResponseCommit struct { - // reserve 1 RetainHeight int64 `protobuf:"varint,3,opt,name=retain_height,json=retainHeight,proto3" json:"retain_height,omitempty"` } @@ -3829,211 +3828,211 @@ func init() { func init() { proto.RegisterFile("tendermint/abci/types.proto", fileDescriptor_252557cfdd89a31a) } var fileDescriptor_252557cfdd89a31a = []byte{ - // 3253 bytes of a gzipped FileDescriptorProto + // 3257 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x5a, 0xcd, 0x73, 0x23, 0xd5, - 0x11, 0xd7, 0xe8, 0x5b, 0xad, 0xaf, 0xf1, 0xb3, 0x59, 0xb4, 0x62, 0xd7, 0x36, 0x43, 0x01, 0xcb, - 0x02, 0x36, 0xf1, 0x66, 0x61, 0xc9, 0x42, 0x28, 0x5b, 0xd6, 0x46, 0xf6, 0x7a, 0x6d, 0x33, 0x96, - 0x4d, 0x91, 0x0f, 0x86, 0xb1, 0xf4, 0x6c, 0x0d, 0x2b, 0x69, 0x86, 0x99, 0x91, 0x91, 0x39, 0x26, - 0xc5, 0x85, 0x43, 0x8a, 0x4b, 0x2a, 0x49, 0x55, 0xb8, 0x25, 0x55, 0xc9, 0x7f, 0x90, 0x5c, 0x72, - 0xca, 0x81, 0x43, 0x0e, 0x9c, 0x52, 0x39, 0x91, 0x14, 0xdc, 0xf8, 0x07, 0x72, 0x4b, 0xa5, 0xde, - 0xc7, 0x7c, 0x49, 0x33, 0xfa, 0x00, 0x8a, 0xaa, 0x54, 0x71, 0x9b, 0xd7, 0xd3, 0xdd, 0xef, 0xab, - 0x5f, 0x77, 0xff, 0xfa, 0x3d, 0x78, 0xcc, 0xc6, 0xfd, 0x36, 0x36, 0x7b, 0x5a, 0xdf, 0x5e, 0x57, - 0x4f, 0x5b, 0xda, 0xba, 0x7d, 0x69, 0x60, 0x6b, 0xcd, 0x30, 0x75, 0x5b, 0x47, 0x65, 0xef, 0xe7, - 0x1a, 0xf9, 0x59, 0xbd, 0xee, 0xe3, 0x6e, 0x99, 0x97, 0x86, 0xad, 0xaf, 0x1b, 0xa6, 0xae, 0x9f, - 0x31, 0xfe, 0xea, 0xb5, 0xf1, 0xdf, 0x0f, 0xf1, 0x25, 0xd7, 0x16, 0x10, 0xa6, 0xbd, 0xac, 0x1b, - 0xaa, 0xa9, 0xf6, 0x9c, 0xdf, 0x2b, 0xe7, 0xba, 0x7e, 0xde, 0xc5, 0xeb, 0xb4, 0x75, 0x3a, 0x38, - 0x5b, 0xb7, 0xb5, 0x1e, 0xb6, 0x6c, 0xb5, 0x67, 0x70, 0x86, 0xa5, 0x73, 0xfd, 0x5c, 0xa7, 0x9f, - 0xeb, 0xe4, 0x8b, 0x51, 0xa5, 0xbf, 0xe4, 0x20, 0x23, 0xe3, 0x77, 0x07, 0xd8, 0xb2, 0xd1, 0x06, - 0x24, 0x71, 0xab, 0xa3, 0x57, 0x84, 0x55, 0xe1, 0x46, 0x7e, 0xe3, 0xda, 0xda, 0xc8, 0xf0, 0xd7, - 0x38, 0x5f, 0xbd, 0xd5, 0xd1, 0x1b, 0x31, 0x99, 0xf2, 0xa2, 0xdb, 0x90, 0x3a, 0xeb, 0x0e, 0xac, - 0x4e, 0x25, 0x4e, 0x85, 0xae, 0x47, 0x09, 0xdd, 0x23, 0x4c, 0x8d, 0x98, 0xcc, 0xb8, 0x49, 0x57, - 0x5a, 0xff, 0x4c, 0xaf, 0x24, 0x26, 0x77, 0xb5, 0xd3, 0x3f, 0xa3, 0x5d, 0x11, 0x5e, 0xb4, 0x05, - 0xa0, 0xf5, 0x35, 0x5b, 0x69, 0x75, 0x54, 0xad, 0x5f, 0x49, 0x52, 0xc9, 0xc7, 0xa3, 0x25, 0x35, - 0xbb, 0x46, 0x18, 0x1b, 0x31, 0x39, 0xa7, 0x39, 0x0d, 0x32, 0xdc, 0x77, 0x07, 0xd8, 0xbc, 0xac, - 0xa4, 0x26, 0x0f, 0xf7, 0x75, 0xc2, 0x44, 0x86, 0x4b, 0xb9, 0xd1, 0x2b, 0x90, 0x6d, 0x75, 0x70, - 0xeb, 0xa1, 0x62, 0x0f, 0x2b, 0x19, 0x2a, 0xb9, 0x12, 0x25, 0x59, 0x23, 0x7c, 0xcd, 0x61, 0x23, - 0x26, 0x67, 0x5a, 0xec, 0x13, 0xdd, 0x81, 0x74, 0x4b, 0xef, 0xf5, 0x34, 0xbb, 0x02, 0x54, 0x76, - 0x39, 0x52, 0x96, 0x72, 0x35, 0x62, 0x32, 0xe7, 0x47, 0xfb, 0x50, 0xea, 0x6a, 0x96, 0xad, 0x58, - 0x7d, 0xd5, 0xb0, 0x3a, 0xba, 0x6d, 0x55, 0xf2, 0x54, 0xc3, 0x93, 0x51, 0x1a, 0xf6, 0x34, 0xcb, - 0x3e, 0x72, 0x98, 0x1b, 0x31, 0xb9, 0xd8, 0xf5, 0x13, 0x88, 0x3e, 0xfd, 0xec, 0x0c, 0x9b, 0xae, - 0xc2, 0x4a, 0x61, 0xb2, 0xbe, 0x03, 0xc2, 0xed, 0xc8, 0x13, 0x7d, 0xba, 0x9f, 0x80, 0x7e, 0x02, - 0x8b, 0x5d, 0x5d, 0x6d, 0xbb, 0xea, 0x94, 0x56, 0x67, 0xd0, 0x7f, 0x58, 0x29, 0x52, 0xa5, 0xcf, - 0x44, 0x0e, 0x52, 0x57, 0xdb, 0x8e, 0x8a, 0x1a, 0x11, 0x68, 0xc4, 0xe4, 0x85, 0xee, 0x28, 0x11, - 0xbd, 0x05, 0x4b, 0xaa, 0x61, 0x74, 0x2f, 0x47, 0xb5, 0x97, 0xa8, 0xf6, 0x9b, 0x51, 0xda, 0x37, - 0x89, 0xcc, 0xa8, 0x7a, 0xa4, 0x8e, 0x51, 0x51, 0x13, 0x44, 0xc3, 0xc4, 0x86, 0x6a, 0x62, 0xc5, - 0x30, 0x75, 0x43, 0xb7, 0xd4, 0x6e, 0xa5, 0x4c, 0x75, 0x3f, 0x1d, 0xa5, 0xfb, 0x90, 0xf1, 0x1f, - 0x72, 0xf6, 0x46, 0x4c, 0x2e, 0x1b, 0x41, 0x12, 0xd3, 0xaa, 0xb7, 0xb0, 0x65, 0x79, 0x5a, 0xc5, - 0x69, 0x5a, 0x29, 0x7f, 0x50, 0x6b, 0x80, 0x84, 0xea, 0x90, 0xc7, 0x43, 0x22, 0xae, 0x5c, 0xe8, - 0x36, 0xae, 0x2c, 0x50, 0x85, 0x52, 0xe4, 0x09, 0xa5, 0xac, 0x27, 0xba, 0x8d, 0x1b, 0x31, 0x19, - 0xb0, 0xdb, 0x42, 0x2a, 0x3c, 0x72, 0x81, 0x4d, 0xed, 0xec, 0x92, 0xaa, 0x51, 0xe8, 0x1f, 0x4b, - 0xd3, 0xfb, 0x15, 0x44, 0x15, 0x3e, 0x1b, 0xa5, 0xf0, 0x84, 0x0a, 0x11, 0x15, 0x75, 0x47, 0xa4, - 0x11, 0x93, 0x17, 0x2f, 0xc6, 0xc9, 0xc4, 0xc4, 0xce, 0xb4, 0xbe, 0xda, 0xd5, 0xde, 0xc7, 0xca, - 0x69, 0x57, 0x6f, 0x3d, 0xac, 0x2c, 0x4e, 0x36, 0xb1, 0x7b, 0x9c, 0x7b, 0x8b, 0x30, 0x13, 0x13, - 0x3b, 0xf3, 0x13, 0xb6, 0x32, 0x90, 0xba, 0x50, 0xbb, 0x03, 0xbc, 0x9b, 0xcc, 0xa6, 0xc5, 0xcc, - 0x6e, 0x32, 0x9b, 0x15, 0x73, 0xbb, 0xc9, 0x6c, 0x4e, 0x04, 0xe9, 0x69, 0xc8, 0xfb, 0x5c, 0x12, - 0xaa, 0x40, 0xa6, 0x87, 0x2d, 0x4b, 0x3d, 0xc7, 0xd4, 0x83, 0xe5, 0x64, 0xa7, 0x29, 0x95, 0xa0, - 0xe0, 0x77, 0x43, 0xd2, 0x47, 0x82, 0x2b, 0x49, 0x3c, 0x0c, 0x91, 0xbc, 0xc0, 0x26, 0x5d, 0x08, - 0x2e, 0xc9, 0x9b, 0xe8, 0x09, 0x28, 0xd2, 0x49, 0x28, 0xce, 0x7f, 0xe2, 0xe6, 0x92, 0x72, 0x81, - 0x12, 0x4f, 0x38, 0xd3, 0x0a, 0xe4, 0x8d, 0x0d, 0xc3, 0x65, 0x49, 0x50, 0x16, 0x30, 0x36, 0x0c, - 0x87, 0xe1, 0x71, 0x28, 0x90, 0x19, 0xbb, 0x1c, 0x49, 0xda, 0x49, 0x9e, 0xd0, 0x38, 0x8b, 0xf4, - 0xf7, 0x38, 0x88, 0xa3, 0xae, 0x0b, 0xdd, 0x81, 0x24, 0xf1, 0xe2, 0xdc, 0x21, 0x57, 0xd7, 0x98, - 0x8b, 0x5f, 0x73, 0x5c, 0xfc, 0x5a, 0xd3, 0x71, 0xf1, 0x5b, 0xd9, 0x4f, 0x3e, 0x5b, 0x89, 0x7d, - 0xf4, 0xaf, 0x15, 0x41, 0xa6, 0x12, 0xe8, 0x2a, 0x71, 0x58, 0xaa, 0xd6, 0x57, 0xb4, 0x36, 0x1d, - 0x72, 0x8e, 0x78, 0x23, 0x55, 0xeb, 0xef, 0xb4, 0xd1, 0x1e, 0x88, 0x2d, 0xbd, 0x6f, 0xe1, 0xbe, - 0x35, 0xb0, 0x14, 0x16, 0x42, 0xb8, 0x1b, 0x0e, 0x38, 0x53, 0x16, 0xc8, 0x6a, 0x0e, 0xe7, 0x21, - 0x65, 0x94, 0xcb, 0xad, 0x20, 0x01, 0xdd, 0x03, 0xb8, 0x50, 0xbb, 0x5a, 0x5b, 0xb5, 0x75, 0xd3, - 0xaa, 0x24, 0x57, 0x13, 0x37, 0xf2, 0x1b, 0xab, 0x63, 0x5b, 0x7d, 0xe2, 0xb0, 0x1c, 0x1b, 0x6d, - 0xd5, 0xc6, 0x5b, 0x49, 0x32, 0x5c, 0xd9, 0x27, 0x89, 0x9e, 0x82, 0xb2, 0x6a, 0x18, 0x8a, 0x65, - 0xab, 0x36, 0x56, 0x4e, 0x2f, 0x6d, 0x6c, 0x51, 0x17, 0x5d, 0x90, 0x8b, 0xaa, 0x61, 0x1c, 0x11, - 0xea, 0x16, 0x21, 0xa2, 0x27, 0xa1, 0x44, 0xbc, 0xb9, 0xa6, 0x76, 0x95, 0x0e, 0xd6, 0xce, 0x3b, - 0x76, 0x25, 0xbd, 0x2a, 0xdc, 0x48, 0xc8, 0x45, 0x4e, 0x6d, 0x50, 0xa2, 0xd4, 0x76, 0x77, 0x9c, - 0x7a, 0x72, 0x84, 0x20, 0xd9, 0x56, 0x6d, 0x95, 0xae, 0x64, 0x41, 0xa6, 0xdf, 0x84, 0x66, 0xa8, - 0x76, 0x87, 0xaf, 0x0f, 0xfd, 0x46, 0x57, 0x20, 0xcd, 0xd5, 0x26, 0xa8, 0x5a, 0xde, 0x42, 0x4b, - 0x90, 0x32, 0x4c, 0xfd, 0x02, 0xd3, 0xad, 0xcb, 0xca, 0xac, 0x21, 0xc9, 0x50, 0x0a, 0x7a, 0x7d, - 0x54, 0x82, 0xb8, 0x3d, 0xe4, 0xbd, 0xc4, 0xed, 0x21, 0x7a, 0x01, 0x92, 0x64, 0x21, 0x69, 0x1f, - 0xa5, 0x90, 0x38, 0xc7, 0xe5, 0x9a, 0x97, 0x06, 0x96, 0x29, 0xa7, 0x54, 0x86, 0x62, 0x20, 0x1a, - 0x48, 0x57, 0x60, 0x29, 0xcc, 0xb9, 0x4b, 0x1d, 0x97, 0x1e, 0x70, 0xd2, 0xe8, 0x36, 0x64, 0x5d, - 0xef, 0xce, 0x0c, 0xe7, 0xea, 0x58, 0xb7, 0x0e, 0xb3, 0xec, 0xb2, 0x12, 0x8b, 0x21, 0x1b, 0xd0, - 0x51, 0x79, 0x2c, 0x2f, 0xc8, 0x19, 0xd5, 0x30, 0x1a, 0xaa, 0xd5, 0x91, 0xde, 0x86, 0x4a, 0x94, - 0xe7, 0xf6, 0x2d, 0x98, 0x40, 0xcd, 0xde, 0x59, 0xb0, 0x2b, 0x90, 0x3e, 0xd3, 0xcd, 0x9e, 0x6a, - 0x53, 0x65, 0x45, 0x99, 0xb7, 0xc8, 0x42, 0x32, 0x2f, 0x9e, 0xa0, 0x64, 0xd6, 0x90, 0x14, 0xb8, - 0x1a, 0xe9, 0xbd, 0x89, 0x88, 0xd6, 0x6f, 0x63, 0xb6, 0xac, 0x45, 0x99, 0x35, 0x3c, 0x45, 0x6c, - 0xb0, 0xac, 0x41, 0xba, 0xb5, 0xe8, 0x5c, 0xa9, 0xfe, 0x9c, 0xcc, 0x5b, 0xd2, 0x9f, 0x12, 0x70, - 0x25, 0xdc, 0x87, 0xa3, 0x55, 0x28, 0xf4, 0xd4, 0xa1, 0x62, 0x0f, 0xb9, 0xd9, 0x09, 0x74, 0xe3, - 0xa1, 0xa7, 0x0e, 0x9b, 0x43, 0x66, 0x73, 0x22, 0x24, 0xec, 0xa1, 0x55, 0x89, 0xaf, 0x26, 0x6e, - 0x14, 0x64, 0xf2, 0x89, 0x8e, 0x61, 0xa1, 0xab, 0xb7, 0xd4, 0xae, 0xd2, 0x55, 0x2d, 0x5b, 0xe1, - 0xc1, 0x9d, 0x1d, 0xa2, 0x27, 0xc6, 0x16, 0x9b, 0x79, 0x63, 0xdc, 0x66, 0xfb, 0x49, 0x1c, 0x0e, - 0xb7, 0xff, 0x32, 0xd5, 0xb1, 0xa7, 0x3a, 0x5b, 0x8d, 0x8e, 0x61, 0xe9, 0xf4, 0xf2, 0x7d, 0xb5, - 0x6f, 0x6b, 0x7d, 0xac, 0x8c, 0x1d, 0xab, 0x71, 0xeb, 0x79, 0xa0, 0x59, 0xa7, 0xb8, 0xa3, 0x5e, - 0x68, 0xba, 0xc9, 0x55, 0x2e, 0xba, 0xf2, 0x27, 0xde, 0xd9, 0xf2, 0xf6, 0x28, 0x15, 0x30, 0x6a, - 0xc7, 0xbd, 0xa4, 0xe7, 0x76, 0x2f, 0x2f, 0xc0, 0x52, 0x1f, 0x0f, 0x6d, 0xdf, 0x18, 0x99, 0xe1, - 0x64, 0xe8, 0x5e, 0x20, 0xf2, 0xcf, 0xeb, 0x9f, 0xd8, 0x10, 0x7a, 0x86, 0x86, 0x45, 0x43, 0xb7, - 0xb0, 0xa9, 0xa8, 0xed, 0xb6, 0x89, 0x2d, 0xab, 0x92, 0xa5, 0xdc, 0x65, 0x87, 0xbe, 0xc9, 0xc8, - 0xd2, 0x6f, 0xfd, 0x7b, 0x15, 0x0c, 0x83, 0x7c, 0x27, 0x04, 0x6f, 0x27, 0x8e, 0x60, 0x89, 0xcb, - 0xb7, 0x03, 0x9b, 0xc1, 0xd2, 0xd1, 0xc7, 0xc6, 0x0f, 0xdc, 0xe8, 0x26, 0x20, 0x47, 0x7c, 0x86, - 0x7d, 0x48, 0x7c, 0xbd, 0x7d, 0x40, 0x90, 0xa4, 0xab, 0x94, 0x64, 0x4e, 0x88, 0x7c, 0xff, 0xbf, - 0xed, 0xcd, 0x6b, 0xb0, 0x30, 0x96, 0x63, 0xb8, 0xf3, 0x12, 0x42, 0xe7, 0x15, 0xf7, 0xcf, 0x4b, - 0xfa, 0x9d, 0x00, 0xd5, 0xe8, 0xa4, 0x22, 0x54, 0xd5, 0xb3, 0xb0, 0xe0, 0xce, 0xc5, 0x1d, 0x1f, - 0x3b, 0xf5, 0xa2, 0xfb, 0x83, 0x0f, 0x30, 0xd2, 0x81, 0x3f, 0x09, 0xa5, 0x91, 0x94, 0x87, 0xed, - 0x42, 0xf1, 0xc2, 0xdf, 0xbf, 0xf4, 0xab, 0x84, 0xeb, 0x55, 0x03, 0x79, 0x49, 0x88, 0xe5, 0xbd, - 0x0e, 0x8b, 0x6d, 0xdc, 0xd2, 0xda, 0x5f, 0xd5, 0xf0, 0x16, 0xb8, 0xf4, 0x77, 0x76, 0x37, 0x83, - 0xdd, 0xfd, 0x12, 0x20, 0x2b, 0x63, 0xcb, 0x20, 0xd9, 0x07, 0xda, 0x82, 0x1c, 0x1e, 0xb6, 0xb0, - 0x61, 0x3b, 0x09, 0x5b, 0x78, 0x2a, 0xcc, 0xb8, 0xeb, 0x0e, 0x27, 0x01, 0x82, 0xae, 0x18, 0xba, - 0xc5, 0xb1, 0x6e, 0x34, 0x6c, 0xe5, 0xe2, 0x7e, 0xb0, 0xfb, 0xa2, 0x03, 0x76, 0x13, 0x91, 0x38, - 0x8e, 0x49, 0x8d, 0xa0, 0xdd, 0x5b, 0x1c, 0xed, 0x26, 0xa7, 0x74, 0x16, 0x80, 0xbb, 0xb5, 0x00, - 0xdc, 0x4d, 0x4d, 0x99, 0x66, 0x04, 0xde, 0x7d, 0xd1, 0xc1, 0xbb, 0xe9, 0x29, 0x23, 0x1e, 0x01, - 0xbc, 0xaf, 0xfa, 0x00, 0x6f, 0x96, 0x8a, 0xae, 0x46, 0x8a, 0x86, 0x20, 0xde, 0x97, 0x5d, 0xc4, - 0x9b, 0x8f, 0x44, 0xcb, 0x5c, 0x78, 0x14, 0xf2, 0x1e, 0x8c, 0x41, 0x5e, 0x06, 0x51, 0x9f, 0x8a, - 0x54, 0x31, 0x05, 0xf3, 0x1e, 0x8c, 0x61, 0xde, 0xe2, 0x14, 0x85, 0x53, 0x40, 0xef, 0x4f, 0xc3, - 0x41, 0x6f, 0x34, 0x2c, 0xe5, 0xc3, 0x9c, 0x0d, 0xf5, 0x2a, 0x11, 0xa8, 0xb7, 0x1c, 0x89, 0xd0, - 0x98, 0xfa, 0x99, 0x61, 0xef, 0x71, 0x08, 0xec, 0x65, 0x00, 0xf5, 0x46, 0xa4, 0xf2, 0x19, 0x70, - 0xef, 0x71, 0x08, 0xee, 0x5d, 0x98, 0xaa, 0x76, 0x2a, 0xf0, 0xbd, 0x17, 0x04, 0xbe, 0x28, 0x22, - 0xc7, 0xf2, 0x4e, 0x7b, 0x04, 0xf2, 0x3d, 0x8d, 0x42, 0xbe, 0x0c, 0x9d, 0x3e, 0x17, 0xa9, 0x71, - 0x0e, 0xe8, 0x7b, 0x30, 0x06, 0x7d, 0x97, 0xa6, 0x58, 0xda, 0xec, 0xd8, 0x37, 0x23, 0x66, 0x19, - 0xea, 0xdd, 0x4d, 0x66, 0x41, 0xcc, 0x4b, 0xcf, 0x90, 0x40, 0x3c, 0xe2, 0xe1, 0x48, 0x4e, 0x8c, - 0x4d, 0x53, 0x37, 0x39, 0x8a, 0x65, 0x0d, 0xe9, 0x06, 0xc1, 0x42, 0x9e, 0x37, 0x9b, 0x80, 0x93, - 0x29, 0xf6, 0xf0, 0x79, 0x30, 0xe9, 0xcf, 0x82, 0x27, 0x4b, 0x91, 0xb2, 0x1f, 0x47, 0xe5, 0x38, - 0x8e, 0xf2, 0xa1, 0xe7, 0x78, 0x10, 0x3d, 0xaf, 0x40, 0x9e, 0x60, 0x8a, 0x11, 0x60, 0xac, 0x1a, - 0x2e, 0x30, 0xbe, 0x09, 0x0b, 0x34, 0x76, 0x32, 0x8c, 0xcd, 0x03, 0x52, 0x92, 0x06, 0xa4, 0x32, - 0xf9, 0xc1, 0xd6, 0x85, 0x45, 0xa6, 0xe7, 0x61, 0xd1, 0xc7, 0xeb, 0x62, 0x15, 0x86, 0x12, 0x45, - 0x97, 0x7b, 0x93, 0x83, 0x96, 0xbf, 0x09, 0xde, 0x0a, 0x79, 0x88, 0x3a, 0x0c, 0xfc, 0x0a, 0xdf, - 0x10, 0xf8, 0x8d, 0x7f, 0x65, 0xf0, 0xeb, 0xc7, 0x5e, 0x89, 0x20, 0xf6, 0xfa, 0x8f, 0xe0, 0xed, - 0x89, 0x0b, 0x65, 0x5b, 0x7a, 0x1b, 0x73, 0x34, 0x44, 0xbf, 0x49, 0x76, 0xd2, 0xd5, 0xcf, 0x39, - 0xe6, 0x21, 0x9f, 0x84, 0xcb, 0x0d, 0x39, 0x39, 0x1e, 0x51, 0x5c, 0x20, 0xc5, 0x42, 0x3e, 0x07, - 0x52, 0x22, 0x24, 0x1e, 0x62, 0x16, 0x20, 0x0a, 0x32, 0xf9, 0x24, 0x7c, 0xd4, 0xec, 0x78, 0xe8, - 0x66, 0x0d, 0x74, 0x07, 0x72, 0xb4, 0x58, 0xad, 0xe8, 0x86, 0xc5, 0x63, 0x42, 0x20, 0xcb, 0x61, - 0x15, 0xeb, 0xb5, 0x43, 0xc2, 0x73, 0x60, 0x58, 0x72, 0xd6, 0xe0, 0x5f, 0xbe, 0x5c, 0x23, 0x17, - 0xc8, 0x35, 0xae, 0x41, 0x8e, 0x8c, 0xde, 0x32, 0xd4, 0x16, 0xa6, 0xa5, 0xd1, 0x9c, 0xec, 0x11, - 0xa4, 0x4f, 0x04, 0x28, 0x8f, 0x84, 0x98, 0xd0, 0xb9, 0x3b, 0x26, 0x19, 0xf7, 0x41, 0xfb, 0xeb, - 0x00, 0xe7, 0xaa, 0xa5, 0xbc, 0xa7, 0xf6, 0x6d, 0xdc, 0xe6, 0xd3, 0xcd, 0x9d, 0xab, 0xd6, 0x1b, - 0x94, 0x10, 0xec, 0x38, 0x3b, 0xd2, 0xb1, 0x0f, 0x43, 0xe6, 0xfc, 0x18, 0x12, 0x55, 0x21, 0x6b, - 0x98, 0x9a, 0x6e, 0x6a, 0xf6, 0x25, 0x1d, 0x6d, 0x42, 0x76, 0xdb, 0xbb, 0xc9, 0x6c, 0x42, 0x4c, - 0xee, 0x26, 0xb3, 0x49, 0x31, 0xe5, 0x16, 0xaa, 0xd8, 0x91, 0xcd, 0x8b, 0x05, 0xe9, 0x83, 0xb8, - 0x67, 0x8b, 0xdb, 0xb8, 0xab, 0x5d, 0x60, 0x73, 0x8e, 0xc9, 0xcc, 0xb6, 0xb9, 0xcb, 0x21, 0x53, - 0xf6, 0x51, 0xc8, 0xe8, 0x49, 0x6b, 0x60, 0xe1, 0x36, 0x2f, 0x99, 0xb8, 0x6d, 0xd4, 0x80, 0x34, - 0xbe, 0xc0, 0x7d, 0xdb, 0xaa, 0x64, 0xa8, 0x0d, 0x5f, 0x19, 0xc7, 0xb0, 0xe4, 0xf7, 0x56, 0x85, - 0x58, 0xee, 0x97, 0x9f, 0xad, 0x88, 0x8c, 0xfb, 0x39, 0xbd, 0xa7, 0xd9, 0xb8, 0x67, 0xd8, 0x97, - 0x32, 0x97, 0x9f, 0xbc, 0xb2, 0xd2, 0x6d, 0x28, 0x05, 0xe3, 0x3e, 0x7a, 0x02, 0x8a, 0x26, 0xb6, - 0x55, 0xad, 0xaf, 0x04, 0xb2, 0xf6, 0x02, 0x23, 0xf2, 0x62, 0xce, 0x21, 0x3c, 0x12, 0x1a, 0xeb, - 0xd1, 0x4b, 0x90, 0xf3, 0xd2, 0x04, 0x81, 0x0e, 0x7d, 0x42, 0xad, 0xc3, 0xe3, 0x95, 0xfe, 0x2a, - 0x78, 0x2a, 0x83, 0xd5, 0x93, 0x3a, 0xa4, 0x4d, 0x6c, 0x0d, 0xba, 0xac, 0x9e, 0x51, 0xda, 0x78, - 0x7e, 0xb6, 0x2c, 0x81, 0x50, 0x07, 0x5d, 0x5b, 0xe6, 0xc2, 0xd2, 0x5b, 0x90, 0x66, 0x14, 0x94, - 0x87, 0xcc, 0xf1, 0xfe, 0xfd, 0xfd, 0x83, 0x37, 0xf6, 0xc5, 0x18, 0x02, 0x48, 0x6f, 0xd6, 0x6a, - 0xf5, 0xc3, 0xa6, 0x28, 0xa0, 0x1c, 0xa4, 0x36, 0xb7, 0x0e, 0xe4, 0xa6, 0x18, 0x27, 0x64, 0xb9, - 0xbe, 0x5b, 0xaf, 0x35, 0xc5, 0x04, 0x5a, 0x80, 0x22, 0xfb, 0x56, 0xee, 0x1d, 0xc8, 0x0f, 0x36, - 0x9b, 0x62, 0xd2, 0x47, 0x3a, 0xaa, 0xef, 0x6f, 0xd7, 0x65, 0x31, 0x25, 0x7d, 0x0f, 0xae, 0x46, - 0xe6, 0x15, 0x5e, 0x69, 0x44, 0xf0, 0x95, 0x46, 0xa4, 0xdf, 0xc4, 0x09, 0xf2, 0x8a, 0x4a, 0x16, - 0xd0, 0xee, 0xc8, 0xc4, 0x37, 0xe6, 0xc8, 0x34, 0x46, 0x66, 0x4f, 0xc0, 0x96, 0x89, 0xcf, 0xb0, - 0xdd, 0xea, 0xb0, 0xe4, 0x85, 0xf9, 0xc6, 0xa2, 0x5c, 0xe4, 0x54, 0x2a, 0x64, 0x31, 0xb6, 0x77, - 0x70, 0xcb, 0x56, 0xd8, 0x09, 0x63, 0x40, 0x27, 0x47, 0xd8, 0x08, 0xf5, 0x88, 0x11, 0xa5, 0xb7, - 0xe7, 0x5a, 0xcb, 0x1c, 0xa4, 0xe4, 0x7a, 0x53, 0x7e, 0x53, 0x4c, 0x20, 0x04, 0x25, 0xfa, 0xa9, - 0x1c, 0xed, 0x6f, 0x1e, 0x1e, 0x35, 0x0e, 0xc8, 0x5a, 0x2e, 0x42, 0xd9, 0x59, 0x4b, 0x87, 0x98, - 0x92, 0xfe, 0x11, 0x87, 0x47, 0x23, 0x52, 0x1d, 0x74, 0x07, 0xc0, 0x1e, 0x2a, 0x26, 0x6e, 0xe9, - 0x66, 0x3b, 0xda, 0xc8, 0x9a, 0x43, 0x99, 0x72, 0xc8, 0x39, 0x9b, 0x7f, 0x59, 0x13, 0x2a, 0x6a, - 0xe8, 0x15, 0xae, 0x94, 0xcc, 0xca, 0x81, 0x77, 0xd7, 0x43, 0x0a, 0x47, 0xb8, 0x45, 0x14, 0xd3, - 0xb5, 0xa5, 0x8a, 0x29, 0x3f, 0x7a, 0xe0, 0x07, 0xc4, 0x03, 0x1a, 0x54, 0x66, 0x2e, 0xbd, 0xfa, - 0x20, 0x33, 0x23, 0x58, 0xe8, 0x4d, 0x78, 0x74, 0x24, 0x26, 0xba, 0x4a, 0x53, 0xb3, 0x86, 0xc6, - 0x47, 0x82, 0xa1, 0x91, 0xab, 0x96, 0x7e, 0x9f, 0xf0, 0x2f, 0x6c, 0x30, 0xb3, 0x3b, 0x80, 0xb4, - 0x65, 0xab, 0xf6, 0xc0, 0xe2, 0x06, 0xf7, 0xd2, 0xac, 0x69, 0xe2, 0x9a, 0xf3, 0x71, 0x44, 0xc5, - 0x65, 0xae, 0xe6, 0xbb, 0xf5, 0xb6, 0x88, 0x83, 0x0d, 0x2e, 0x4e, 0xf4, 0x91, 0xf1, 0x7c, 0x4e, - 0x5c, 0xba, 0x0b, 0x68, 0x3c, 0x81, 0x0e, 0x29, 0x99, 0x08, 0x61, 0x25, 0x93, 0x3f, 0x08, 0xf0, - 0xd8, 0x84, 0x64, 0x19, 0xbd, 0x3e, 0xb2, 0xcf, 0x2f, 0xcf, 0x93, 0x6a, 0xaf, 0x31, 0x5a, 0x70, - 0xa7, 0xa5, 0x5b, 0x50, 0xf0, 0xd3, 0x67, 0x9b, 0xe4, 0x97, 0x71, 0xcf, 0xe7, 0x07, 0x6b, 0x3b, - 0x5e, 0xf8, 0x13, 0xbe, 0x66, 0xf8, 0x0b, 0xda, 0x59, 0x7c, 0x4e, 0x3b, 0x3b, 0x0a, 0xb3, 0xb3, - 0xc4, 0x5c, 0x59, 0xe5, 0x5c, 0xd6, 0x96, 0xfc, 0x7a, 0xd6, 0x16, 0x38, 0x70, 0xa9, 0x60, 0xda, - 0xfa, 0x26, 0x80, 0x57, 0xf0, 0x22, 0x01, 0xc9, 0xd4, 0x07, 0xfd, 0x36, 0xb5, 0x80, 0x94, 0xcc, - 0x1a, 0xe8, 0x36, 0xa4, 0x88, 0x25, 0x39, 0xeb, 0x34, 0xee, 0x54, 0x89, 0x25, 0xf8, 0x0a, 0x66, - 0x8c, 0x5b, 0xd2, 0x00, 0x8d, 0x57, 0xd4, 0x23, 0xba, 0x78, 0x35, 0xd8, 0xc5, 0xe3, 0x91, 0xb5, - 0xf9, 0xf0, 0xae, 0xde, 0x87, 0x14, 0xdd, 0x79, 0x92, 0x70, 0xd1, 0x6b, 0x1c, 0x0e, 0x7b, 0xc8, - 0x37, 0xfa, 0x19, 0x80, 0x6a, 0xdb, 0xa6, 0x76, 0x3a, 0xf0, 0x3a, 0x58, 0x09, 0xb7, 0x9c, 0x4d, - 0x87, 0x6f, 0xeb, 0x1a, 0x37, 0xa1, 0x25, 0x4f, 0xd4, 0x67, 0x46, 0x3e, 0x85, 0xd2, 0x3e, 0x94, - 0x82, 0xb2, 0x4e, 0xa2, 0xce, 0xc6, 0x10, 0x4c, 0xd4, 0x19, 0xee, 0xe2, 0x89, 0xba, 0x9b, 0xe6, - 0x27, 0xd8, 0x5d, 0x15, 0x6d, 0x48, 0xff, 0x15, 0xa0, 0xe0, 0x37, 0xbc, 0x6f, 0x38, 0xfd, 0x9c, - 0x92, 0x71, 0x5f, 0x1d, 0xcb, 0x3e, 0x33, 0xe7, 0xaa, 0x75, 0xfc, 0x6d, 0x26, 0x9f, 0x1f, 0x08, - 0x90, 0x75, 0x27, 0x1f, 0xbc, 0xb6, 0x0a, 0xdc, 0xf3, 0xb1, 0xb5, 0x8b, 0xfb, 0xef, 0x9a, 0xd8, - 0xad, 0x5e, 0xc2, 0xbd, 0xd5, 0xbb, 0xeb, 0xe6, 0x4a, 0x51, 0x15, 0x3d, 0xff, 0x4a, 0x73, 0x9b, - 0x72, 0x52, 0xc3, 0x5f, 0xf3, 0x71, 0x90, 0x24, 0x01, 0xfd, 0x00, 0xd2, 0x6a, 0xcb, 0xad, 0x63, - 0x96, 0x42, 0x0a, 0x7c, 0x0e, 0xeb, 0x5a, 0x73, 0xb8, 0x49, 0x39, 0x65, 0x2e, 0xc1, 0x47, 0x15, - 0x77, 0x46, 0x25, 0xbd, 0x46, 0xf4, 0x32, 0x9e, 0xa0, 0x47, 0x2c, 0x01, 0x1c, 0xef, 0x3f, 0x38, - 0xd8, 0xde, 0xb9, 0xb7, 0x53, 0xdf, 0xe6, 0xd9, 0xd2, 0xf6, 0x76, 0x7d, 0x5b, 0x8c, 0x13, 0x3e, - 0xb9, 0xfe, 0xe0, 0xe0, 0xa4, 0xbe, 0x2d, 0x26, 0xa4, 0xbb, 0x90, 0x73, 0xbd, 0x0a, 0x41, 0xf5, - 0x4e, 0x4d, 0x56, 0xe0, 0x67, 0x9b, 0x97, 0xd8, 0x97, 0x20, 0x65, 0xe8, 0xef, 0xf1, 0x2b, 0xb6, - 0x84, 0xcc, 0x1a, 0x52, 0x1b, 0xca, 0x23, 0x2e, 0x09, 0xdd, 0x85, 0x8c, 0x31, 0x38, 0x55, 0x1c, - 0xa3, 0x1d, 0xa9, 0x60, 0x3b, 0x78, 0x71, 0x70, 0xda, 0xd5, 0x5a, 0xf7, 0xf1, 0xa5, 0xb3, 0x4c, - 0xc6, 0xe0, 0xf4, 0x3e, 0xb3, 0x6d, 0xd6, 0x4b, 0xdc, 0xdf, 0xcb, 0x05, 0x64, 0x9d, 0xa3, 0x8a, - 0x7e, 0x08, 0x39, 0xd7, 0xdb, 0xb9, 0x57, 0xe4, 0x91, 0x6e, 0x92, 0xab, 0xf7, 0x44, 0xd0, 0x4d, - 0x58, 0xb0, 0xb4, 0xf3, 0xbe, 0x53, 0xbf, 0x67, 0x15, 0x9b, 0x38, 0x3d, 0x33, 0x65, 0xf6, 0x63, - 0xcf, 0x29, 0x2a, 0x90, 0x20, 0x27, 0x8e, 0xfa, 0x8a, 0x6f, 0x73, 0x00, 0x21, 0xc1, 0x38, 0x11, - 0x16, 0x8c, 0x7f, 0x11, 0x87, 0xbc, 0xef, 0x56, 0x00, 0x7d, 0xdf, 0xe7, 0xb8, 0x4a, 0x21, 0x51, - 0xc4, 0xc7, 0xeb, 0xdd, 0x41, 0x07, 0x27, 0x16, 0x9f, 0x7f, 0x62, 0x51, 0x97, 0x30, 0xce, 0xe5, - 0x42, 0x72, 0xee, 0xcb, 0x85, 0xe7, 0x00, 0xd9, 0xba, 0xad, 0x76, 0x95, 0x0b, 0xdd, 0xd6, 0xfa, - 0xe7, 0x0a, 0x33, 0x0d, 0xe6, 0x66, 0x44, 0xfa, 0xe7, 0x84, 0xfe, 0x38, 0xa4, 0x56, 0xf2, 0x73, - 0x01, 0xb2, 0x2e, 0xa2, 0x9b, 0xf7, 0x86, 0xfa, 0x0a, 0xa4, 0x39, 0x68, 0x61, 0x57, 0xd4, 0xbc, - 0x15, 0x7a, 0x8b, 0x52, 0x85, 0x6c, 0x0f, 0xdb, 0x2a, 0xf5, 0x99, 0x2c, 0x02, 0xba, 0xed, 0x9b, - 0x2f, 0x43, 0xde, 0x77, 0xbb, 0x4f, 0xdc, 0xe8, 0x7e, 0xfd, 0x0d, 0x31, 0x56, 0xcd, 0x7c, 0xf8, - 0xf1, 0x6a, 0x62, 0x1f, 0xbf, 0x47, 0x4e, 0x98, 0x5c, 0xaf, 0x35, 0xea, 0xb5, 0xfb, 0xa2, 0x50, - 0xcd, 0x7f, 0xf8, 0xf1, 0x6a, 0x46, 0xc6, 0xb4, 0x80, 0x7e, 0xf3, 0x3e, 0x94, 0x47, 0x36, 0x26, - 0x78, 0xa0, 0x11, 0x94, 0xb6, 0x8f, 0x0f, 0xf7, 0x76, 0x6a, 0x9b, 0xcd, 0xba, 0x72, 0x72, 0xd0, - 0xac, 0x8b, 0x02, 0x7a, 0x14, 0x16, 0xf7, 0x76, 0x7e, 0xd4, 0x68, 0x2a, 0xb5, 0xbd, 0x9d, 0xfa, - 0x7e, 0x53, 0xd9, 0x6c, 0x36, 0x37, 0x6b, 0xf7, 0xc5, 0xf8, 0xc6, 0x1f, 0xf3, 0x50, 0xde, 0xdc, - 0xaa, 0xed, 0x10, 0xd8, 0xa6, 0xb5, 0x54, 0xea, 0x1e, 0x6a, 0x90, 0xa4, 0xa5, 0xc0, 0x89, 0x6f, - 0xfc, 0xaa, 0x93, 0x6f, 0x45, 0xd0, 0x3d, 0x48, 0xd1, 0x2a, 0x21, 0x9a, 0xfc, 0xe8, 0xaf, 0x3a, - 0xe5, 0x9a, 0x84, 0x0c, 0x86, 0x1e, 0xa7, 0x89, 0xaf, 0x00, 0xab, 0x93, 0x6f, 0x4d, 0xd0, 0x1e, - 0x64, 0x9c, 0x22, 0xd1, 0xb4, 0xa7, 0x79, 0xd5, 0xa9, 0x57, 0x19, 0x64, 0x6a, 0xac, 0xd8, 0x36, - 0xf9, 0x81, 0x60, 0x75, 0xca, 0x7d, 0x0a, 0xda, 0x81, 0x34, 0x2f, 0x74, 0x4c, 0x79, 0xf3, 0x57, - 0x9d, 0x76, 0x43, 0x82, 0x64, 0xc8, 0x79, 0x65, 0xcc, 0xe9, 0xcf, 0x1e, 0xab, 0x33, 0x5c, 0x15, - 0xa1, 0xb7, 0xa0, 0x18, 0x2c, 0xa8, 0xcc, 0xf6, 0xae, 0xb0, 0x3a, 0xe3, 0x5d, 0x0c, 0xd1, 0x1f, - 0xac, 0xae, 0xcc, 0xf6, 0xce, 0xb0, 0x3a, 0xe3, 0xd5, 0x0c, 0x7a, 0x07, 0x16, 0xc6, 0xab, 0x1f, - 0xb3, 0x3f, 0x3b, 0xac, 0xce, 0x71, 0x59, 0x83, 0x7a, 0x80, 0x42, 0xaa, 0x26, 0x73, 0xbc, 0x42, - 0xac, 0xce, 0x73, 0x77, 0x83, 0xda, 0x50, 0x1e, 0xad, 0x44, 0xcc, 0xfa, 0x2a, 0xb1, 0x3a, 0xf3, - 0x3d, 0x0e, 0xeb, 0x25, 0x08, 0xcb, 0x67, 0x7d, 0xa5, 0x58, 0x9d, 0xf9, 0x5a, 0x07, 0x1d, 0x03, - 0xf8, 0x60, 0xe5, 0x0c, 0xaf, 0x16, 0xab, 0xb3, 0x5c, 0xf0, 0x20, 0x03, 0x16, 0xc3, 0xf0, 0xe6, - 0x3c, 0x8f, 0x18, 0xab, 0x73, 0xdd, 0xfb, 0x10, 0x7b, 0x0e, 0x22, 0xc7, 0xd9, 0x1e, 0x35, 0x56, - 0x67, 0xbc, 0x00, 0xda, 0xaa, 0x7f, 0xf2, 0xf9, 0xb2, 0xf0, 0xe9, 0xe7, 0xcb, 0xc2, 0xbf, 0x3f, - 0x5f, 0x16, 0x3e, 0xfa, 0x62, 0x39, 0xf6, 0xe9, 0x17, 0xcb, 0xb1, 0x7f, 0x7e, 0xb1, 0x1c, 0xfb, - 0xf1, 0xb3, 0xe7, 0x9a, 0xdd, 0x19, 0x9c, 0xae, 0xb5, 0xf4, 0xde, 0xba, 0xff, 0x1d, 0x78, 0xd8, - 0xeb, 0xf3, 0xd3, 0x34, 0x0d, 0xa8, 0xb7, 0xfe, 0x17, 0x00, 0x00, 0xff, 0xff, 0x4c, 0xb4, 0x7f, - 0x53, 0x9d, 0x2e, 0x00, 0x00, + 0x11, 0xd7, 0xe8, 0x5b, 0x2d, 0x4b, 0x1a, 0x3f, 0x9b, 0x45, 0x2b, 0x76, 0x6d, 0x33, 0x14, 0xb0, + 0x2c, 0x60, 0x13, 0x6f, 0x16, 0x96, 0x2c, 0x84, 0x92, 0x65, 0x6d, 0x64, 0xaf, 0xd7, 0x36, 0x63, + 0xd9, 0x14, 0xf9, 0x60, 0x18, 0x4b, 0xcf, 0xd6, 0xb0, 0x92, 0x66, 0x98, 0x19, 0x19, 0x99, 0x63, + 0x12, 0xaa, 0x52, 0x1c, 0x52, 0xdc, 0xc2, 0x21, 0xdc, 0x92, 0xaa, 0xfc, 0x09, 0xc9, 0x25, 0xa7, + 0x1c, 0x38, 0xe4, 0xc0, 0x29, 0x95, 0x13, 0x49, 0xc1, 0x8d, 0x7f, 0x20, 0xb7, 0x54, 0xea, 0x7d, + 0xcc, 0x97, 0x34, 0xa3, 0x0f, 0xa0, 0xa8, 0x4a, 0x15, 0xb7, 0x79, 0x3d, 0xdd, 0xfd, 0xbe, 0xfa, + 0x75, 0xf7, 0xaf, 0xdf, 0x83, 0xc7, 0x6c, 0xdc, 0x6f, 0x63, 0xb3, 0xa7, 0xf5, 0xed, 0x0d, 0xf5, + 0xb4, 0xa5, 0x6d, 0xd8, 0x97, 0x06, 0xb6, 0xd6, 0x0d, 0x53, 0xb7, 0x75, 0x54, 0xf2, 0x7e, 0xae, + 0x93, 0x9f, 0x95, 0xeb, 0x3e, 0xee, 0x96, 0x79, 0x69, 0xd8, 0xfa, 0x86, 0x61, 0xea, 0xfa, 0x19, + 0xe3, 0xaf, 0x5c, 0x1b, 0xff, 0xfd, 0x10, 0x5f, 0x72, 0x6d, 0x01, 0x61, 0xda, 0xcb, 0x86, 0xa1, + 0x9a, 0x6a, 0xcf, 0xf9, 0xbd, 0x7a, 0xae, 0xeb, 0xe7, 0x5d, 0xbc, 0x41, 0x5b, 0xa7, 0x83, 0xb3, + 0x0d, 0x5b, 0xeb, 0x61, 0xcb, 0x56, 0x7b, 0x06, 0x67, 0x58, 0x3e, 0xd7, 0xcf, 0x75, 0xfa, 0xb9, + 0x41, 0xbe, 0x18, 0x55, 0xfa, 0x4b, 0x0e, 0x32, 0x32, 0x7e, 0x77, 0x80, 0x2d, 0x1b, 0x6d, 0x42, + 0x12, 0xb7, 0x3a, 0x7a, 0x59, 0x58, 0x13, 0x6e, 0xe4, 0x37, 0xaf, 0xad, 0x8f, 0x0c, 0x7f, 0x9d, + 0xf3, 0xd5, 0x5b, 0x1d, 0xbd, 0x11, 0x93, 0x29, 0x2f, 0xba, 0x0d, 0xa9, 0xb3, 0xee, 0xc0, 0xea, + 0x94, 0xe3, 0x54, 0xe8, 0x7a, 0x94, 0xd0, 0x3d, 0xc2, 0xd4, 0x88, 0xc9, 0x8c, 0x9b, 0x74, 0xa5, + 0xf5, 0xcf, 0xf4, 0x72, 0x62, 0x72, 0x57, 0x3b, 0xfd, 0x33, 0xda, 0x15, 0xe1, 0x45, 0x5b, 0x00, + 0x5a, 0x5f, 0xb3, 0x95, 0x56, 0x47, 0xd5, 0xfa, 0xe5, 0x24, 0x95, 0x7c, 0x3c, 0x5a, 0x52, 0xb3, + 0x6b, 0x84, 0xb1, 0x11, 0x93, 0x73, 0x9a, 0xd3, 0x20, 0xc3, 0x7d, 0x77, 0x80, 0xcd, 0xcb, 0x72, + 0x6a, 0xf2, 0x70, 0x5f, 0x27, 0x4c, 0x64, 0xb8, 0x94, 0x1b, 0xbd, 0x02, 0xd9, 0x56, 0x07, 0xb7, + 0x1e, 0x2a, 0xf6, 0xb0, 0x9c, 0xa1, 0x92, 0xab, 0x51, 0x92, 0x35, 0xc2, 0xd7, 0x1c, 0x36, 0x62, + 0x72, 0xa6, 0xc5, 0x3e, 0xd1, 0x1d, 0x48, 0xb7, 0xf4, 0x5e, 0x4f, 0xb3, 0xcb, 0x40, 0x65, 0x57, + 0x22, 0x65, 0x29, 0x57, 0x23, 0x26, 0x73, 0x7e, 0xb4, 0x0f, 0xc5, 0xae, 0x66, 0xd9, 0x8a, 0xd5, + 0x57, 0x0d, 0xab, 0xa3, 0xdb, 0x56, 0x39, 0x4f, 0x35, 0x3c, 0x19, 0xa5, 0x61, 0x4f, 0xb3, 0xec, + 0x23, 0x87, 0xb9, 0x11, 0x93, 0x0b, 0x5d, 0x3f, 0x81, 0xe8, 0xd3, 0xcf, 0xce, 0xb0, 0xe9, 0x2a, + 0x2c, 0x2f, 0x4c, 0xd6, 0x77, 0x40, 0xb8, 0x1d, 0x79, 0xa2, 0x4f, 0xf7, 0x13, 0xd0, 0xcf, 0x60, + 0xa9, 0xab, 0xab, 0x6d, 0x57, 0x9d, 0xd2, 0xea, 0x0c, 0xfa, 0x0f, 0xcb, 0x05, 0xaa, 0xf4, 0x99, + 0xc8, 0x41, 0xea, 0x6a, 0xdb, 0x51, 0x51, 0x23, 0x02, 0x8d, 0x98, 0xbc, 0xd8, 0x1d, 0x25, 0xa2, + 0xb7, 0x60, 0x59, 0x35, 0x8c, 0xee, 0xe5, 0xa8, 0xf6, 0x22, 0xd5, 0x7e, 0x33, 0x4a, 0x7b, 0x95, + 0xc8, 0x8c, 0xaa, 0x47, 0xea, 0x18, 0x15, 0x35, 0x41, 0x34, 0x4c, 0x6c, 0xa8, 0x26, 0x56, 0x0c, + 0x53, 0x37, 0x74, 0x4b, 0xed, 0x96, 0x4b, 0x54, 0xf7, 0xd3, 0x51, 0xba, 0x0f, 0x19, 0xff, 0x21, + 0x67, 0x6f, 0xc4, 0xe4, 0x92, 0x11, 0x24, 0x31, 0xad, 0x7a, 0x0b, 0x5b, 0x96, 0xa7, 0x55, 0x9c, + 0xa6, 0x95, 0xf2, 0x07, 0xb5, 0x06, 0x48, 0xa8, 0x0e, 0x79, 0x3c, 0x24, 0xe2, 0xca, 0x85, 0x6e, + 0xe3, 0xf2, 0x22, 0x55, 0x28, 0x45, 0x9e, 0x50, 0xca, 0x7a, 0xa2, 0xdb, 0xb8, 0x11, 0x93, 0x01, + 0xbb, 0x2d, 0xa4, 0xc2, 0x23, 0x17, 0xd8, 0xd4, 0xce, 0x2e, 0xa9, 0x1a, 0x85, 0xfe, 0xb1, 0x34, + 0xbd, 0x5f, 0x46, 0x54, 0xe1, 0xb3, 0x51, 0x0a, 0x4f, 0xa8, 0x10, 0x51, 0x51, 0x77, 0x44, 0x1a, + 0x31, 0x79, 0xe9, 0x62, 0x9c, 0x4c, 0x4c, 0xec, 0x4c, 0xeb, 0xab, 0x5d, 0xed, 0x7d, 0xac, 0x9c, + 0x76, 0xf5, 0xd6, 0xc3, 0xf2, 0xd2, 0x64, 0x13, 0xbb, 0xc7, 0xb9, 0xb7, 0x08, 0x33, 0x31, 0xb1, + 0x33, 0x3f, 0x61, 0x2b, 0x03, 0xa9, 0x0b, 0xb5, 0x3b, 0xc0, 0xbb, 0xc9, 0x6c, 0x5a, 0xcc, 0xec, + 0x26, 0xb3, 0x59, 0x31, 0xb7, 0x9b, 0xcc, 0xe6, 0x44, 0x90, 0x9e, 0x86, 0xbc, 0xcf, 0x25, 0xa1, + 0x32, 0x64, 0x7a, 0xd8, 0xb2, 0xd4, 0x73, 0x4c, 0x3d, 0x58, 0x4e, 0x76, 0x9a, 0x52, 0x11, 0x16, + 0xfc, 0x6e, 0x48, 0xfa, 0x48, 0x70, 0x25, 0x89, 0x87, 0x21, 0x92, 0x17, 0xd8, 0xa4, 0x0b, 0xc1, + 0x25, 0x79, 0x13, 0x3d, 0x01, 0x05, 0x3a, 0x09, 0xc5, 0xf9, 0x4f, 0xdc, 0x5c, 0x52, 0x5e, 0xa0, + 0xc4, 0x13, 0xce, 0xb4, 0x0a, 0x79, 0x63, 0xd3, 0x70, 0x59, 0x12, 0x94, 0x05, 0x8c, 0x4d, 0xc3, + 0x61, 0x78, 0x1c, 0x16, 0xc8, 0x8c, 0x5d, 0x8e, 0x24, 0xed, 0x24, 0x4f, 0x68, 0x9c, 0x45, 0xfa, + 0x7b, 0x1c, 0xc4, 0x51, 0xd7, 0x85, 0xee, 0x40, 0x92, 0x78, 0x71, 0xee, 0x90, 0x2b, 0xeb, 0xcc, + 0xc5, 0xaf, 0x3b, 0x2e, 0x7e, 0xbd, 0xe9, 0xb8, 0xf8, 0xad, 0xec, 0xa7, 0x9f, 0xaf, 0xc6, 0x3e, + 0xfa, 0xd7, 0xaa, 0x20, 0x53, 0x09, 0x74, 0x95, 0x38, 0x2c, 0x55, 0xeb, 0x2b, 0x5a, 0x9b, 0x0e, + 0x39, 0x47, 0xbc, 0x91, 0xaa, 0xf5, 0x77, 0xda, 0x68, 0x0f, 0xc4, 0x96, 0xde, 0xb7, 0x70, 0xdf, + 0x1a, 0x58, 0x0a, 0x0b, 0x21, 0xdc, 0x0d, 0x07, 0x9c, 0x29, 0x0b, 0x64, 0x35, 0x87, 0xf3, 0x90, + 0x32, 0xca, 0xa5, 0x56, 0x90, 0x80, 0xee, 0x01, 0x5c, 0xa8, 0x5d, 0xad, 0xad, 0xda, 0xba, 0x69, + 0x95, 0x93, 0x6b, 0x89, 0x1b, 0xf9, 0xcd, 0xb5, 0xb1, 0xad, 0x3e, 0x71, 0x58, 0x8e, 0x8d, 0xb6, + 0x6a, 0xe3, 0xad, 0x24, 0x19, 0xae, 0xec, 0x93, 0x44, 0x4f, 0x41, 0x49, 0x35, 0x0c, 0xc5, 0xb2, + 0x55, 0x1b, 0x2b, 0xa7, 0x97, 0x36, 0xb6, 0xa8, 0x8b, 0x5e, 0x90, 0x0b, 0xaa, 0x61, 0x1c, 0x11, + 0xea, 0x16, 0x21, 0xa2, 0x27, 0xa1, 0x48, 0xbc, 0xb9, 0xa6, 0x76, 0x95, 0x0e, 0xd6, 0xce, 0x3b, + 0x76, 0x39, 0xbd, 0x26, 0xdc, 0x48, 0xc8, 0x05, 0x4e, 0x6d, 0x50, 0xa2, 0xd4, 0x76, 0x77, 0x9c, + 0x7a, 0x72, 0x84, 0x20, 0xd9, 0x56, 0x6d, 0x95, 0xae, 0xe4, 0x82, 0x4c, 0xbf, 0x09, 0xcd, 0x50, + 0xed, 0x0e, 0x5f, 0x1f, 0xfa, 0x8d, 0xae, 0x40, 0x9a, 0xab, 0x4d, 0x50, 0xb5, 0xbc, 0x85, 0x96, + 0x21, 0x65, 0x98, 0xfa, 0x05, 0xa6, 0x5b, 0x97, 0x95, 0x59, 0x43, 0x92, 0xa1, 0x18, 0xf4, 0xfa, + 0xa8, 0x08, 0x71, 0x7b, 0xc8, 0x7b, 0x89, 0xdb, 0x43, 0xf4, 0x02, 0x24, 0xc9, 0x42, 0xd2, 0x3e, + 0x8a, 0x21, 0x71, 0x8e, 0xcb, 0x35, 0x2f, 0x0d, 0x2c, 0x53, 0x4e, 0xa9, 0x04, 0x85, 0x40, 0x34, + 0x90, 0xae, 0xc0, 0x72, 0x98, 0x73, 0x97, 0x3a, 0x2e, 0x3d, 0xe0, 0xa4, 0xd1, 0x6d, 0xc8, 0xba, + 0xde, 0x9d, 0x19, 0xce, 0xd5, 0xb1, 0x6e, 0x1d, 0x66, 0xd9, 0x65, 0x25, 0x16, 0x43, 0x36, 0xa0, + 0xa3, 0xf2, 0x58, 0xbe, 0x20, 0x67, 0x54, 0xc3, 0x68, 0xa8, 0x56, 0x47, 0x7a, 0x1b, 0xca, 0x51, + 0x9e, 0xdb, 0xb7, 0x60, 0x02, 0x35, 0x7b, 0x67, 0xc1, 0xae, 0x40, 0xfa, 0x4c, 0x37, 0x7b, 0xaa, + 0x4d, 0x95, 0x15, 0x64, 0xde, 0x22, 0x0b, 0xc9, 0xbc, 0x78, 0x82, 0x92, 0x59, 0x43, 0x52, 0xe0, + 0x6a, 0xa4, 0xf7, 0x26, 0x22, 0x5a, 0xbf, 0x8d, 0xd9, 0xb2, 0x16, 0x64, 0xd6, 0xf0, 0x14, 0xb1, + 0xc1, 0xb2, 0x06, 0xe9, 0xd6, 0xa2, 0x73, 0xa5, 0xfa, 0x73, 0x32, 0x6f, 0x49, 0x1f, 0x27, 0xe0, + 0x4a, 0xb8, 0x0f, 0x47, 0x6b, 0xb0, 0xd0, 0x53, 0x87, 0x8a, 0x3d, 0xe4, 0x66, 0x27, 0xd0, 0x8d, + 0x87, 0x9e, 0x3a, 0x6c, 0x0e, 0x99, 0xcd, 0x89, 0x90, 0xb0, 0x87, 0x56, 0x39, 0xbe, 0x96, 0xb8, + 0xb1, 0x20, 0x93, 0x4f, 0x74, 0x0c, 0x8b, 0x5d, 0xbd, 0xa5, 0x76, 0x95, 0xae, 0x6a, 0xd9, 0x0a, + 0x0f, 0xee, 0xec, 0x10, 0x3d, 0x31, 0xb6, 0xd8, 0xcc, 0x1b, 0xe3, 0x36, 0xdb, 0x4f, 0xe2, 0x70, + 0xb8, 0xfd, 0x97, 0xa8, 0x8e, 0x3d, 0xd5, 0xd9, 0x6a, 0xb4, 0x0d, 0xf9, 0x9e, 0x66, 0x9d, 0xe2, + 0x8e, 0x7a, 0xa1, 0xe9, 0x26, 0x3f, 0x4d, 0xe3, 0x46, 0xf3, 0xc0, 0xe3, 0xe1, 0x9a, 0xfc, 0x62, + 0xbe, 0x2d, 0x49, 0x05, 0x6c, 0xd8, 0xf1, 0x26, 0xe9, 0xb9, 0xbd, 0xc9, 0x0b, 0xb0, 0xdc, 0xc7, + 0x43, 0x5b, 0xf1, 0xce, 0x2b, 0xb3, 0x93, 0x0c, 0x5d, 0x7a, 0x44, 0xfe, 0xb9, 0x27, 0xdc, 0x22, + 0x26, 0x83, 0x9e, 0xa1, 0x51, 0xd0, 0xd0, 0x2d, 0x6c, 0x2a, 0x6a, 0xbb, 0x6d, 0x62, 0xcb, 0x2a, + 0x67, 0x29, 0x77, 0xc9, 0xa1, 0x57, 0x19, 0x59, 0xfa, 0x8d, 0x7f, 0x6b, 0x82, 0x51, 0x8f, 0x2f, + 0xbc, 0xe0, 0x2d, 0xfc, 0x11, 0x2c, 0x73, 0xf9, 0x76, 0x60, 0xed, 0x59, 0xf6, 0xf9, 0xd8, 0xf8, + 0xf9, 0x1a, 0x5d, 0x73, 0xe4, 0x88, 0x47, 0x2f, 0x7b, 0xe2, 0xeb, 0x2d, 0x3b, 0x82, 0x24, 0x5d, + 0x94, 0x24, 0x73, 0x31, 0xe4, 0xfb, 0xff, 0x6d, 0x2b, 0x5e, 0x83, 0xc5, 0xb1, 0x0c, 0xc2, 0x9d, + 0x97, 0x10, 0x3a, 0xaf, 0xb8, 0x7f, 0x5e, 0xd2, 0xef, 0x05, 0xa8, 0x44, 0xa7, 0x0c, 0xa1, 0xaa, + 0x9e, 0x85, 0x45, 0x77, 0x2e, 0xee, 0xf8, 0xd8, 0x99, 0x16, 0xdd, 0x1f, 0x7c, 0x80, 0x91, 0xee, + 0xf9, 0x49, 0x28, 0x8e, 0x24, 0x34, 0x6c, 0x17, 0x0a, 0x17, 0xfe, 0xfe, 0xa5, 0x5f, 0x27, 0x5c, + 0x9f, 0x19, 0xc8, 0x3a, 0x42, 0x0c, 0xed, 0x75, 0x58, 0x6a, 0xe3, 0x96, 0xd6, 0xfe, 0xba, 0x76, + 0xb6, 0xc8, 0xa5, 0xbf, 0x37, 0xb3, 0x71, 0x33, 0xfb, 0x2d, 0x40, 0x56, 0xc6, 0x96, 0x41, 0x52, + 0x09, 0xb4, 0x05, 0x39, 0x3c, 0x6c, 0x61, 0xc3, 0x76, 0xb2, 0xaf, 0xf0, 0xbc, 0x96, 0x71, 0xd7, + 0x1d, 0x4e, 0x82, 0xea, 0x5c, 0x31, 0x74, 0x8b, 0x03, 0xd7, 0x68, 0x0c, 0xca, 0xc5, 0xfd, 0xc8, + 0xf5, 0x45, 0x07, 0xb9, 0x26, 0x22, 0x41, 0x19, 0x93, 0x1a, 0x81, 0xae, 0xb7, 0x38, 0x74, 0x4d, + 0x4e, 0xe9, 0x2c, 0x80, 0x5d, 0x6b, 0x01, 0xec, 0x9a, 0x9a, 0x32, 0xcd, 0x08, 0xf0, 0xfa, 0xa2, + 0x03, 0x5e, 0xd3, 0x53, 0x46, 0x3c, 0x82, 0x5e, 0x5f, 0xf5, 0xa1, 0xd7, 0x2c, 0x15, 0x5d, 0x8b, + 0x14, 0x0d, 0x81, 0xaf, 0x2f, 0xbb, 0xf0, 0x35, 0x1f, 0x09, 0x7d, 0xb9, 0xf0, 0x28, 0x7e, 0x3d, + 0x18, 0xc3, 0xaf, 0x0c, 0x6f, 0x3e, 0x15, 0xa9, 0x62, 0x0a, 0x80, 0x3d, 0x18, 0x03, 0xb0, 0x85, + 0x29, 0x0a, 0xa7, 0x20, 0xd8, 0x9f, 0x87, 0x23, 0xd8, 0x68, 0x8c, 0xc9, 0x87, 0x39, 0x1b, 0x84, + 0x55, 0x22, 0x20, 0x6c, 0x29, 0x12, 0x6e, 0x31, 0xf5, 0x33, 0x63, 0xd8, 0xe3, 0x10, 0x0c, 0xcb, + 0xd0, 0xe6, 0x8d, 0x48, 0xe5, 0x33, 0x80, 0xd8, 0xe3, 0x10, 0x10, 0xbb, 0x38, 0x55, 0xed, 0x54, + 0x14, 0x7b, 0x2f, 0x88, 0x62, 0x51, 0x44, 0xc2, 0xe4, 0x9d, 0xf6, 0x08, 0x18, 0x7b, 0x1a, 0x05, + 0x63, 0x19, 0xd4, 0x7c, 0x2e, 0x52, 0xe3, 0x1c, 0x38, 0xf6, 0x60, 0x0c, 0xc7, 0x2e, 0x4f, 0xb1, + 0xb4, 0xd9, 0x81, 0x6c, 0x46, 0xcc, 0x32, 0x08, 0xbb, 0x9b, 0xcc, 0x82, 0x98, 0x97, 0x9e, 0x21, + 0x71, 0x77, 0xc4, 0xc3, 0x91, 0x04, 0x17, 0x9b, 0xa6, 0x6e, 0x72, 0x48, 0xca, 0x1a, 0xd2, 0x0d, + 0x02, 0x6c, 0x3c, 0x6f, 0x36, 0x01, 0xf4, 0x52, 0x20, 0xe1, 0xf3, 0x60, 0xd2, 0x9f, 0x05, 0x4f, + 0x96, 0xc2, 0x5e, 0x3f, 0x28, 0xca, 0x71, 0x50, 0xe4, 0x83, 0xc2, 0xf1, 0x20, 0x14, 0x5e, 0x85, + 0x3c, 0x01, 0x08, 0x23, 0x28, 0x57, 0x35, 0x5c, 0x94, 0x7b, 0x13, 0x16, 0x69, 0xa8, 0x64, 0x80, + 0x99, 0x07, 0xa4, 0x24, 0x0d, 0x48, 0x25, 0xf2, 0x83, 0xad, 0x0b, 0x8b, 0x4c, 0xcf, 0xc3, 0x92, + 0x8f, 0xd7, 0x05, 0x1e, 0x0c, 0xf2, 0x89, 0x2e, 0x77, 0x95, 0x23, 0x90, 0xbf, 0x09, 0xde, 0x0a, + 0x79, 0xf0, 0x38, 0x0c, 0xc9, 0x0a, 0xdf, 0x12, 0x92, 0x8d, 0x7f, 0x6d, 0x24, 0xeb, 0x07, 0x52, + 0x89, 0x20, 0x90, 0xfa, 0x8f, 0xe0, 0xed, 0x89, 0x8b, 0x4b, 0x5b, 0x7a, 0x1b, 0x73, 0x68, 0x43, + 0xbf, 0x49, 0x32, 0xd2, 0xd5, 0xcf, 0x39, 0x80, 0x21, 0x9f, 0x84, 0xcb, 0x0d, 0x39, 0x39, 0x1e, + 0x51, 0x5c, 0x54, 0xc4, 0x42, 0x3e, 0x47, 0x45, 0x22, 0x24, 0x1e, 0x62, 0x16, 0x20, 0x16, 0x64, + 0xf2, 0x49, 0xf8, 0xa8, 0xd9, 0xf1, 0xd0, 0xcd, 0x1a, 0xe8, 0x0e, 0xe4, 0x68, 0xe5, 0x59, 0xd1, + 0x0d, 0x8b, 0xc7, 0x84, 0x40, 0x52, 0xc3, 0xca, 0xcf, 0xeb, 0x87, 0x84, 0xe7, 0xc0, 0xb0, 0xe4, + 0xac, 0xc1, 0xbf, 0x7c, 0xb9, 0x46, 0x2e, 0x90, 0x6b, 0x5c, 0x83, 0x1c, 0x19, 0xbd, 0x65, 0xa8, + 0x2d, 0x4c, 0xeb, 0x9c, 0x39, 0xd9, 0x23, 0x48, 0x9f, 0x0a, 0x50, 0x1a, 0x09, 0x31, 0xa1, 0x73, + 0x77, 0x4c, 0x32, 0xee, 0xc3, 0xe9, 0xd7, 0x01, 0xce, 0x55, 0x4b, 0x79, 0x4f, 0xed, 0xdb, 0xb8, + 0xcd, 0xa7, 0x9b, 0x3b, 0x57, 0xad, 0x37, 0x28, 0x21, 0xd8, 0x71, 0x76, 0xa4, 0x63, 0x1f, 0x20, + 0xcc, 0xf9, 0x01, 0x21, 0xaa, 0x40, 0xd6, 0x30, 0x35, 0xdd, 0xd4, 0xec, 0x4b, 0x3a, 0xda, 0x84, + 0xec, 0xb6, 0x77, 0x93, 0xd9, 0x84, 0x98, 0xdc, 0x4d, 0x66, 0x93, 0x62, 0xca, 0xad, 0x3a, 0xb1, + 0x23, 0x9b, 0x17, 0x17, 0xa4, 0x0f, 0xe2, 0x9e, 0x2d, 0x6e, 0xe3, 0xae, 0x76, 0x81, 0xcd, 0x39, + 0x26, 0x33, 0xdb, 0xe6, 0xae, 0x84, 0x4c, 0xd9, 0x47, 0x21, 0xa3, 0x27, 0xad, 0x81, 0x85, 0xdb, + 0xbc, 0xfe, 0xe1, 0xb6, 0x51, 0x03, 0xd2, 0xf8, 0x02, 0xf7, 0x6d, 0xab, 0x9c, 0xa1, 0x36, 0x7c, + 0x65, 0x1c, 0x90, 0x92, 0xdf, 0x5b, 0x65, 0x62, 0xb9, 0x5f, 0x7d, 0xbe, 0x2a, 0x32, 0xee, 0xe7, + 0xf4, 0x9e, 0x66, 0xe3, 0x9e, 0x61, 0x5f, 0xca, 0x5c, 0x7e, 0xf2, 0xca, 0x4a, 0x55, 0x28, 0x06, + 0xe3, 0x3e, 0x7a, 0x02, 0x0a, 0x26, 0xb6, 0x55, 0xad, 0xaf, 0x04, 0x92, 0xf4, 0x05, 0x46, 0x64, + 0x27, 0x7f, 0x37, 0x99, 0x15, 0xc4, 0xf8, 0x6e, 0x32, 0x1b, 0x17, 0x13, 0xd2, 0x21, 0x3c, 0x12, + 0x1a, 0xf7, 0xd1, 0x4b, 0x90, 0xf3, 0x52, 0x06, 0x81, 0x4e, 0x63, 0x42, 0x11, 0xc3, 0xe3, 0x95, + 0xfe, 0x2a, 0x78, 0x2a, 0x83, 0x65, 0x91, 0x3a, 0xa4, 0x4d, 0x6c, 0x0d, 0xba, 0xac, 0x50, 0x51, + 0xdc, 0x7c, 0x7e, 0xb6, 0x8c, 0x81, 0x50, 0x07, 0x5d, 0x5b, 0xe6, 0xc2, 0xd2, 0x5b, 0x90, 0x66, + 0x14, 0x94, 0x87, 0xcc, 0xf1, 0xfe, 0xfd, 0xfd, 0x83, 0x37, 0xf6, 0xc5, 0x18, 0x02, 0x48, 0x57, + 0x6b, 0xb5, 0xfa, 0x61, 0x53, 0x14, 0x50, 0x0e, 0x52, 0xd5, 0xad, 0x03, 0xb9, 0x29, 0xc6, 0x09, + 0x59, 0xae, 0xef, 0xd6, 0x6b, 0x4d, 0x31, 0x81, 0x16, 0xa1, 0xc0, 0xbe, 0x95, 0x7b, 0x07, 0xf2, + 0x83, 0x6a, 0x53, 0x4c, 0xfa, 0x48, 0x47, 0xf5, 0xfd, 0xed, 0xba, 0x2c, 0xa6, 0xa4, 0x1f, 0xc0, + 0xd5, 0xc8, 0x1c, 0xc3, 0xab, 0x79, 0x08, 0xbe, 0x9a, 0x87, 0xf4, 0x71, 0x9c, 0x80, 0xae, 0xa8, + 0xc4, 0x01, 0xed, 0x8e, 0x4c, 0x7c, 0x73, 0x8e, 0xac, 0x63, 0x64, 0xf6, 0x04, 0x67, 0x99, 0xf8, + 0x0c, 0xdb, 0xad, 0x0e, 0x4b, 0x64, 0x98, 0x9f, 0x2c, 0xc8, 0x05, 0x4e, 0xa5, 0x42, 0x16, 0x63, + 0x7b, 0x07, 0xb7, 0x6c, 0x85, 0x9d, 0x36, 0x8b, 0x82, 0x9d, 0x1c, 0x61, 0x23, 0xd4, 0x23, 0x46, + 0x94, 0xde, 0x9e, 0x6b, 0x2d, 0x73, 0x90, 0x92, 0xeb, 0x4d, 0xf9, 0x4d, 0x31, 0x81, 0x10, 0x14, + 0xe9, 0xa7, 0x72, 0xb4, 0x5f, 0x3d, 0x3c, 0x6a, 0x1c, 0x90, 0xb5, 0x5c, 0x82, 0x92, 0xb3, 0x96, + 0x0e, 0x31, 0x25, 0xfd, 0x23, 0x0e, 0x8f, 0x46, 0xa4, 0x3d, 0xe8, 0x0e, 0x80, 0x3d, 0x54, 0x4c, + 0xdc, 0xd2, 0xcd, 0x76, 0xb4, 0x91, 0x35, 0x87, 0x32, 0xe5, 0x90, 0x73, 0x36, 0xff, 0xb2, 0x26, + 0x94, 0xca, 0xd0, 0x2b, 0x5c, 0x29, 0x99, 0x95, 0xc5, 0x21, 0xde, 0xf5, 0x90, 0x8a, 0x10, 0x6e, + 0x11, 0xc5, 0x74, 0x6d, 0xa9, 0x62, 0xca, 0x8f, 0x1e, 0xf8, 0xb1, 0xf0, 0x80, 0x06, 0x98, 0x99, + 0x6b, 0xaa, 0x3e, 0xb4, 0xcc, 0x08, 0x16, 0x7a, 0x13, 0x1e, 0x1d, 0x89, 0x8f, 0xae, 0xd2, 0xd4, + 0xac, 0x61, 0xf2, 0x91, 0x60, 0x98, 0xe4, 0xaa, 0xa5, 0x3f, 0x24, 0xfc, 0x0b, 0x1b, 0xcc, 0xf2, + 0x0e, 0x20, 0x6d, 0xd9, 0xaa, 0x3d, 0xb0, 0xb8, 0xc1, 0xbd, 0x34, 0x6b, 0xca, 0xb8, 0xee, 0x7c, + 0x1c, 0x51, 0x71, 0x99, 0xab, 0xf9, 0x7e, 0xbd, 0x2d, 0xe9, 0x36, 0x14, 0x83, 0x8b, 0x13, 0x7d, + 0x64, 0x3c, 0x9f, 0x13, 0x97, 0xee, 0x02, 0x1a, 0x4f, 0xa6, 0x43, 0xaa, 0x25, 0x42, 0x58, 0xb5, + 0xe4, 0x8f, 0x02, 0x3c, 0x36, 0x21, 0x71, 0x46, 0xaf, 0x8f, 0xec, 0xf3, 0xcb, 0xf3, 0xa4, 0xdd, + 0xeb, 0x8c, 0x16, 0xdc, 0x69, 0xe9, 0x16, 0x2c, 0xf8, 0xe9, 0xb3, 0x4d, 0xf2, 0xab, 0xb8, 0xe7, + 0xf3, 0x83, 0x65, 0x1d, 0x2f, 0x14, 0x0a, 0xdf, 0x30, 0x14, 0x06, 0xed, 0x2c, 0x3e, 0xa7, 0x9d, + 0x1d, 0x85, 0xd9, 0x59, 0x62, 0xae, 0x0c, 0x73, 0x2e, 0x6b, 0x4b, 0x7e, 0x33, 0x6b, 0x0b, 0x1c, + 0xb8, 0x54, 0x30, 0x85, 0x7d, 0x13, 0xc0, 0xab, 0x75, 0x91, 0x80, 0x64, 0xea, 0x83, 0x7e, 0x9b, + 0x5a, 0x40, 0x4a, 0x66, 0x0d, 0x74, 0x1b, 0x52, 0xc4, 0x92, 0x9c, 0x75, 0x1a, 0x77, 0xaa, 0xc4, + 0x12, 0x7c, 0xb5, 0x32, 0xc6, 0x2d, 0x69, 0x80, 0xc6, 0x4b, 0xe5, 0x11, 0x5d, 0xbc, 0x1a, 0xec, + 0xe2, 0xf1, 0xc8, 0xa2, 0x7b, 0x78, 0x57, 0xef, 0x43, 0x8a, 0xee, 0x3c, 0x49, 0xbe, 0xe8, 0xfd, + 0x0c, 0x87, 0x40, 0xe4, 0x1b, 0xfd, 0x02, 0x40, 0xb5, 0x6d, 0x53, 0x3b, 0x1d, 0x78, 0x1d, 0xac, + 0x86, 0x5b, 0x4e, 0xd5, 0xe1, 0xdb, 0xba, 0xc6, 0x4d, 0x68, 0xd9, 0x13, 0xf5, 0x99, 0x91, 0x4f, + 0xa1, 0xb4, 0x0f, 0xc5, 0xa0, 0xac, 0x93, 0xb4, 0xb3, 0x31, 0x04, 0x93, 0x76, 0x86, 0xc1, 0x78, + 0xd2, 0xee, 0xa6, 0xfc, 0x09, 0x76, 0x09, 0x45, 0x1b, 0xd2, 0x7f, 0x05, 0x58, 0xf0, 0x1b, 0xde, + 0xb7, 0x9c, 0x8a, 0x4e, 0xc9, 0xbe, 0xaf, 0x8e, 0x65, 0xa2, 0x99, 0x73, 0xd5, 0x3a, 0xfe, 0x2e, + 0x13, 0xd1, 0x0f, 0x04, 0xc8, 0xba, 0x93, 0x0f, 0xde, 0x47, 0x05, 0x2e, 0xf0, 0xd8, 0xda, 0xc5, + 0xfd, 0x97, 0x48, 0xec, 0xba, 0x2e, 0xe1, 0x5e, 0xd7, 0xdd, 0x75, 0x73, 0xa5, 0xa8, 0xea, 0x9e, + 0x7f, 0xa5, 0xb9, 0x4d, 0x39, 0xa9, 0xe1, 0xef, 0xf8, 0x38, 0x48, 0x92, 0x80, 0x7e, 0x04, 0x69, + 0xb5, 0xe5, 0xd6, 0x34, 0x8b, 0x21, 0xc5, 0x3e, 0x87, 0x75, 0xbd, 0x39, 0xac, 0x52, 0x4e, 0x99, + 0x4b, 0xf0, 0x51, 0xc5, 0x9d, 0x51, 0x49, 0xaf, 0x11, 0xbd, 0x8c, 0x27, 0xe8, 0x11, 0x8b, 0x00, + 0xc7, 0xfb, 0x0f, 0x0e, 0xb6, 0x77, 0xee, 0xed, 0xd4, 0xb7, 0x79, 0xb6, 0xb4, 0xbd, 0x5d, 0xdf, + 0x16, 0xe3, 0x84, 0x4f, 0xae, 0x3f, 0x38, 0x38, 0xa9, 0x6f, 0x8b, 0x09, 0xe9, 0x2e, 0xe4, 0x5c, + 0xaf, 0x42, 0x10, 0xbe, 0x53, 0x9f, 0x15, 0xf8, 0xd9, 0xe6, 0xd5, 0xf5, 0x65, 0x48, 0x19, 0xfa, + 0x7b, 0xfc, 0xee, 0x2c, 0x21, 0xb3, 0x86, 0xd4, 0x86, 0xd2, 0x88, 0x4b, 0x42, 0x77, 0x21, 0x63, + 0x0c, 0x4e, 0x15, 0xc7, 0x68, 0x47, 0xaa, 0xd8, 0x0e, 0x76, 0x1c, 0x9c, 0x76, 0xb5, 0xd6, 0x7d, + 0x7c, 0xe9, 0x2c, 0x93, 0x31, 0x38, 0xbd, 0xcf, 0x6c, 0x9b, 0xf5, 0x12, 0xf7, 0xf7, 0x72, 0x01, + 0x59, 0xe7, 0xa8, 0xa2, 0x1f, 0x43, 0xce, 0xf5, 0x76, 0xee, 0xdd, 0x77, 0xa4, 0x9b, 0xe4, 0xea, + 0x3d, 0x11, 0x74, 0x13, 0x16, 0x2d, 0xed, 0xbc, 0xef, 0x94, 0xee, 0x59, 0xf5, 0x26, 0x4e, 0xcf, + 0x4c, 0x89, 0xfd, 0xd8, 0x73, 0x0a, 0x0c, 0x24, 0xc8, 0x89, 0xa3, 0xbe, 0xe2, 0xbb, 0x1c, 0x40, + 0x48, 0x30, 0x4e, 0x84, 0x05, 0xe3, 0x5f, 0xc5, 0x21, 0xef, 0xbb, 0x19, 0x40, 0x3f, 0xf4, 0x39, + 0xae, 0x62, 0x48, 0x14, 0xf1, 0xf1, 0x7a, 0x97, 0xcb, 0xc1, 0x89, 0xc5, 0xe7, 0x9f, 0x58, 0xd4, + 0xfd, 0x8b, 0x73, 0xd1, 0x90, 0x9c, 0xfb, 0xa2, 0xe1, 0x39, 0x40, 0xb6, 0x6e, 0xab, 0x5d, 0xe5, + 0x42, 0xb7, 0xb5, 0xfe, 0xb9, 0xc2, 0x4c, 0x83, 0xb9, 0x19, 0x91, 0xfe, 0x39, 0xa1, 0x3f, 0x0e, + 0xa9, 0x95, 0xfc, 0x52, 0x80, 0xac, 0x8b, 0xe8, 0xe6, 0xbd, 0x7a, 0xbe, 0x02, 0x69, 0x0e, 0x5a, + 0xd8, 0xdd, 0x33, 0x6f, 0x85, 0xde, 0xa8, 0x54, 0x20, 0xdb, 0xc3, 0xb6, 0x4a, 0x7d, 0x26, 0x8b, + 0x80, 0x6e, 0xfb, 0xe6, 0xcb, 0x90, 0xf7, 0x5d, 0xdb, 0x13, 0x37, 0xba, 0x5f, 0x7f, 0x43, 0x8c, + 0x55, 0x32, 0x1f, 0x7e, 0xb2, 0x96, 0xd8, 0xc7, 0xef, 0x91, 0x13, 0x26, 0xd7, 0x6b, 0x8d, 0x7a, + 0xed, 0xbe, 0x28, 0x54, 0xf2, 0x1f, 0x7e, 0xb2, 0x96, 0x91, 0x31, 0x2d, 0xa6, 0xdf, 0xbc, 0x0f, + 0xa5, 0x91, 0x8d, 0x09, 0x1e, 0x68, 0x04, 0xc5, 0xed, 0xe3, 0xc3, 0xbd, 0x9d, 0x5a, 0xb5, 0x59, + 0x57, 0x4e, 0x0e, 0x9a, 0x75, 0x51, 0x40, 0x8f, 0xc2, 0xd2, 0xde, 0xce, 0x4f, 0x1a, 0x4d, 0xa5, + 0xb6, 0xb7, 0x53, 0xdf, 0x6f, 0x2a, 0xd5, 0x66, 0xb3, 0x5a, 0xbb, 0x2f, 0xc6, 0x37, 0xff, 0x94, + 0x87, 0x52, 0x75, 0xab, 0xb6, 0x43, 0x60, 0x9b, 0xd6, 0x52, 0xa9, 0x7b, 0xa8, 0x41, 0x92, 0x96, + 0x05, 0x27, 0x3e, 0xde, 0xab, 0x4c, 0xbe, 0x21, 0x41, 0xf7, 0x20, 0x45, 0x2b, 0x86, 0x68, 0xf2, + 0x6b, 0xbe, 0xca, 0x94, 0x2b, 0x13, 0x32, 0x18, 0x7a, 0x9c, 0x26, 0x3e, 0xef, 0xab, 0x4c, 0xbe, + 0x41, 0x41, 0x7b, 0x90, 0x71, 0x0a, 0x46, 0xd3, 0xde, 0xdc, 0x55, 0xa6, 0x5e, 0x6b, 0x90, 0xa9, + 0xb1, 0xc2, 0xdb, 0xe4, 0x97, 0x7f, 0x95, 0x29, 0x77, 0x2b, 0x68, 0x07, 0xd2, 0xbc, 0xe8, 0x31, + 0xe5, 0x31, 0x5f, 0x65, 0xda, 0x6d, 0x09, 0x92, 0x21, 0xe7, 0x95, 0x34, 0xa7, 0xbf, 0x67, 0xac, + 0xcc, 0x70, 0x6d, 0x84, 0xde, 0x82, 0x42, 0xb0, 0xa0, 0x32, 0xdb, 0x83, 0xc1, 0xca, 0x8c, 0xf7, + 0x32, 0x44, 0x7f, 0xb0, 0xba, 0x32, 0xdb, 0x03, 0xc2, 0xca, 0x8c, 0xd7, 0x34, 0xe8, 0x1d, 0x58, + 0x1c, 0xaf, 0x7e, 0xcc, 0xfe, 0x9e, 0xb0, 0x32, 0xc7, 0xc5, 0x0d, 0xea, 0x01, 0x0a, 0xa9, 0x9a, + 0xcc, 0xf1, 0xbc, 0xb0, 0x32, 0xcf, 0x3d, 0x0e, 0x6a, 0x43, 0x69, 0xb4, 0x12, 0x31, 0xeb, 0x73, + 0xc3, 0xca, 0xcc, 0x77, 0x3a, 0xac, 0x97, 0x20, 0x2c, 0x9f, 0xf5, 0xf9, 0x61, 0x65, 0xe6, 0x2b, + 0x1e, 0x74, 0x0c, 0xe0, 0x83, 0x95, 0x33, 0x3c, 0x47, 0xac, 0xcc, 0x72, 0xd9, 0x83, 0x0c, 0x58, + 0x0a, 0xc3, 0x9b, 0xf3, 0xbc, 0x4e, 0xac, 0xcc, 0x75, 0x07, 0x44, 0xec, 0x39, 0x88, 0x1c, 0x67, + 0x7b, 0xad, 0x58, 0x99, 0xf1, 0x32, 0x68, 0xab, 0xfe, 0xe9, 0x17, 0x2b, 0xc2, 0x67, 0x5f, 0xac, + 0x08, 0xff, 0xfe, 0x62, 0x45, 0xf8, 0xe8, 0xcb, 0x95, 0xd8, 0x67, 0x5f, 0xae, 0xc4, 0xfe, 0xf9, + 0xe5, 0x4a, 0xec, 0xa7, 0xcf, 0x9e, 0x6b, 0x76, 0x67, 0x70, 0xba, 0xde, 0xd2, 0x7b, 0x1b, 0xfe, + 0x07, 0xde, 0x61, 0xcf, 0xca, 0x4f, 0xd3, 0x34, 0xa0, 0xde, 0xfa, 0x5f, 0x00, 0x00, 0x00, 0xff, + 0xff, 0xdc, 0x85, 0x19, 0x4e, 0x76, 0x2e, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -5510,10 +5509,10 @@ func (m *RequestPrepareProposal) MarshalToSizedBuffer(dAtA []byte) (int, error) i-- dAtA[i] = 0x28 } - if len(m.ByzantineValidators) > 0 { - for iNdEx := len(m.ByzantineValidators) - 1; iNdEx >= 0; iNdEx-- { + if len(m.Misbehavior) > 0 { + for iNdEx := len(m.Misbehavior) - 1; iNdEx >= 0; iNdEx-- { { - size, err := m.ByzantineValidators[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + size, err := m.Misbehavior[iNdEx].MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } @@ -5605,10 +5604,10 @@ func (m *RequestProcessProposal) MarshalToSizedBuffer(dAtA []byte) (int, error) i-- dAtA[i] = 0x22 } - if len(m.ByzantineValidators) > 0 { - for iNdEx := len(m.ByzantineValidators) - 1; iNdEx >= 0; iNdEx-- { + if len(m.Misbehavior) > 0 { + for iNdEx := len(m.Misbehavior) - 1; iNdEx >= 0; iNdEx-- { { - size, err := m.ByzantineValidators[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + size, err := m.Misbehavior[iNdEx].MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } @@ -5779,10 +5778,10 @@ func (m *RequestFinalizeBlock) MarshalToSizedBuffer(dAtA []byte) (int, error) { i-- dAtA[i] = 0x22 } - if len(m.ByzantineValidators) > 0 { - for iNdEx := len(m.ByzantineValidators) - 1; iNdEx >= 0; iNdEx-- { + if len(m.Misbehavior) > 0 { + for iNdEx := len(m.Misbehavior) - 1; iNdEx >= 0; iNdEx-- { { - size, err := m.ByzantineValidators[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + size, err := m.Misbehavior[iNdEx].MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } @@ -8145,8 +8144,8 @@ func (m *RequestPrepareProposal) Size() (n int) { } l = m.LocalLastCommit.Size() n += 1 + l + sovTypes(uint64(l)) - if len(m.ByzantineValidators) > 0 { - for _, e := range m.ByzantineValidators { + if len(m.Misbehavior) > 0 { + for _, e := range m.Misbehavior { l = e.Size() n += 1 + l + sovTypes(uint64(l)) } @@ -8181,8 +8180,8 @@ func (m *RequestProcessProposal) Size() (n int) { } l = m.ProposedLastCommit.Size() n += 1 + l + sovTypes(uint64(l)) - if len(m.ByzantineValidators) > 0 { - for _, e := range m.ByzantineValidators { + if len(m.Misbehavior) > 0 { + for _, e := range m.Misbehavior { l = e.Size() n += 1 + l + sovTypes(uint64(l)) } @@ -8261,8 +8260,8 @@ func (m *RequestFinalizeBlock) Size() (n int) { } l = m.DecidedLastCommit.Size() n += 1 + l + sovTypes(uint64(l)) - if len(m.ByzantineValidators) > 0 { - for _, e := range m.ByzantineValidators { + if len(m.Misbehavior) > 0 { + for _, e := range m.Misbehavior { l = e.Size() n += 1 + l + sovTypes(uint64(l)) } @@ -11139,7 +11138,7 @@ func (m *RequestPrepareProposal) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 4: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field ByzantineValidators", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Misbehavior", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -11166,8 +11165,8 @@ func (m *RequestPrepareProposal) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.ByzantineValidators = append(m.ByzantineValidators, Misbehavior{}) - if err := m.ByzantineValidators[len(m.ByzantineValidators)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + m.Misbehavior = append(m.Misbehavior, Misbehavior{}) + if err := m.Misbehavior[len(m.Misbehavior)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -11408,7 +11407,7 @@ func (m *RequestProcessProposal) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 3: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field ByzantineValidators", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Misbehavior", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -11435,8 +11434,8 @@ func (m *RequestProcessProposal) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.ByzantineValidators = append(m.ByzantineValidators, Misbehavior{}) - if err := m.ByzantineValidators[len(m.ByzantineValidators)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + m.Misbehavior = append(m.Misbehavior, Misbehavior{}) + if err := m.Misbehavior[len(m.Misbehavior)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -11985,7 +11984,7 @@ func (m *RequestFinalizeBlock) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 3: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field ByzantineValidators", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Misbehavior", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -12012,8 +12011,8 @@ func (m *RequestFinalizeBlock) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.ByzantineValidators = append(m.ByzantineValidators, Misbehavior{}) - if err := m.ByzantineValidators[len(m.ByzantineValidators)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + m.Misbehavior = append(m.Misbehavior, Misbehavior{}) + if err := m.Misbehavior[len(m.Misbehavior)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex diff --git a/docs/rfc/rfc-013-abci++.md b/docs/rfc/rfc-013-abci++.md index 0289c187e..6e83c9aa2 100644 --- a/docs/rfc/rfc-013-abci++.md +++ b/docs/rfc/rfc-013-abci++.md @@ -3,7 +3,7 @@ ## Changelog - 2020-01-11: initialized -- 2021-02-11: Migrate RFC to tendermint repo (Originally [RFC 004](https://github.com/tendermint/spec/pull/254)) +- 2022-02-11: Migrate RFC to tendermint repo (Originally [RFC 004](https://github.com/tendermint/spec/pull/254)) ## Author(s) diff --git a/docs/tools/debugging/proposer-based-timestamps-runbook.md b/docs/tools/debugging/proposer-based-timestamps-runbook.md index a817bd29e..cb32248dd 100644 --- a/docs/tools/debugging/proposer-based-timestamps-runbook.md +++ b/docs/tools/debugging/proposer-based-timestamps-runbook.md @@ -213,4 +213,4 @@ documentation](https://hub.cosmos.network/main/governance/submitting.html#sendin If the application does not implement a way to update the consensus parameters programatically, then the application itself must be updated to do so. More information on updating -the consensus parameters via ABCI can be found in the [FinalizeBlock documentation](https://github.com/tendermint/tendermint/blob/master/spec/abci++/abci++_methods_002_draft.md#finalizeblock). +the consensus parameters via ABCI can be found in the [FinalizeBlock documentation](../../../spec/abci++/abci%2B%2B_methods.md#finalizeblock). diff --git a/internal/state/execution.go b/internal/state/execution.go index 2710478e6..2638a67c6 100644 --- a/internal/state/execution.go +++ b/internal/state/execution.go @@ -105,14 +105,14 @@ func (blockExec *BlockExecutor) CreateProposalBlock( rpp, err := blockExec.appClient.PrepareProposal( ctx, &abci.RequestPrepareProposal{ - MaxTxBytes: maxDataBytes, - Txs: block.Txs.ToSliceOfBytes(), - LocalLastCommit: buildExtendedCommitInfo(lastExtCommit, blockExec.store, state.InitialHeight, state.ConsensusParams.ABCI), - ByzantineValidators: block.Evidence.ToABCI(), - Height: block.Height, - Time: block.Time, - NextValidatorsHash: block.NextValidatorsHash, - ProposerAddress: block.ProposerAddress, + MaxTxBytes: maxDataBytes, + Txs: block.Txs.ToSliceOfBytes(), + LocalLastCommit: buildExtendedCommitInfo(lastExtCommit, blockExec.store, state.InitialHeight, state.ConsensusParams.ABCI), + Misbehavior: block.Evidence.ToABCI(), + Height: block.Height, + Time: block.Time, + NextValidatorsHash: block.NextValidatorsHash, + ProposerAddress: block.ProposerAddress, }, ) if err != nil { @@ -147,14 +147,14 @@ func (blockExec *BlockExecutor) ProcessProposal( state State, ) (bool, error) { resp, err := blockExec.appClient.ProcessProposal(ctx, &abci.RequestProcessProposal{ - Hash: block.Header.Hash(), - Height: block.Header.Height, - Time: block.Header.Time, - Txs: block.Data.Txs.ToSliceOfBytes(), - ProposedLastCommit: buildLastCommitInfo(block, blockExec.store, state.InitialHeight), - ByzantineValidators: block.Evidence.ToABCI(), - ProposerAddress: block.ProposerAddress, - NextValidatorsHash: block.NextValidatorsHash, + Hash: block.Header.Hash(), + Height: block.Header.Height, + Time: block.Header.Time, + Txs: block.Data.Txs.ToSliceOfBytes(), + ProposedLastCommit: buildLastCommitInfo(block, blockExec.store, state.InitialHeight), + Misbehavior: block.Evidence.ToABCI(), + ProposerAddress: block.ProposerAddress, + NextValidatorsHash: block.NextValidatorsHash, }) if err != nil { return false, ErrInvalidBlock(err) @@ -208,14 +208,14 @@ func (blockExec *BlockExecutor) ApplyBlock( fBlockRes, err := blockExec.appClient.FinalizeBlock( ctx, &abci.RequestFinalizeBlock{ - Hash: block.Hash(), - Height: block.Header.Height, - Time: block.Header.Time, - Txs: block.Txs.ToSliceOfBytes(), - DecidedLastCommit: buildLastCommitInfo(block, blockExec.store, state.InitialHeight), - ByzantineValidators: block.Evidence.ToABCI(), - ProposerAddress: block.ProposerAddress, - NextValidatorsHash: block.NextValidatorsHash, + Hash: block.Hash(), + Height: block.Header.Height, + Time: block.Header.Time, + Txs: block.Txs.ToSliceOfBytes(), + DecidedLastCommit: buildLastCommitInfo(block, blockExec.store, state.InitialHeight), + Misbehavior: block.Evidence.ToABCI(), + ProposerAddress: block.ProposerAddress, + NextValidatorsHash: block.NextValidatorsHash, }, ) endTime := time.Now().UnixNano() @@ -677,12 +677,12 @@ func ExecCommitBlock( finalizeBlockResponse, err := appConn.FinalizeBlock( ctx, &abci.RequestFinalizeBlock{ - Hash: block.Hash(), - Height: block.Height, - Time: block.Time, - Txs: block.Txs.ToSliceOfBytes(), - DecidedLastCommit: buildLastCommitInfo(block, store, initialHeight), - ByzantineValidators: block.Evidence.ToABCI(), + Hash: block.Hash(), + Height: block.Height, + Time: block.Time, + Txs: block.Txs.ToSliceOfBytes(), + DecidedLastCommit: buildLastCommitInfo(block, store, initialHeight), + Misbehavior: block.Evidence.ToABCI(), }, ) diff --git a/internal/state/execution_test.go b/internal/state/execution_test.go index 79557b787..93b0d1cfb 100644 --- a/internal/state/execution_test.go +++ b/internal/state/execution_test.go @@ -158,8 +158,8 @@ func TestFinalizeBlockDecidedLastCommit(t *testing.T) { } } -// TestFinalizeBlockByzantineValidators ensures we send byzantine validators list. -func TestFinalizeBlockByzantineValidators(t *testing.T) { +// TestFinalizeBlockMisbehavior ensures we send misbehavior list. +func TestFinalizeBlockMisbehavior(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -274,7 +274,7 @@ func TestFinalizeBlockByzantineValidators(t *testing.T) { require.NoError(t, err) // TODO check state and mempool - assert.Equal(t, abciMb, app.ByzantineValidators) + assert.Equal(t, abciMb, app.Misbehavior) } func TestProcessProposal(t *testing.T) { @@ -338,11 +338,11 @@ func TestProcessProposal(t *testing.T) { block1.Txs = txs expectedRpp := &abci.RequestProcessProposal{ - Txs: block1.Txs.ToSliceOfBytes(), - Hash: block1.Hash(), - Height: block1.Header.Height, - Time: block1.Header.Time, - ByzantineValidators: block1.Evidence.ToABCI(), + Txs: block1.Txs.ToSliceOfBytes(), + Hash: block1.Hash(), + Height: block1.Header.Height, + Time: block1.Header.Time, + Misbehavior: block1.Evidence.ToABCI(), ProposedLastCommit: abci.CommitInfo{ Round: 0, Votes: voteInfos, diff --git a/internal/state/helpers_test.go b/internal/state/helpers_test.go index 354a2874f..bfdc8b1f5 100644 --- a/internal/state/helpers_test.go +++ b/internal/state/helpers_test.go @@ -266,9 +266,9 @@ func makeRandomStateFromConsensusParams( type testApp struct { abci.BaseApplication - CommitVotes []abci.VoteInfo - ByzantineValidators []abci.Misbehavior - ValidatorUpdates []abci.ValidatorUpdate + CommitVotes []abci.VoteInfo + Misbehavior []abci.Misbehavior + ValidatorUpdates []abci.ValidatorUpdate } var _ abci.Application = (*testApp)(nil) @@ -279,7 +279,7 @@ func (app *testApp) Info(_ context.Context, req *abci.RequestInfo) (*abci.Respon func (app *testApp) FinalizeBlock(_ context.Context, req *abci.RequestFinalizeBlock) (*abci.ResponseFinalizeBlock, error) { app.CommitVotes = req.DecidedLastCommit.Votes - app.ByzantineValidators = req.ByzantineValidators + app.Misbehavior = req.Misbehavior resTxs := make([]*abci.ExecTxResult, len(req.Txs)) for i, tx := range req.Txs { diff --git a/proto/tendermint/abci/types.proto b/proto/tendermint/abci/types.proto index aa9e3fcbe..7f9e57df5 100644 --- a/proto/tendermint/abci/types.proto +++ b/proto/tendermint/abci/types.proto @@ -109,7 +109,7 @@ message RequestPrepareProposal { // sent to the app for possible modifications. repeated bytes txs = 2; ExtendedCommitInfo local_last_commit = 3 [(gogoproto.nullable) = false]; - repeated Misbehavior byzantine_validators = 4 [(gogoproto.nullable) = false]; + repeated Misbehavior misbehavior = 4 [(gogoproto.nullable) = false]; int64 height = 5; google.protobuf.Timestamp time = 6 [(gogoproto.nullable) = false, (gogoproto.stdtime) = true]; bytes next_validators_hash = 7; @@ -120,7 +120,7 @@ message RequestPrepareProposal { message RequestProcessProposal { repeated bytes txs = 1; CommitInfo proposed_last_commit = 2 [(gogoproto.nullable) = false]; - repeated Misbehavior byzantine_validators = 3 [(gogoproto.nullable) = false]; + repeated Misbehavior misbehavior = 3 [(gogoproto.nullable) = false]; // hash is the merkle root hash of the fields of the proposed block. bytes hash = 4; int64 height = 5; @@ -147,8 +147,8 @@ message RequestVerifyVoteExtension { message RequestFinalizeBlock { repeated bytes txs = 1; CommitInfo decided_last_commit = 2 [(gogoproto.nullable) = false]; - repeated Misbehavior byzantine_validators = 3 [(gogoproto.nullable) = false]; - // hash is the merkle root hash of the fields of the proposed block. + repeated Misbehavior misbehavior = 3 [(gogoproto.nullable) = false]; + // hash is the merkle root hash of the fields of the decided block. bytes hash = 4; int64 height = 5; google.protobuf.Timestamp time = 6 [(gogoproto.nullable) = false, (gogoproto.stdtime) = true]; @@ -248,7 +248,7 @@ message ResponseDeliverTx { } message ResponseCommit { - // reserve 1 + reserved 1, 2; int64 retain_height = 3; } diff --git a/spec/abci++/README.md b/spec/abci++/README.md index a22babfee..2cd2bb483 100644 --- a/spec/abci++/README.md +++ b/spec/abci++/README.md @@ -25,14 +25,14 @@ This allows Tendermint to run with applications written in many programming lang This specification is split as follows: -- [Overview and basic concepts](./abci++_basic_concepts_002_draft.md) - interface's overview and concepts +- [Overview and basic concepts](./abci++_basic_concepts.md) - interface's overview and concepts needed to understand other parts of this specification. -- [Methods](./abci++_methods_002_draft.md) - complete details on all ABCI++ methods +- [Methods](./abci++_methods.md) - complete details on all ABCI++ methods and message types. -- [Requirements for the Application](./abci++_app_requirements_002_draft.md) - formal requirements +- [Requirements for the Application](./abci++_app_requirements.md) - formal requirements on the Application's logic to ensure Tendermint properties such as liveness. These requirements define what Tendermint expects from the Application; second part on managing ABCI application state and related topics. -- [Tendermint's expected behavior](./abci++_tmint_expected_behavior_002_draft.md) - specification of +- [Tendermint's expected behavior](./abci++_tmint_expected_behavior.md) - specification of how the different ABCI++ methods may be called by Tendermint. This explains what the Application is to expect from Tendermint. - [Client and Server](../abci/client-server.md) - for those looking to implement their diff --git a/spec/abci++/abci++_app_requirements_002_draft.md b/spec/abci++/abci++_app_requirements.md similarity index 93% rename from spec/abci++/abci++_app_requirements_002_draft.md rename to spec/abci++/abci++_app_requirements.md index 3203f7a95..cd4d877c4 100644 --- a/spec/abci++/abci++_app_requirements_002_draft.md +++ b/spec/abci++/abci++_app_requirements.md @@ -1,23 +1,24 @@ --- order: 3 -title: Application Requirements +title: Requirements for the Application --- -# Application Requirements +# Requirements for the Application ## Formal Requirements This section specifies what Tendermint expects from the Application. It is structured as a set of formal requirements that can be used for testing and verification of the Application's logic. -Let *p* and *q* be two different correct proposers in rounds *rp* and *rq* -respectively, in height *h*. +Let *p* and *q* be two correct processes. +Let *rp* (resp. *rq*) be a round of height *h* where *p* (resp. *q*) is the +proposer. Let *sp,h-1* be *p*'s Application's state committed for height *h-1*. Let *vp* (resp. *vq*) be the block that *p*'s (resp. *q*'s) Tendermint passes on to the Application via `RequestPrepareProposal` as proposer of round *rp* (resp *rq*), height *h*, also known as the raw proposal. -Let *v'p* (resp. *v'q*) the possibly modified block *p*'s (resp. *q*'s) Application +Let *up* (resp. *uq*) the possibly modified block *p*'s (resp. *q*'s) Application returns via `ResponsePrepareProposal` to Tendermint, also known as the prepared proposal. Process *p*'s prepared proposal can differ in two different rounds where *p* is the proposer. @@ -49,6 +50,7 @@ same-block execution mode and *does not* provide values for Full execution of blocks at `PrepareProposal` time stands on Tendermint's critical path. Thus, Requirement 3 ensures the Application will set a value for `TimeoutPropose` such that the time it takes to fully execute blocks in `PrepareProposal` does not interfere with Tendermint's propose timer. +Note that violation of Requirement 3 may just lead to further rounds, but will not compromise liveness. * Requirement 4 [`PrepareProposal`, tx-size]: When *p*'s Application calls `ResponsePrepareProposal`, the total size in bytes of the transactions returned does not exceed `RequestPrepareProposal.max_tx_bytes`. @@ -62,7 +64,7 @@ transaction list returned by the application will never cause the resulting bloc limit. * Requirement 5 [`PrepareProposal`, `ProcessProposal`, coherence]: For any two correct processes *p* and *q*, - if *q*'s Tendermint calls `RequestProcessProposal` on *v'p*, + if *q*'s Tendermint calls `RequestProcessProposal` on *up*, *q*'s Application returns Accept in `ResponseProcessProposal`. Requirement 5 makes sure that blocks proposed by correct processes *always* pass the correct receiving process's @@ -75,14 +77,14 @@ serious consequences on Tendermint's liveness that this entails. Due to its crit target for extensive testing and automated verification. * Requirement 6 [`ProcessProposal`, determinism-1]: `ProcessProposal` is a (deterministic) function of the current - state and the block that is about to be applied. In other words, for any correct process *p*, and any arbitrary block *v'*, - if *p*'s Tendermint calls `RequestProcessProposal` on *v'* at height *h*, - then *p*'s Application's acceptance or rejection **exclusively** depends on *v'* and *sp,h-1*. + state and the block that is about to be applied. In other words, for any correct process *p*, and any arbitrary block *u*, + if *p*'s Tendermint calls `RequestProcessProposal` on *u* at height *h*, + then *p*'s Application's acceptance or rejection **exclusively** depends on *u* and *sp,h-1*. * Requirement 7 [`ProcessProposal`, determinism-2]: For any two correct processes *p* and *q*, and any arbitrary - block *v'*, - if *p*'s (resp. *q*'s) Tendermint calls `RequestProcessProposal` on *v'* at height *h*, - then *p*'s Application accepts *v'* if and only if *q*'s Application accepts *v'*. + block *u*, + if *p*'s (resp. *q*'s) Tendermint calls `RequestProcessProposal` on *u* at height *h*, + then *p*'s Application accepts *u* if and only if *q*'s Application accepts *u*. Note that this requirement follows from Requirement 6 and the Agreement property of consensus. Requirements 6 and 7 ensure that all correct processes will react in the same way to a proposed block, even @@ -97,7 +99,7 @@ of `ProcessProposal`. As a general rule `ProcessProposal` SHOULD always accept t According to the Tendermint algorithm, a correct process can broadcast at most one precommit message in round *r*, height *h*. -Since, as stated in the [Methods](./abci++_methods_002_draft.md#extendvote) section, `ResponseExtendVote` +Since, as stated in the [Methods](./abci++_methods.md#extendvote) section, `ResponseExtendVote` is only called when Tendermint is about to broadcast a non-`nil` precommit message, a correct process can only produce one vote extension in round *r*, height *h*. @@ -106,9 +108,9 @@ Let *erp* be the vote extension that the Application of a Let *wrp* be the proposed block that *p*'s Tendermint passes to the Application via `RequestExtendVote` in round *r*, height *h*. -* Requirement 8 [`ExtendVote`, `VerifyVoteExtension`, coherence]: For any two correct processes *p* and *q*, if *q* -receives *erp* - from *p* in height *h*, *q*'s Application returns Accept in `ResponseVerifyVoteExtension`. +* Requirement 8 [`ExtendVote`, `VerifyVoteExtension`, coherence]: For any two different correct + processes *p* and *q*, if *q* receives *erp* from *p* in height *h*, *q*'s + Application returns Accept in `ResponseVerifyVoteExtension`. Requirement 8 constrains the creation and handling of vote extensions in a similar way as Requirement 5 constrains the creation and handling of proposed blocks. @@ -170,8 +172,8 @@ Finally, notice that neither `PrepareProposal` nor `ExtendVote` have determinism requirements associated. Indeed, `PrepareProposal` is not required to be deterministic: -* *v'p* may depend on *vp* and *sp,h-1*, but may also depend on other values or operations. -* *vp = vq ⇏ v'p = v'q*. +* *up* may depend on *vp* and *sp,h-1*, but may also depend on other values or operations. +* *vp = vq ⇏ up = uq*. Likewise, `ExtendVote` can also be non-deterministic: @@ -365,12 +367,12 @@ For more information, see Section [State Sync](#state-sync). ### Transaction Results The Application is expected to return a list of -[`ExecTxResult`](./abci%2B%2B_methods_002_draft.md#exectxresult) in -[`ResponseFinalizeBlock`](./abci%2B%2B_methods_002_draft.md#finalizeblock). The list of transaction +[`ExecTxResult`](./abci%2B%2B_methods.md#exectxresult) in +[`ResponseFinalizeBlock`](./abci%2B%2B_methods.md#finalizeblock). The list of transaction results must respect the same order as the list of transactions delivered via -[`RequestFinalizeBlock`](./abci%2B%2B_methods_002_draft.md#finalizeblock). +[`RequestFinalizeBlock`](./abci%2B%2B_methods.md#finalizeblock). This section discusses the fields inside this structure, along with the fields in -[`ResponseCheckTx`](./abci%2B%2B_methods_002_draft.md#checktx), +[`ResponseCheckTx`](./abci%2B%2B_methods.md#checktx), whose semantics are similar. The `Info` and `Log` fields are @@ -471,12 +473,12 @@ events took place during their execution. ### Updating the Validator Set The application may set the validator set during -[`InitChain`](./abci%2B%2B_methods_002_draft.md#initchain), and may update it during -[`FinalizeBlock`](./abci%2B%2B_methods_002_draft.md#finalizeblock) +[`InitChain`](./abci%2B%2B_methods.md#initchain), and may update it during +[`FinalizeBlock`](./abci%2B%2B_methods.md#finalizeblock) (next block execution mode) or -[`PrepareProposal`](./abci%2B%2B_methods_002_draft.md#prepareproposal)/[`ProcessProposal`](./abci%2B%2B_methods_002_draft.md#processproposal) +[`PrepareProposal`](./abci%2B%2B_methods.md#prepareproposal)/[`ProcessProposal`](./abci%2B%2B_methods.md#processproposal) (same block execution mode). In all cases, a structure of type -[`ValidatorUpdate`](./abci%2B%2B_methods_002_draft.md#validatorupdate) is returned. +[`ValidatorUpdate`](./abci%2B%2B_methods.md#validatorupdate) is returned. The `InitChain` method, used to initialize the Application, can return a list of validators. If the list is empty, Tendermint will use the validators loaded from the genesis @@ -510,7 +512,7 @@ Applications must ensure that `MaxTotalVotingPower = MaxInt64 / 8` Note the updates returned after processing the block at height `H` will only take effect -at block `H+2` (see Section [Methods](./abci%2B%2B_methods_002_draft.md)). +at block `H+2` (see Section [Methods](./abci%2B%2B_methods.md)). ### Consensus Parameters @@ -518,10 +520,10 @@ at block `H+2` (see Section [Methods](./abci%2B%2B_methods_002_draft.md)). They enforce certain limits in the blockchain, like the maximum size of blocks, amount of gas used in a block, and the maximum acceptable age of evidence. They can be set in -[`InitChain`](./abci%2B%2B_methods_002_draft.md#initchain), and updated in -[`FinalizeBlock`](./abci%2B%2B_methods_002_draft.md#finalizeblock) +[`InitChain`](./abci%2B%2B_methods.md#initchain), and updated in +[`FinalizeBlock`](./abci%2B%2B_methods.md#finalizeblock) (next block execution mode) or -[`PrepareProposal`](./abci%2B%2B_methods_002_draft.md#prepareproposal)/[`ProcessProposal`](./abci%2B%2B_methods_002_draft.md#processproposal) +[`PrepareProposal`](./abci%2B%2B_methods.md#prepareproposal)/[`ProcessProposal`](./abci%2B%2B_methods.md#processproposal) (same block execution model). These parameters are deterministically set and/or updated by the Application, so all full nodes have the same value at a given height. @@ -672,14 +674,33 @@ node has received all precommits for a block, forgoing the remaining commit time Setting this parameter to `false` (the default) causes Tendermint to wait for the full commit timeout configured in `TimeoutParams.Commit`. +##### ABCIParams.VoteExtensionsEnableHeight + +This parameter is either 0 or a positive height at which vote extensions +become mandatory. If the value is zero (which is the default), vote +extensions are not required. Otherwise, at all heights greater than the +configured height `H` vote extensions must be present (even if empty). +When the configured height `H` is reached, `PrepareProposal` will not +include vote extensions yet, but `ExtendVote` and `VerifyVoteExtension` will +be called. Then, when reaching height `H+1`, `PrepareProposal` will +include the vote extensions from height `H`. For all heights after `H` + +* vote extensions cannot be disabled, +* they are mandatory: all precommit messages sent MUST have an extension + attached. Nevetheless, the application MAY provide 0-length + extensions. + +Must always be set to a future height. Once set to a value different from +0, its value must not be changed. + #### Updating Consensus Parameters The application may set the `ConsensusParams` during -[`InitChain`](./abci%2B%2B_methods_002_draft.md#initchain), +[`InitChain`](./abci%2B%2B_methods.md#initchain), and update them during -[`FinalizeBlock`](./abci%2B%2B_methods_002_draft.md#finalizeblock) +[`FinalizeBlock`](./abci%2B%2B_methods.md#finalizeblock) (next block execution mode) or -[`PrepareProposal`](./abci%2B%2B_methods_002_draft.md#prepareproposal)/[`ProcessProposal`](./abci%2B%2B_methods_002_draft.md#processproposal) +[`PrepareProposal`](./abci%2B%2B_methods.md#prepareproposal)/[`ProcessProposal`](./abci%2B%2B_methods.md#processproposal) (same block execution mode). If the `ConsensusParams` is empty, it will be ignored. Each field that is not empty will be applied in full. For instance, if updating the @@ -885,7 +906,7 @@ truncated block history - users are advised to consider the broader network impl terms of block availability and auditability. This functionality may be added in the future. For details on the specific ABCI calls and types, see the -[methods](abci%2B%2B_methods_002_draft.md) section. +[methods](abci%2B%2B_methods.md) section. #### Taking Snapshots diff --git a/spec/abci++/abci++_basic_concepts.md b/spec/abci++/abci++_basic_concepts.md new file mode 100644 index 000000000..a467b623e --- /dev/null +++ b/spec/abci++/abci++_basic_concepts.md @@ -0,0 +1,468 @@ +--- +order: 1 +title: Overview and basic concepts +--- + +## Outline + +- [ABCI++ vs. ABCI](#abci-vs-abci) +- [Method overview](#method-overview) + - [Consensus/block execution methods](#consensusblock-execution-methods) + - [Mempool methods](#mempool-methods) + - [Info methods](#info-methods) + - [State-sync methods](#state-sync-methods) +- [Next-block execution vs. same-block execution](#next-block-execution-vs-same-block-execution) +- [Tendermint proposal timeout](#tendermint-proposal-timeout) +- [Deterministic State-Machine Replication](#deterministic-state-machine-replication) +- [Events](#events) +- [Evidence](#evidence) +- [Errors](#errors) + +# Overview and basic concepts + +## ABCI++ vs. ABCI + +[↑ Back to Outline](#outline) + +The Application's main role is to execute blocks decided (a.k.a. finalized) by consensus. The +decided blocks are the main consensus's ouput to the (replicated) Application. With ABCI, the +application only interacts with consensus at *decision* time. This restricted mode of interaction +prevents numerous features for the Application, including many scalability improvements that are +now better understood than when ABCI was first written. For example, many ideas proposed to improve +scalability can be boiled down to "make the block proposers do work, so the network does not have +to". This includes optimizations such as transaction level signature aggregation, state transition +proofs, etc. Furthermore, many new security properties cannot be achieved in the current paradigm, +as the Application cannot require validators to do more than execute the transactions contained in +finalized blocks. This includes features such as threshold cryptography, and guaranteed IBC +connection attempts. + +ABCI++ addresses these limitations by allowing the application to intervene at three key places of +consensus execution: (a) at the moment a new proposal is to be created, (b) at the moment a +proposal is to be validated, and (c) at the moment a (precommit) vote is sent/received. The new +interface allows block proposers to perform application-dependent work in a block through the +`PrepareProposal` method (a); validators to perform application-dependent work and checks in a +proposed block through the `ProcessProposal` method (b); and applications to require their validators +do more than just validate blocks through the `ExtendVote` and `VerifyVoteExtension` methods (c). +Furthermore, ABCI++ coalesces {`BeginBlock`, [`DeliverTx`], `EndBlock`} into `FinalizeBlock`, as a +simplified, efficient way to deliver a decided block to the Application. + +## Method overview + +[↑ Back to Outline](#outline) + +Methods can be classified into four categories: *consensus*, *mempool*, *info*, and *state-sync*. + +### Consensus/block execution methods + +The first time a new blockchain is started, Tendermint calls `InitChain`. From then on, method +`FinalizeBlock` is executed upon the decision of each block, resulting in an updated Application +state. During the execution of an instance of consensus, which decides the block for a given +height, and before method `FinalizeBlock` is called, methods `PrepareProposal`, `ProcessProposal`, +`ExtendVote`, and `VerifyVoteExtension` may be called several times. See +[Tendermint's expected behavior](abci++_tmint_expected_behavior.md) for details on the possible +call sequences of these methods. + +- [**InitChain:**](./abci++_methods.md#initchain) This method initializes the blockchain. + Tendermint calls it once upon genesis. + +- [**PrepareProposal:**](./abci++_methods.md#prepareproposal) It allows the block + proposer to perform application-dependent work in a block before proposing it. + This enables, for instance, batch optimizations to a block, which has been empirically + demonstrated to be a key component for improved performance. Method `PrepareProposal` is called + every time Tendermint is about to broadcast a Proposal message, but no previous proposal has + been locked at the Tendermint level. Tendermint gathers outstanding transactions from the + mempool, generates a block header, and uses them to create a block to propose. Then, it calls + `RequestPrepareProposal` with the newly created proposal, called *raw proposal*. The Application + can make changes to the raw proposal, such as modifying transactions, and returns the + (potentially) modified proposal, called *prepared proposal* in the `ResponsePrepareProposal` + call. The logic modifying the raw proposal can be non-deterministic. + +- [**ProcessProposal:**](./abci++_methods.md#processproposal) It allows a validator to + perform application-dependent work in a proposed block. This enables features such as immediate + block execution, and allows the Application to reject invalid blocks. + Tendermint calls it when it receives a proposal and the Tendermint algorithms has not locked on a + value. The Application cannot modify the proposal at this point but can reject it if it is + invalid. If that is the case, Tendermint will prevote `nil` on the proposal, which has + strong liveness implications for Tendermint. As a general rule, the Application + SHOULD accept a prepared proposal passed via `ProcessProposal`, even if a part of + the proposal is invalid (e.g., an invalid transaction); the Application can + ignore the invalid part of the prepared proposal at block execution time. + +- [**ExtendVote:**](./abci++_methods.md#extendvote) It allows applications to force their + validators to do more than just validate within consensus. `ExtendVote` allows applications to + include non-deterministic data, opaque to Tendermint, to precommit messages (the final round of + voting). The data, called *vote extension*, will be broadcast and received together with the + vote it is extending, and will be made available to the Application in the next height, + in the rounds where the local process is the proposer. + Tendermint calls `ExtendVote` when it is about to send a non-`nil` precommit message. + If the Application does not have vote extension information to provide at that time, it returns + a 0-length byte array as its vote extension. + +- [**VerifyVoteExtension:**](./abci++_methods.md#verifyvoteextension) It allows + validators to validate the vote extension data attached to a precommit message. If the validation + fails, the whole precommit message will be deemed invalid and ignored by Tendermint. + This has a negative impact on Tendermint's liveness, i.e., if vote extensions repeatedly cannot be + verified by correct validators, Tendermint may not be able to finalize a block even if sufficiently + many (+2/3) validators send precommit votes for that block. Thus, `VerifyVoteExtension` + should be used with special care. + As a general rule, an Application that detects an invalid vote extension SHOULD + accept it in `ResponseVerifyVoteExtension` and ignore it in its own logic. Tendermint calls it when + a process receives a precommit message with a (possibly empty) vote extension. + +- [**FinalizeBlock:**](./abci++_methods.md#finalizeblock) It delivers a decided block to the + Application. The Application must execute the transactions in the block deterministically and + update its state accordingly. Cryptographic commitments to the block and transaction results, + returned via the corresponding parameters in `ResponseFinalizeBlock`, are included in the header + of the next block. Tendermint calls it when a new block is decided. + +- [**Commit:**](./abci++_methods.md#commit) Instructs the Application to persist its + state. It is a fundamental part of Tendermint's crash-recovery mechanism that ensures the + synchronization between Tendermint and the Applicatin upon recovery. Tendermint calls it just after + having persisted the data returned by `ResponseFinalizeBlock`. The Application can now discard + any state or data except the one resulting from executing the transactions in the decided block. + +### Mempool methods + +- [**CheckTx:**](./abci++_methods.md#checktx) This method allows the Application to validate + transactions. Validation can be stateless (e.g., checking signatures ) or stateful + (e.g., account balances). The type of validation performed is up to the application. If a + transaction passes the validation, then Tendermint adds it to the mempool; otherwise the + transaction is discarded. + Tendermint calls it when it receives a new transaction either coming from an external + user (e.g., a client) or another node. Furthermore, Tendermint can be configured to call + re-`CheckTx` on all outstanding transactions in the mempool after calling `Commit`for a block. + +### Info methods + +- [**Info:**](./abci++_methods.md#info) Used to sync Tendermint with the Application during a + handshake that happens upon recovery, or on startup when state-sync is used. + +- [**Query:**](./abci++_methods.md#query) This method can be used to query the Application for + information about the application state. + +### State-sync methods + +State sync allows new nodes to rapidly bootstrap by discovering, fetching, and applying +state machine (application) snapshots instead of replaying historical blocks. For more details, see the +[state sync documentation](../p2p/messages/state-sync.md). + +New nodes discover and request snapshots from other nodes in the P2P network. +A Tendermint node that receives a request for snapshots from a peer will call +`ListSnapshots` on its Application. The Application returns the list of locally available +snapshots. +Note that the list does not contain the actual snapshots but metadata about them: height at which +the snapshot was taken, application-specific verification data and more (see +[snapshot data type](./abci++_methods.md#snapshot) for more details). After receiving a +list of available snapshots from a peer, the new node can offer any of the snapshots in the list to +its local Application via the `OfferSnapshot` method. The Application can check at this point the +validity of the snapshot metadata. + +Snapshots may be quite large and are thus broken into smaller "chunks" that can be +assembled into the whole snapshot. Once the Application accepts a snapshot and +begins restoring it, Tendermint will fetch snapshot "chunks" from existing nodes. +The node providing "chunks" will fetch them from its local Application using +the `LoadSnapshotChunk` method. + +As the new node receives "chunks" it will apply them sequentially to the local +application with `ApplySnapshotChunk`. When all chunks have been applied, the +Application's `AppHash` is retrieved via an `Info` query. +To ensure that the sync proceeded correctly, Tendermint compares the local Application's `AppHash` +to the `AppHash` stored on the blockchain (verified via +[light client verification](../light-client/verification/README.md)). + +In summary: + +- [**ListSnapshots:**](./abci++_methods.md#listsnapshots) Used by nodes to discover available + snapshots on peers. + +- [**OfferSnapshot:**](./abci++_methods.md#offersnapshot) When a node receives a snapshot from a + peer, Tendermint uses this method to offer the snapshot to the Application. + +- [**LoadSnapshotChunk:**](./abci++_methods.md#loadsnapshotchunk) Used by Tendermint to retrieve + snapshot chunks from the Application to send to peers. + +- [**ApplySnapshotChunk:**](./abci++_methods.md#applysnapshotchunk) Used by Tendermint to hand + snapshot chunks to the Application. + +### Other methods + +Additionally, there is a [**Flush**](./abci++_methods.md#flush) method that is called on every connection, +and an [**Echo**](./abci++_methods.md#echo) method that is used for debugging. + +More details on managing state across connections can be found in the section on +[Managing Application State](./abci%2B%2B_app_requirements.md#managing-the-application-state-and-related-topics). + +## Next-block execution vs. same-block execution + +[↑ Back to Outline](#outline) + +In the original ABCI protocol, the only moment when the Application had access to a +block was after it was decided. This led to a block execution model, called *next-block +execution*, where some fields hashed in a block header refer to the execution of the +previous block, namely: + +- the Merkle root of the Application's state +- the transaction results +- the consensus parameter updates +- the validator updates + +With ABCI++, an Application may be configured to keep using the next-block execution model, by +executing the decided block in `FinalizeBlock`. However, the new methods introduced — +`PrepareProposal` and `ProcessProposal` — disclose the entire proposed block to the +Application, allowing for its immediate exectution. An Application implementing immediate execution +may additionally wish to store certain data resulting from the block's execution in the same block +that has just been executed. This brings about a new execution model, called +*same-block execution*. An Application implementing this execution model, upon receiving a raw +proposal via `RequestPrepareProposal` and potentially modifying its transaction list, fully +executes the resulting prepared proposal as though it was the decided block (immediate execution), +and the results of the block execution are used as follows: + +- The block execution may generate a set of events. The Application should store these events and + return them back to Tendermint during the `FinalizeBlock` call if the block is finally decided. +- The Merkle root resulting from executing the prepared proposal is provided in + `ResponsePrepareProposal` and thus refers to the **current block**. Tendermint + will use it in the prepared proposal's header. +- Likewise, the transaction results from executing the prepared proposal are + provided in `ResponsePrepareProposal` and refer to the transactions in the + **current block**. Tendermint will use them to calculate the results hash + in the prepared proposal's header. +- The consensus parameter updates and validator updates are also provided in + `ResponsePrepareProposal` and reflect the result of the prepared proposal's + execution. They come into force in height H+1 (as opposed to the H+2 rule + in next-block execution model). + +If the Application is configured to keep the next-block execution model, it will not +provide any data in `ResponsePrepareProposal`, other than a potentially modified +transaction list. The Application may nevertheless choose to perform immediate execution even in +next-block execution mode, however same-block execution mode *requires* immediate execution. + +The long term plan is for the execution model to be set in a new boolean parameter *same_block* in +`ConsensusParams`. Once this parameter is introduced, it **must not** be changed once the +blockchain has started, unless the Application developers *really* know what they are doing. +However, modifying `ConsensusParams` structure cannot be done lightly if we are to +preserve blockchain compatibility. Therefore we need an interim solution until +soft upgrades are specified and implemented in Tendermint. This somewhat *unsafe* +solution consists in Tendermint assuming same-block execution if the Application +fills the above mentioned fields in `ResponsePrepareProposal`. + +## Tendermint proposal timeout + +Immediate execution requires the Application to fully execute the prepared block +before returning from `PrepareProposal`, this means that Tendermint cannot make progress +during the block execution. +This stands on Tendermint's critical path: if the Application takes a long time +executing the block, the default value of *TimeoutPropose* might not be sufficient +to accommodate the long block execution time and non-proposer nodes might time +out and prevote `nil`. The proposal, in this case, will probably be rejected and a new round will be necessary. + +The Application is the best suited to provide a value for *TimeoutPropose* so +that the block execution time upon `PrepareProposal` fits well in the propose +timeout interval. Thus, the Application can adapt the value of *TimeoutPropose* at every height via +`TimeoutParams.Propose`, contained in `ConsensusParams`. + +## Deterministic State-Machine Replication + +[↑ Back to Outline](#outline) + +ABCI++ applications must implement deterministic finite-state machines to be +securely replicated by the Tendermint consensus engine. This means block execution +must be strictly deterministic: given the same +ordered set of transactions, all nodes will compute identical responses, for all +successive `FinalizeBlock` calls. This is critical because the +responses are included in the header of the next block, either via a Merkle root +or directly, so all nodes must agree on exactly what they are. + +For this reason, it is recommended that application state is not exposed to any +external user or process except via the ABCI connections to a consensus engine +like Tendermint Core. The Application must only change its state based on input +from block execution (`FinalizeBlock` calls), and not through +any other kind of request. This is the only way to ensure all nodes see the same +transactions and compute the same results. + +Some Applications may choose to implement immediate execution, which entails executing the blocks +that are about to be proposed (via `PrepareProposal`), and those that the Application is asked to +validate (via `ProcessProposal`). However, the state changes caused by processing those +proposed blocks must never replace the previous state until `FinalizeBlock` confirms +the block decided. + +Additionally, vote extensions or the validation thereof (via `ExtendVote` or +`VerifyVoteExtension`) must *never* have side effects on the current state. +They can only be used when their data is provided in a `RequestPrepareProposal` call. + +If there is some non-determinism in the state machine, consensus will eventually +fail as nodes disagree over the correct values for the block header. The +non-determinism must be fixed and the nodes restarted. + +Sources of non-determinism in applications may include: + +- Hardware failures + - Cosmic rays, overheating, etc. +- Node-dependent state + - Random numbers + - Time +- Underspecification + - Library version changes + - Race conditions + - Floating point numbers + - JSON or protobuf serialization + - Iterating through hash-tables/maps/dictionaries +- External Sources + - Filesystem + - Network calls (eg. some external REST API service) + +See [#56](https://github.com/tendermint/abci/issues/56) for the original discussion. + +Note that some methods (`Query, CheckTx, FinalizeBlock`) return non-deterministic data in the form +of `Info` and `Log` fields. The `Log` is intended for the literal output from the Application's +logger, while the `Info` is any additional info that should be returned. These are the only fields +that are not included in block header computations, so we don't need agreement +on them. All other fields in the `Response*` must be strictly deterministic. + +## Events + +[↑ Back to Outline](#outline) + +Method `FinalizeBlock` includes an `events` field at the top level in its +`Response*`, and one `events` field per transaction included in the block. +Applications may respond to this ABCI++ method with an event list for each executed +transaction, and a general event list for the block itself. +Events allow applications to associate metadata with transactions and blocks. +Events returned via `FinalizeBlock` do not impact Tendermint consensus in any way +and instead exist to power subscriptions and queries of Tendermint state. + +An `Event` contains a `type` and a list of `EventAttributes`, which are key-value +string pairs denoting metadata about what happened during the method's (or transaction's) +execution. `Event` values can be used to index transactions and blocks according to what +happened during their execution. + +Each event has a `type` which is meant to categorize the event for a particular +`Response*` or `Tx`. A `Response*` or `Tx` may contain multiple events with duplicate +`type` values, where each distinct entry is meant to categorize attributes for a +particular event. Every key and value in an event's attributes must be UTF-8 +encoded strings along with the event type itself. + +```protobuf +message Event { + string type = 1; + repeated EventAttribute attributes = 2; +} +``` + +The attributes of an `Event` consist of a `key`, a `value`, and an `index` flag. The +index flag notifies the Tendermint indexer to index the attribute. The value of +the `index` flag is non-deterministic and may vary across different nodes in the network. + +```protobuf +message EventAttribute { + bytes key = 1; + bytes value = 2; + bool index = 3; // nondeterministic +} +``` + +Example: + +```go + abci.ResponseFinalizeBlock{ + // ... + Events: []abci.Event{ + { + Type: "validator.provisions", + Attributes: []abci.EventAttribute{ + abci.EventAttribute{Key: []byte("address"), Value: []byte("..."), Index: true}, + abci.EventAttribute{Key: []byte("amount"), Value: []byte("..."), Index: true}, + abci.EventAttribute{Key: []byte("balance"), Value: []byte("..."), Index: true}, + }, + }, + { + Type: "validator.provisions", + Attributes: []abci.EventAttribute{ + abci.EventAttribute{Key: []byte("address"), Value: []byte("..."), Index: true}, + abci.EventAttribute{Key: []byte("amount"), Value: []byte("..."), Index: false}, + abci.EventAttribute{Key: []byte("balance"), Value: []byte("..."), Index: false}, + }, + }, + { + Type: "validator.slashed", + Attributes: []abci.EventAttribute{ + abci.EventAttribute{Key: []byte("address"), Value: []byte("..."), Index: false}, + abci.EventAttribute{Key: []byte("amount"), Value: []byte("..."), Index: true}, + abci.EventAttribute{Key: []byte("reason"), Value: []byte("..."), Index: true}, + }, + }, + // ... + }, +} +``` + +## Evidence + +[↑ Back to Outline](#outline) + +Tendermint's security model relies on the use of evidences of misbehavior. An evidence is an +irrefutable proof of malicious behavior by a network participant. It is the responsibility of +Tendermint to detect such malicious behavior. When malicious behavior is detected, Tendermint +will gossip evidences of misbehavior to other nodes and commit the evidences to +the chain once they are verified by a subset of validators. These evidences will then be +passed on to the Application through ABCI++. It is the responsibility of the +Application to handle evidence of misbehavior and exercise punishment. + +There are two forms of evidence: Duplicate Vote and Light Client Attack. More +information can be found in either [data structures](../core/data_structures.md) +or [accountability](../light-client/accountability/). + +EvidenceType has the following protobuf format: + +```protobuf +enum EvidenceType { + UNKNOWN = 0; + DUPLICATE_VOTE = 1; + LIGHT_CLIENT_ATTACK = 2; +} +``` + +## Errors + +[↑ Back to Outline](#outline) + +The `Query`, and `CheckTx` methods include a `Code` field in their `Response*`. +Field `Code` is meant to contain an application-specific response code. +A response code of `0` indicates no error. Any other response code +indicates to Tendermint that an error occurred. + +These methods also return a `Codespace` string to Tendermint. This field is +used to disambiguate `Code` values returned by different domains of the +Application. The `Codespace` is a namespace for the `Code`. + +Methods `Echo`, `Info`, and `InitChain` do not return errors. +An error in any of these methods represents a critical issue that Tendermint +has no reasonable way to handle. If there is an error in one +of these methods, the Application must crash to ensure that the error is safely +handled by an operator. + +Method `FinalizeBlock` is a special case. It contains a number of +`Code` and `Codespace` fields as part of type `ExecTxResult`. Each of +these codes reports errors related to the transaction it is attached to. +However, `FinalizeBlock` does not return errors at the top level, so the +same considerations on critical issues made for `Echo`, `Info`, and +`InitChain` also apply here. + +The handling of non-zero response codes by Tendermint is described below. + +### `CheckTx` + +When Tendermint receives a `ResponseCheckTx` with a non-zero `Code`, the associated +transaction will not be added to Tendermint's mempool or it will be removed if +it is already included. + +### `ExecTxResult` (as part of `FinalizeBlock`) + +The `ExecTxResult` type delivers transaction results from the Application to Tendermint. When +Tendermint receives a `ResponseFinalizeBlock` containing an `ExecTxResult` with a non-zero `Code`, +the response code is logged. Past `Code` values can be queried by clients. As the transaction was +part of a decided block, the `Code` does not influence Tendermint consensus. + +### `Query` + +When Tendermint receives a `ResponseQuery` with a non-zero `Code`, this code is +returned directly to the client that initiated the query. diff --git a/spec/abci++/abci++_basic_concepts_002_draft.md b/spec/abci++/abci++_basic_concepts_002_draft.md deleted file mode 100644 index a1ad038a5..000000000 --- a/spec/abci++/abci++_basic_concepts_002_draft.md +++ /dev/null @@ -1,404 +0,0 @@ ---- -order: 1 -title: Overview and basic concepts ---- - -## Outline -- [ABCI++ vs. ABCI](#abci-vs-abci) -- [Methods overview](#methods-overview) - - [Consensus methods](#consensus-methods) - - [Mempool methods](#mempool-methods) - - [Info methods](#info-methods) - - [State-sync methods](#state-sync-methods) -- [Next-block execution vs. same-block execution](#next-block-execution-vs-same-block-execution) - - [Tendermint timeouts](#tendermint-timeouts-in-same-block-execution) -- [Determinism](#determinism) -- [Errors](#errors) -- [Events](#events) -- [Evidence](#evidence) - -# Overview and basic concepts - -## ABCI++ vs. ABCI -[↑ Back to Outline](#outline) - -With ABCI, the application can only act at one phase in consensus, immediately after a block has been finalized. This restriction on the application prevents numerous features for the application, including many scalability improvements that are now better understood than when ABCI was first written. For example, many of the scalability proposals can be boiled down to "Make the miner / block proposers / validators do work, so the network does not have to". This includes optimizations such as tx-level signature aggregation, state transition proofs, etc. Furthermore, many new security properties cannot be achieved in the current paradigm, as the application cannot enforce validators to do more than just finalize txs. This includes features such as threshold cryptography, and guaranteed IBC connection attempts. - -ABCI++ overcomes these limitations by allowing the application to intervene at three key places of the block execution. The new interface allows block proposers to perform application-dependent work in a block through the `PrepareProposal` method; validators to perform application-dependent work in a proposed block through the `ProcessProposal` method; and applications to require their validators do more than just validate blocks, e.g., validator guaranteed IBC connection attempts, through the `ExtendVote` and `VerifyVoteExtension` methods. Furthermore, ABCI++ renames {`BeginBlock`, [`DeliverTx`], `EndBlock`} to `FinalizeBlock`, as a simplified way to deliver a decided block to the Application. - -## Methods overview -[↑ Back to Outline](#outline) - -Methods can be classified into four categories: consensus, mempool, info, and state-sync. - -### Consensus/block execution methods - -The first time a new blockchain is started, Tendermint calls -`InitChain`. From then on, method `FinalizeBlock` is executed at the end of each -block, resulting in an updated Application state. -During consensus execution of a block height, before method `FinalizeBlock` is -called, methods `PrepareProposal`, `ProcessProposal`, `ExtendVote`, and -`VerifyVoteExtension` may be called several times. -See [Tendermint's expected behavior](abci++_tmint_expected_behavior_002_draft.md) -for details on the possible call sequences of these methods. - -* [**InitChain:**](./abci++_methods_002_draft.md#initchain) This method initializes the blockchain. Tendermint calls it once upon genesis. - -* [**PrepareProposal:**](./abci++_methods_002_draft.md#prepareproposal) It allows the block proposer to perform application-dependent work in a block before using it as its proposal. This enables, for instance, batch optimizations to a block, which has been empirically demonstrated to be a key component for scaling. Method `PrepareProposal` is called every time Tendermint is about to send -a proposal message, but no previous proposal has been locked at Tendermint level. -Tendermint gathers outstanding transactions from the mempool, generates a block header, and uses -them to create a block to propose. Then, it calls `RequestPrepareProposal` -with the newly created proposal, called _raw proposal_. The Application can -make changes to the raw proposal, such as modifying transactions, and returns -the (potentially) modified proposal, called _prepared proposal_ in the -`Response*` call. The logic modifying the raw proposal can be non-deterministic. - -* [**ProcessProposal:**](./abci++_methods_002_draft.md#processproposal) It allows a validator to perform application-dependent work in a proposed block. This enables features such as allowing validators to reject a block according to whether the state machine deems it valid, and changing the block execution pipeline. Tendermint calls it when it receives a proposal and it is not locked on a block. The Application cannot -modify the proposal at this point but can reject it if it realizes it is invalid. -If that is the case, Tendermint will prevote `nil` on the proposal, which has -strong liveness implications for Tendermint. As a general rule, the Application -SHOULD accept a prepared proposal passed via `ProcessProposal`, even if a part of -the proposal is invalid (e.g., an invalid transaction); the Application can -ignore the invalid part of the prepared proposal at block execution time. - -* [**ExtendVote:**](./abci++_methods_002_draft.md#extendvote) It allows applications to force their validators to do more than just validate within consensus. `ExtendVote` allows applications to include non-deterministic data, opaque to Tendermint, to precommit messages (the final round of voting). -The data, called _vote extension_, will also be made available to the -application in the next height, along with the vote it is extending, in the rounds -where the local process is the proposer. -If the Application does not have vote extension information to provide, it returns a 0-length byte array as its vote extension. -Tendermint calls `ExtendVote` when is about to send a non-`nil` precommit message. - -* [**VerifyVoteExtension:**](./abci++_methods_002_draft.md#verifyvoteextension) It allows validators to validate the vote extension data attached to a precommit message. If the validation fails, the precommit message will be deemed invalid and ignored -by Tendermint. This has a negative impact on Tendermint's liveness, i.e., if vote extensions repeatedly cannot be verified by correct validators, Tendermint may not be able to finalize a block even if sufficiently many (+2/3) of the validators send precommit votes for that block. Thus, `VerifyVoteExtension` should be used with special care. -As a general rule, an Application that detects an invalid vote extension SHOULD -accept it in `ResponseVerifyVoteExtension` and ignore it in its own logic. Tendermint calls it when -a process receives a precommit message with a (possibly empty) vote extension. - -* [**FinalizeBlock:**](./abci++_methods_002_draft.md#finalizeblock) It delivers a decided block to the Application. The Application must execute the transactions in the block in order and update its state accordingly. Cryptographic commitments to the block and transaction results, via the corresponding -parameters in `ResponseFinalizeBlock`, are included in the header of the next block. Tendermint calls it when a new block is decided. - -### Mempool methods - -* [**CheckTx:**](./abci++_methods_002_draft.md#checktx) This method allows the Application to validate transactions against its current state, e.g., checking signatures and account balances. If a transaction passes the validation, then tendermint adds it to its local mempool, discarding it otherwise. Tendermint calls it when it receives a new transaction either coming from an external user or another node. Furthermore, Tendermint can be configured to re-call `CheckTx` on any decided transaction (after `FinalizeBlock`). - -### Info methods - -* [**Info:**](./abci++_methods_002_draft.md#info) Used to sync Tendermint with the Application during a handshake that happens on startup. - -* [**Query:**](./abci++_methods_002_draft.md#query) Clients can use this method to query the Application for information about the application state. - -### State-sync methods - -State sync allows new nodes to rapidly bootstrap by discovering, fetching, and applying -state machine snapshots instead of replaying historical blocks. For more details, see the -[state sync section](../p2p/messages/state-sync.md). - -New nodes will discover and request snapshots from other nodes in the P2P network. -A Tendermint node that receives a request for snapshots from a peer will call -`ListSnapshots` on its Application. The Application returns the list of locally avaiable snapshots. -Note that the list does not contain the actual snapshot but metadata about it: height at which the snapshot was taken, application-specific verification data and more (see [snapshot data type](./abci++_methods_002_draft.md#snapshot) for more details). After receiving a list of available snapshots from a peer, the new node can offer any of the snapshots in the list to its local Application via the `OfferSnapshot` method. The Application can check at this point the validity of the snapshot metadata. - -Snapshots may be quite large and are thus broken into smaller "chunks" that can be -assembled into the whole snapshot. Once the Application accepts a snapshot and -begins restoring it, Tendermint will fetch snapshot "chunks" from existing nodes. -The node providing "chunks" will fetch them from its local Application using -the `LoadSnapshotChunk` method. - -As the new node receives "chunks" it will apply them sequentially to the local -application with `ApplySnapshotChunk`. When all chunks have been applied, the -Application's `AppHash` is retrieved via an `Info` query. -To ensure that the sync proceeded correctly, Tendermint compares the local Application's `AppHash` to the `AppHash` stored on the blockchain (verified via -[light client verification](../light-client/verification/README.md)). - -In summary: - -* [**ListSnapshots:**](./abci++_methods_002_draft.md#listsnapshots) Used by nodes to discover available snapshots on peers. - -* [**LoadSnapshotChunk:**](./abci++_methods_002_draft.md#loadsnapshotchunk) Used by Tendermint to retrieve snapshot chunks from the application to send to peers. - -* [**OfferSnapshot:**](./abci++_methods_002_draft.md#offersnapshot) When a node receives a snapshot from a peer, Tendermint uses this method to offer the snapshot to the Application. - -* [**ApplySnapshotChunk:**](./abci++_methods_002_draft.md#applysnapshotchunk) Used by Tendermint to hand snapshot chunks to the Application. - -### Other methods - -Additionally, there is a [**Flush**](./abci++_methods_002_draft.md#flush) method that is called on every connection, -and an [**Echo**](./abci++_methods_002_draft.md#echo) method that is just for debugging. - -More details on managing state across connections can be found in the section on -[ABCI Applications](../abci/apps.md). - -## Next-block execution vs. same-block execution -[↑ Back to Outline](#outline) - -In the original ABCI protocol, the only moment when the Application had access to a -block was after it was decided. This led to a block execution model, called _next-block -execution_, where some fields hashed in a block header refer to the execution of the -previous block, namely: - -* the Merkle root of the Application's state -* the transaction results -* the consensus parameter updates -* the validator updates - -With ABCI++, an Application may decide to keep using the next-block execution model, by doing all its processing in `FinalizeBlock`; -however the new methods introduced, `PrepareProposal` and `ProcessProposal` allow -for a new execution model, called _same-block execution_. An Application implementing -this execution model, upon receiving a raw proposal via `RequestPrepareProposal` -and potentially modifying its transaction list, -fully executes the resulting prepared proposal as though it was the decided block. -The results of the block execution are used as follows: - -* The block execution may generate a set of events. The Application should store these events and return them back to Tendermint during the `FinalizeBlock` call if the block is finally decided. -* The Merkle root resulting from executing the prepared proposal is provided in - `ResponsePrepareProposal` and thus refers to the **current block**. Tendermint - will use it in the prepared proposal's header. -* likewise, the transaction results from executing the prepared proposal are - provided in `ResponsePrepareProposal` and refer to the transactions in the - **current block**. Tendermint will use them to calculate the results hash - in the prepared proposal's header. -* The consensus parameter updates and validator updates are also provided in - `ResponsePrepareProposal` and reflect the result of the prepared proposal's - execution. They come into force in height H+1 (as opposed to the H+2 rule - in next-block execution model). - -If the Application decides to keep the next-block execution model, it will not -provide any data in `ResponsePrepareProposal`, other than an optionally modified -transaction list. - -In the long term, the execution model will be set in a new boolean parameter -*same_block* in `ConsensusParams`. -It **must not** be changed once the blockchain has started unless the Application -developers _really_ know what they are doing. -However, modifying `ConsensusParams` structure cannot be done lightly if we are to -preserve blockchain compatibility. Therefore we need an interim solution until -soft upgrades are specified and implemented in Tendermint. This somewhat _unsafe_ -solution consists in Tendermint assuming same-block execution if the Application -fills the above mentioned fields in `ResponsePrepareProposal`. - -### Tendermint timeouts in same-block execution - -The new same-block execution mode requires the Application to fully execute the -prepared block at `PrepareProposal` time. This execution is synchronous, so -Tendermint cannot make progress until the Application returns from `PrepareProposal`. -This stands on Tendermint's critical path: if the Application takes a long time -executing the block, the default value of _TimeoutPropose_ might not be sufficient -to accommodate the long block execution time and non-proposer processes might time -out and prevote `nil`, thus starting a further round unnecessarily. - -The Application is the best suited to provide a value for _TimeoutPropose_ so -that the block execution time upon `PrepareProposal` fits well in the propose -timeout interval. - -Currently, the Application can override the value of _TimeoutPropose_ via the -`config.toml` file. In the future, `ConsensusParams` will have an extra field -with the current _TimeoutPropose_ value so that the Application can adapt it at every height. - -## Determinism -[↑ Back to Outline](#outline) - -ABCI++ applications must implement deterministic finite-state machines to be -securely replicated by the Tendermint consensus engine. This means block execution -over the Consensus Connection must be strictly deterministic: given the same -ordered set of transactions, all nodes will compute identical responses, for all -successive `FinalizeBlock` calls. This is critical because the -responses are included in the header of the next block, either via a Merkle root -or directly, so all nodes must agree on exactly what they are. - -For this reason, it is recommended that application state is not exposed to any -external user or process except via the ABCI connections to a consensus engine -like Tendermint Core. The Application must only change its state based on input -from block execution (`FinalizeBlock` calls), and not through -any other kind of request. This is the only way to ensure all nodes see the same -transactions and compute the same results. - -Some Applications may choose to execute the blocks that are about to be proposed -(via `PrepareProposal`), or those that the Application is asked to validate -(via `ProcessProposal`). However, the state changes caused by processing those -proposed blocks must never replace the previous state until `FinalizeBlock` confirms -the block decided. - -Additionally, vote extensions or the validation thereof (via `ExtendVote` or -`VerifyVoteExtension`) must _never_ have side effects on the current state. -They can only be used when their data is provided in a `RequestPrepareProposal` call. - -If there is some non-determinism in the state machine, consensus will eventually -fail as nodes disagree over the correct values for the block header. The -non-determinism must be fixed and the nodes restarted. - -Sources of non-determinism in applications may include: - -* Hardware failures - * Cosmic rays, overheating, etc. -* Node-dependent state - * Random numbers - * Time -* Underspecification - * Library version changes - * Race conditions - * Floating point numbers - * JSON or protobuf serialization - * Iterating through hash-tables/maps/dictionaries -* External Sources - * Filesystem - * Network calls (eg. some external REST API service) - -See [#56](https://github.com/tendermint/abci/issues/56) for original discussion. - -Note that some methods (`Query, CheckTx, FinalizeBlock`) return -explicitly non-deterministic data in the form of `Info` and `Log` fields. The `Log` is -intended for the literal output from the Application's logger, while the -`Info` is any additional info that should be returned. These are the only fields -that are not included in block header computations, so we don't need agreement -on them. All other fields in the `Response*` must be strictly deterministic. - -## Errors -[↑ Back to Outline](#outline) - -The `Query`, and `CheckTx` methods include a `Code` field in their `Response*`. -The `Code` field is also included in type `TxResult`, used by -method `FinalizeBlock`'s `Response*`. -Field `Code` is meant to contain an application-specific response code. -A response code of `0` indicates no error. Any other response code -indicates to Tendermint that an error occurred. - -These methods also return a `Codespace` string to Tendermint. This field is -used to disambiguate `Code` values returned by different domains of the -Application. The `Codespace` is a namespace for the `Code`. - -Methods `Echo`, `Info`, and `InitChain` do not return errors. -An error in any of these methods represents a critical issue that Tendermint -has no reasonable way to handle. If there is an error in one -of these methods, the Application must crash to ensure that the error is safely -handled by an operator. - -Method `FinalizeBlock` is a special case. It contains a number of -`Code` and `Codespace` fields as part of type `TxResult`. Each of -these codes reports errors related to the transaction it is attached to. -However, `FinalizeBlock` does not return errors at the top level, so the -same considerations on critical issues made for `Echo`, `Info`, and -`InitChain` also apply here. - -The handling of non-zero response codes by Tendermint is described below - -### `CheckTx` - -When Tendermint receives a `ResponseCheckTx` with a non-zero `Code`, the associated -transaction will not be added to Tendermint's mempool or it will be removed if -it is already included. - -### `TxResult` (as part of `FinalizeBlock`) - -The `TxResult` type delivers transactions from Tendermint to the Application. -When Tendermint receives a `ResponseFinalizeBlock` containing a `TxResult` -with a non-zero `Code`, the response code is logged. -The transaction was already included in a block, so the `Code` does not influence -Tendermint consensus. - -### `Query` - -When Tendermint receives a `ResponseQuery` with a non-zero `Code`, this code is -returned directly to the client that initiated the query. - -## Events -[↑ Back to Outline](#outline) - -Method `CheckTx` includes an `Events` field in its `Response*`. -Method `FinalizeBlock` includes an `Events` field at the top level in its -`Response*`, and one `events` field per transaction included in the block. -Applications may respond to these ABCI++ methods with a set of events. -Events allow applications to associate metadata about ABCI++ method execution with the -transactions and blocks this metadata relates to. -Events returned via these ABCI++ methods do not impact Tendermint consensus in any way -and instead exist to power subscriptions and queries of Tendermint state. - -An `Event` contains a `type` and a list of `EventAttributes`, which are key-value -string pairs denoting metadata about what happened during the method's (or transaction's) -execution. `Event` values can be used to index transactions and blocks according to what -happened during their execution. - -Each event has a `type` which is meant to categorize the event for a particular -`Response*` or `Tx`. A `Response*` or `Tx` may contain multiple events with duplicate -`type` values, where each distinct entry is meant to categorize attributes for a -particular event. Every key and value in an event's attributes must be UTF-8 -encoded strings along with the event type itself. - -```protobuf -message Event { - string type = 1; - repeated EventAttribute attributes = 2; -} -``` - -The attributes of an `Event` consist of a `key`, a `value`, and an `index` flag. The -index flag notifies the Tendermint indexer to index the attribute. The value of -the `index` flag is non-deterministic and may vary across different nodes in the network. - -```protobuf -message EventAttribute { - bytes key = 1; - bytes value = 2; - bool index = 3; // nondeterministic -} -``` - -Example: - -```go - abci.ResponseCheckTx{ - // ... - Events: []abci.Event{ - { - Type: "validator.provisions", - Attributes: []abci.EventAttribute{ - abci.EventAttribute{Key: []byte("address"), Value: []byte("..."), Index: true}, - abci.EventAttribute{Key: []byte("amount"), Value: []byte("..."), Index: true}, - abci.EventAttribute{Key: []byte("balance"), Value: []byte("..."), Index: true}, - }, - }, - { - Type: "validator.provisions", - Attributes: []abci.EventAttribute{ - abci.EventAttribute{Key: []byte("address"), Value: []byte("..."), Index: true}, - abci.EventAttribute{Key: []byte("amount"), Value: []byte("..."), Index: false}, - abci.EventAttribute{Key: []byte("balance"), Value: []byte("..."), Index: false}, - }, - }, - { - Type: "validator.slashed", - Attributes: []abci.EventAttribute{ - abci.EventAttribute{Key: []byte("address"), Value: []byte("..."), Index: false}, - abci.EventAttribute{Key: []byte("amount"), Value: []byte("..."), Index: true}, - abci.EventAttribute{Key: []byte("reason"), Value: []byte("..."), Index: true}, - }, - }, - // ... - }, -} -``` - -## Evidence -[↑ Back to Outline](#outline) - -Tendermint's security model relies on the use of "evidence". Evidence is proof of -malicious behavior by a network participant. It is the responsibility of Tendermint -to detect such malicious behavior. When malicious behavior is detected, Tendermint -will gossip evidence of the behavior to other nodes and commit the evidence to -the chain once it is verified by all validators. This evidence will then be -passed on to the Application through ABCI++. It is the responsibility of the -Application to handle the evidence and exercise punishment. - -EvidenceType has the following protobuf format: - -```protobuf -enum EvidenceType { - UNKNOWN = 0; - DUPLICATE_VOTE = 1; - LIGHT_CLIENT_ATTACK = 2; -} -``` - -There are two forms of evidence: Duplicate Vote and Light Client Attack. More -information can be found in either [data structures](../core/data_structures.md) -or [accountability](../light-client/accountability/) - diff --git a/spec/abci++/abci++_client_server_002_draft.md b/spec/abci++/abci++_client_server.md similarity index 91% rename from spec/abci++/abci++_client_server_002_draft.md rename to spec/abci++/abci++_client_server.md index f26ee8cd5..652652dc9 100644 --- a/spec/abci++/abci++_client_server_002_draft.md +++ b/spec/abci++/abci++_client_server.md @@ -9,10 +9,10 @@ This section is for those looking to implement their own ABCI Server, perhaps in a new programming language. You are expected to have read all previous sections of ABCI++ specification, namely -[Basic Concepts](./abci%2B%2B_basic_concepts_002_draft.md), -[Methods](./abci%2B%2B_methods_002_draft.md), -[Application Requirements](./abci%2B%2B_app_requirements_002_draft.md), and -[Expected Behavior](./abci%2B%2B_tmint_expected_behavior_002_draft.md). +[Basic Concepts](./abci%2B%2B_basic_concepts.md), +[Methods](./abci%2B%2B_methods.md), +[Application Requirements](./abci%2B%2B_app_requirements.md), and +[Expected Behavior](./abci%2B%2B_tmint_expected_behavior.md). ## Message Protocol and Synchrony @@ -25,7 +25,7 @@ or custom protobuf types. For more details on protobuf, see the [documentation](https://developers.google.com/protocol-buffers/docs/overview). As of v0.36 requests are synchronous. For each of ABCI++'s four connections (see -[Connections](./abci%2B%2B_app_requirements_002_draft.md)), when Tendermint issues a request to the +[Connections](./abci%2B%2B_app_requirements.md)), when Tendermint issues a request to the Application, it will wait for the response before continuing execution. As a side effect, requests and responses are ordered for each connection, but not necessarily across connections. diff --git a/spec/abci++/abci++_methods_002_draft.md b/spec/abci++/abci++_methods.md similarity index 71% rename from spec/abci++/abci++_methods_002_draft.md rename to spec/abci++/abci++_methods.md index 4eb1bb295..9d33652dd 100644 --- a/spec/abci++/abci++_methods_002_draft.md +++ b/spec/abci++/abci++_methods.md @@ -38,22 +38,21 @@ title: Methods * **Response**: - | Name | Type | Description | Field Number | - |---------------------|--------|--------------------------------------------------|--------------| - | data | string | Some arbitrary information | 1 | - | version | string | The application software semantic version | 2 | - | app_version | uint64 | The application protocol version | 3 | - | last_block_height | int64 | Latest block for which the app has called Commit | 4 | - | last_block_app_hash | bytes | Latest result of Commit | 5 | + | Name | Type | Description | Field Number | + |---------------------|--------|-----------------------------------------------------|--------------| + | data | string | Some arbitrary information | 1 | + | version | string | The application software semantic version | 2 | + | app_version | uint64 | The application protocol version | 3 | + | last_block_height | int64 | Latest height for which the app persisted its state | 4 | + | last_block_app_hash | bytes | Latest AppHash returned by `FinalizeBlock` | 5 | * **Usage**: * Return information about the application state. * Used to sync Tendermint with the application during a handshake - that happens on startup. + that happens on startup or on recovery. * The returned `app_version` will be included in the Header of every block. * Tendermint expects `last_block_app_hash` and `last_block_height` to - be updated during `Commit`, ensuring that `Commit` is never - called twice for the same block height. + be updated during `FinalizeBlock` and persisted during `Commit`. > Note: Semantic version is a reference to [semantic versioning](https://semver.org/). Semantic versions in info will be displayed as X.X.x. @@ -61,22 +60,22 @@ title: Methods * **Request**: - | Name | Type | Description | Field Number | - |------------------|--------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------|--------------| - | time | [google.protobuf.Timestamp](https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#google.protobuf.Timestamp) | Genesis time | 1 | - | chain_id | string | ID of the blockchain. | 2 | - | consensus_params | [ConsensusParams](#consensusparams) | Initial consensus-critical parameters. | 3 | - | validators | repeated [ValidatorUpdate](#validatorupdate) | Initial genesis validators, sorted by voting power. | 4 | - | app_state_bytes | bytes | Serialized initial application state. JSON bytes. | 5 | - | initial_height | int64 | Height of the initial block (typically `1`). | 6 | + | Name | Type | Description | Field Number | + |------------------|-------------------------------------------------|-----------------------------------------------------|--------------| + | time | [google.protobuf.Timestamp][protobuf-timestamp] | Genesis time | 1 | + | chain_id | string | ID of the blockchain. | 2 | + | consensus_params | [ConsensusParams](#consensusparams) | Initial consensus-critical parameters. | 3 | + | validators | repeated [ValidatorUpdate](#validatorupdate) | Initial genesis validators, sorted by voting power. | 4 | + | app_state_bytes | bytes | Serialized initial application state. JSON bytes. | 5 | + | initial_height | int64 | Height of the initial block (typically `1`). | 6 | * **Response**: - | Name | Type | Description | Field Number | - |------------------|----------------------------------------------|-------------------------------------------------|--------------| + | Name | Type | Description | Field Number | + |------------------|----------------------------------------------|--------------------------------------------------|--------------| | consensus_params | [ConsensusParams](#consensusparams) | Initial consensus-critical parameters (optional) | 1 | - | validators | repeated [ValidatorUpdate](#validatorupdate) | Initial validator set (optional). | 2 | - | app_hash | bytes | Initial application hash. | 3 | + | validators | repeated [ValidatorUpdate](#validatorupdate) | Initial validator set (optional). | 2 | + | app_hash | bytes | Initial application hash. | 3 | * **Usage**: * Called once upon genesis. @@ -84,10 +83,10 @@ title: Methods * If `ResponseInitChain.Validators` is not empty, it will be the initial validator set (regardless of what is in `RequestInitChain.Validators`). * This allows the app to decide if it wants to accept the initial validator - set proposed by tendermint (ie. in the genesis file), or if it wants to use + set proposed by Tendermint (ie. in the genesis file), or if it wants to use a different one (perhaps computed based on some application specific information in the genesis file). - * Both `ResponseInitChain.Validators` and `ResponseInitChain.Validators` are [ValidatorUpdate](#validatorupdate) structs. + * Both `RequestInitChain.Validators` and `ResponseInitChain.Validators` are [ValidatorUpdate](#validatorupdate) structs. So, technically, they both are _updating_ the set of validators from the empty set. ### Query @@ -97,7 +96,7 @@ title: Methods | Name | Type | Description | Field Number | |--------|--------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------| | data | bytes | Raw query bytes. Can be used with or in lieu of Path. | 1 | - | path | string | Path field of the request URI. Can be used with or in lieu of `data`. Apps MUST interpret `/store` as a query by key on the underlying store. The key SHOULD be specified in the `data` field. Apps SHOULD allow queries over specific types like `/accounts/...` or `/votes/...` | 2 | + | path | string | Path field of the request URI. Can be used with or in lieu of `data`. Apps MUST interpret `/store` as a query by key on the underlying store. The key SHOULD be specified in the `data` field. Apps SHOULD allow queries over specific types like `/accounts/...` or `/votes/...` | 2 | | height | int64 | The block height for which you want the query (default=0 returns data for the latest committed block). Note that this is the height of the block containing the application's Merkle root hash, which represents the state as it was after committing the block at Height-1 | 3 | | prove | bool | Return Merkle proof with response if possible | 4 | @@ -145,15 +144,40 @@ title: Methods * Technically optional - not involved in processing blocks. * Guardian of the mempool: every node runs `CheckTx` before letting a - transaction into its local mempool. + transaction into its local mempool. * The transaction may come from an external user or another node * `CheckTx` validates the transaction against the current state of the application, - for example, checking signatures and account balances, but does not apply any - of the state changes described in the transaction. - not running code in a virtual machine. - * Transactions where `ResponseCheckTx.Code != 0` will be rejected - they will not be broadcast to - other nodes or included in a proposal block. - * Tendermint attributes no other value to the response code + for example, checking signatures and account balances, but does not apply any + of the state changes described in the transaction. + * Transactions where `ResponseCheckTx.Code != 0` will be rejected - they will not be broadcast + to other nodes or included in a proposal block. + Tendermint attributes no other value to the response code. + +### Commit + +#### Parameters and Types + +* **Request**: + + | Name | Type | Description | Field Number | + |--------|-------|-------------|--------------| + + Commit signals the application to persist application state. It takes no parameters. + +* **Response**: + + | Name | Type | Description | Field Number | + |---------------|-------|------------------------------------------------------------------------|--------------| + | retain_height | int64 | Blocks below this height may be removed. Defaults to `0` (retain all). | 3 | + +* **Usage**: + + * Signal the Application to persist the application state. + Application is expected to persist its state at the end of this call, before calling `ResponseCommit`. + * Use `ResponseCommit.retain_height` with caution! If all nodes in the network remove historical + blocks then this data is permanently lost, and no new nodes will be able to join the network and + bootstrap. Historical blocks may also be required for other purposes, e.g. auditing, replay of + non-persisted heights, light client verification, and so on. ### ListSnapshots @@ -269,7 +293,7 @@ title: Methods `Snapshot.Metadata` and/or incrementally verifying contents against `AppHash`. * When all chunks have been accepted, Tendermint will make an ABCI `Info` call to verify that `LastBlockAppHash` and `LastBlockHeight` matches the expected values, and record the - `AppVersion` in the node state. It then switches to fast sync or consensus and joins the + `AppVersion` in the node state. It then switches to block sync or consensus and joins the network. * If Tendermint is unable to retrieve the next chunk after some time (e.g. because no suitable peers are available), it will reject the snapshot and try a different one via `OfferSnapshot`. @@ -283,16 +307,16 @@ title: Methods * **Request**: - | Name | Type | Description | Field Number | - |-------------------------|---------------------------------------------|------------------------------------------------------------------------------------------------------------------|--------------| - | max_tx_bytes | int64 | Currently configured maximum size in bytes taken by the modified transactions. | 1 | - | txs | repeated bytes | Preliminary list of transactions that have been picked as part of the block to propose. | 2 | - | local_last_commit | [ExtendedCommitInfo](#extendedcommitinfo) | Info about the last commit, obtained locally from Tendermint's data structures. | 3 | - | byzantine_validators | repeated [Misbehavior](#misbehavior) | List of information about validators that acted incorrectly. | 4 | - | height | int64 | The height of the block that will be proposed. | 5 | - | time | [google.protobuf.Timestamp](https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#google.protobuf.Timestamp) | Timestamp of the block that that will be proposed. | 6 | - | next_validators_hash | bytes | Merkle root of the next validator set. | 7 | - | proposer_address | bytes | [Address](../core/data_structures.md#address) of the validator that is creating the proposal. | 8 | + | Name | Type | Description | Field Number | + |----------------------|-------------------------------------------------|-----------------------------------------------------------------------------------------------|--------------| + | max_tx_bytes | int64 | Currently configured maximum size in bytes taken by the modified transactions. | 1 | + | txs | repeated bytes | Preliminary list of transactions that have been picked as part of the block to propose. | 2 | + | local_last_commit | [ExtendedCommitInfo](#extendedcommitinfo) | Info about the last commit, obtained locally from Tendermint's data structures. | 3 | + | misbehavior | repeated [Misbehavior](#misbehavior) | List of information about validators that misbehaved. | 4 | + | height | int64 | The height of the block that will be proposed. | 5 | + | time | [google.protobuf.Timestamp][protobuf-timestamp] | Timestamp of the block that that will be proposed. | 6 | + | next_validators_hash | bytes | Merkle root of the next validator set. | 7 | + | proposer_address | bytes | [Address](../core/data_structures.md#address) of the validator that is creating the proposal. | 8 | * **Response**: @@ -302,92 +326,128 @@ title: Methods | app_hash | bytes | The Merkle root hash of the application state. | 3 | | tx_results | repeated [ExecTxResult](#exectxresult) | List of structures containing the data resulting from executing the transactions | 4 | | validator_updates | repeated [ValidatorUpdate](#validatorupdate) | Changes to validator set (set voting power to 0 to remove). | 5 | - | consensus_param_updates | [ConsensusParams](#consensusparams) | Changes to consensus-critical gas, size, and other parameters. | 6 | + | consensus_param_updates | [ConsensusParams](#consensusparams) | Changes to gas, size, and other consensus-related parameters. | 6 | * **Usage**: - * The first six parameters of `RequestPrepareProposal` are the same as `RequestProcessProposal` + * `RequestPrepareProposal`'s parameters `txs`, `misbehavior`, `height`, `time`, + `next_validators_hash`, and `proposer_address` are the same as in `RequestProcessProposal` and `RequestFinalizeBlock`. - * The height and time values match the values from the header of the proposed block. - * `RequestPrepareProposal` contains a preliminary set of transactions `txs` that Tendermint considers to be a good block proposal, called _raw proposal_. The Application can modify this set via `ResponsePrepareProposal.tx_records` (see [TxRecord](#txrecord)). - * The Application _can_ reorder, remove or add transactions to the raw proposal. Let `tx` be a transaction in `txs`: - * If the Application considers that `tx` should not be proposed in this block, e.g., there are other transactions with higher priority, then it should not include it in `tx_records`. In this case, Tendermint won't remove `tx` from the mempool. The Application should be extra-careful, as abusing this feature may cause transactions to stay forever in the mempool. - * If the Application considers that a `tx` should not be included in the proposal and removed from the mempool, then the Application should include it in `tx_records` and _mark_ it as `REMOVED`. In this case, Tendermint will remove `tx` from the mempool. - * If the Application wants to add a new transaction, then the Application should include it in `tx_records` and _mark_ it as `ADD`. In this case, Tendermint will add it to the mempool. - * The Application should be aware that removing and adding transactions may compromise _traceability_. - > Consider the following example: the Application transforms a client-submitted transaction `t1` into a second transaction `t2`, i.e., the Application asks Tendermint to remove `t1` and add `t2` to the mempool. If a client wants to eventually check what happened to `t1`, it will discover that `t_1` is not in the mempool or in a committed block, getting the wrong idea that `t_1` did not make it into a block. Note that `t_2` _will be_ in a committed block, but unless the Application tracks this information, no component will be aware of it. Thus, if the Application wants traceability, it is its responsability to support it. For instance, the Application could attach to a transformed transaction a list with the hashes of the transactions it derives from. - * Tendermint MAY include a list of transactions in `RequestPrepareProposal.txs` whose total size in bytes exceeds `RequestPrepareProposal.max_tx_bytes`. - Therefore, if the size of `RequestPrepareProposal.txs` is greater than `RequestPrepareProposal.max_tx_bytes`, the Application MUST make sure that the - `RequestPrepareProposal.max_tx_bytes` limit is respected by those transaction records returned in `ResponsePrepareProposal.tx_records` that are marked as `UNMODIFIED` or `ADDED`. - * In same-block execution mode, the Application must provide values for `ResponsePrepareProposal.app_hash`, - `ResponsePrepareProposal.tx_results`, `ResponsePrepareProposal.validator_updates`, and + * `RequestPrepareProposal.local_last_commit` is a set of the precommit votes that allowed the + decision of the previous block, together with their corresponding vote extensions. + * The `height`, `time`, and `proposer_address` values match the values from the header of the + proposed block. + * `RequestPrepareProposal` contains a preliminary set of transactions `txs` that Tendermint + retrieved from the mempool, called _raw proposal_. The Application can modify this + set via `ResponsePrepareProposal.tx_records` (see [TxRecord](#txrecord)). + * The Application _can_ modify the raw proposal: it can reorder, remove or add transactions. + Let `tx` be a transaction in `txs`: + * If the Application considers that `tx` should not be proposed in this block, e.g., + there are other transactions with higher priority, then it should not include it in + `tx_records`. In this case, Tendermint will not remove `tx` from the mempool. The + Application should be extra-careful, as abusing this feature may cause transactions + to stay much longer than needed in the mempool. + * If the Application considers that `tx` should not be included in the proposal and + removed from the mempool, then the Application should include it in `tx_records` and + _mark_ it as `REMOVED`. In this case, Tendermint will remove `tx` from the mempool. + * If the Application wants to add a new transaction to the proposed block, then the + Application includes it in `tx_records` and _marks_ it as `ADDED`. In this case, Tendermint + will also add the transaction to the mempool. + * The Application should be aware that removing and adding transactions may compromise + _traceability_. + > Consider the following example: the Application transforms a client-submitted + transaction `t1` into a second transaction `t2`, i.e., the Application asks Tendermint + to remove `t1` and add `t2` to the mempool. If a client wants to eventually check what + happened to `t1`, it will discover that `t1` is neither in the mempool nor in a + committed block, getting the wrong idea that `t1` did not make it into a block. Note + that `t2` _will be_ in a committed block, but unless the Application tracks this + information, no component will be aware of it. Thus, if the Application wants + traceability, it is its responsability to support it. For instance, the Application + could attach to a transformed transaction a list with the hashes of the transactions it + derives from. + * Tendermint MAY include a list of transactions in `RequestPrepareProposal.txs` whose total + size in bytes exceeds `RequestPrepareProposal.max_tx_bytes`. + Therefore, if the size of `RequestPrepareProposal.txs` is greater than + `RequestPrepareProposal.max_tx_bytes`, the Application MUST remove transactions to ensure + that the `RequestPrepareProposal.max_tx_bytes` limit is respected by those transaction + records returned in `ResponsePrepareProposal.tx_records` that are marked as `UNMODIFIED` or + `ADDED`. + * In same-block execution mode, the Application must provide values for + `ResponsePrepareProposal.app_hash`, `ResponsePrepareProposal.tx_results`, + `ResponsePrepareProposal.validator_updates`, and `ResponsePrepareProposal.consensus_param_updates`, as a result of fully executing the block. * The values for `ResponsePrepareProposal.validator_updates`, or `ResponsePrepareProposal.consensus_param_updates` may be empty. In this case, Tendermint will keep the current values. * `ResponsePrepareProposal.validator_updates`, triggered by block `H`, affect validation for blocks `H+1`, and `H+2`. Heights following a validator update are affected in the following way: - * `H`: `NextValidatorsHash` includes the new `validator_updates` value. - * `H+1`: The validator set change takes effect and `ValidatorsHash` is updated. - * `H+2`: `local_last_commit` now includes the altered validator set. + * Height `H`: `NextValidatorsHash` includes the new `validator_updates` value. + * Height `H+1`: The validator set change takes effect and `ValidatorsHash` is updated. + * Height `H+2`: `*_last_commit` fields in `PrepareProposal`, `ProcessProposal`, and + `FinalizeBlock` now include the altered validator set. * `ResponseFinalizeBlock.consensus_param_updates` returned for block `H` apply to the consensus params for block `H+1` even if the change is agreed in block `H`. For more information on the consensus parameters, - see the [application spec entry on consensus parameters](../abci/apps.md#consensus-parameters). - * It is the responsibility of the Application to set the right value for _TimeoutPropose_ so that + see the [consensus parameters](./abci%2B%2B_app_requirements.md#consensus-parameters) + section. + * It is the Application's responsibility to set the right value for _TimeoutPropose_ so that the (synchronous) execution of the block does not cause other processes to prevote `nil` because their propose timeout goes off. - * In next-block execution mode, Tendermint will ignore parameters `ResponsePrepareProposal.tx_results`, + * In next-block execution mode, Tendermint will ignore parameters + `ResponsePrepareProposal.app_hash`, `ResponsePrepareProposal.tx_results`, `ResponsePrepareProposal.validator_updates`, and `ResponsePrepareProposal.consensus_param_updates`. - * As a result of executing the prepared proposal, the Application may produce header events or transaction events. + * As a result of executing the prepared proposal, the Application may produce block events or transaction events. The Application must keep those events until a block is decided and then pass them on to Tendermint via `ResponseFinalizeBlock`. - * Likewise, in next-block execution mode, the Application must keep all responses to executing transactions - until it can call `ResponseFinalizeBlock`. + * Likewise, in next-block execution mode, the Application must keep all responses to executing + transactions until it can call `ResponseFinalizeBlock`. * As a sanity check, Tendermint will check the returned parameters for validity if the Application modified them. In particular, `ResponsePrepareProposal.tx_records` will be deemed invalid if * There is a duplicate transaction in the list. - * A new or modified transaction is marked as `UNMODIFIED` or `REMOVED`. - * An unmodified transaction is marked as `ADDED`. + * A new transaction is marked as `UNMODIFIED` or `REMOVED`. + * An existing transaction is marked as `ADDED`. * A transaction is marked as `UNKNOWN`. - * If Tendermint fails to validate the `ResponsePrepareProposal`, Tendermint will assume the application is faulty and crash. + * If Tendermint fails to validate the `ResponsePrepareProposal`, Tendermint will assume the + Application is faulty and crash. * The implementation of `PrepareProposal` can be non-deterministic. -#### When does Tendermint call it? +#### When does Tendermint call `PrepareProposal`? When a validator _p_ enters Tendermint consensus round _r_, height _h_, in which _p_ is the proposer, and _p_'s _validValue_ is `nil`: -1. _p_'s Tendermint collects outstanding transactions from the mempool - * The transactions will be collected in order of priority - * Let $C$ the list of currently collected transactions - * The collection stops when any of the following conditions are met - * the mempool is empty - * the total size of transactions $\in C$ is greater than or equal to `consensusParams.block.max_bytes` - * the sum of `GasWanted` field of transactions $\in C$ is greater than or equal to - `consensusParams.block.max_gas` +1. Tendermint collects outstanding transactions from _p_'s mempool + * the transactions will be collected in order of priority * _p_'s Tendermint creates a block header. -2. _p_'s Tendermint calls `RequestPrepareProposal` with the newly generated block. - The call is synchronous: Tendermint's execution will block until the Application returns from the call. -3. The Application checks the block (hashes, transactions, commit info, misbehavior). Besides, - * in same-block execution mode, the Application can (and should) provide `ResponsePrepareProposal.app_hash`, - `ResponsePrepareProposal.validator_updates`, or +2. _p_'s Tendermint calls `RequestPrepareProposal` with the newly generated block, the local + commit of the previous height (with vote extensions), and any outstanding evidence of + misbehavior. The call is synchronous: Tendermint's execution will block until the Application + returns from the call. +3. The Application uses the information received (transactions, commit info, misbehavior, time) to + (potentially) modify the proposal. + * in same-block execution mode, the Application fully executes the block and provides values + for `ResponsePrepareProposal.app_hash`, `ResponsePrepareProposal.tx_results`, + `ResponsePrepareProposal.validator_updates`, and `ResponsePrepareProposal.consensus_param_updates`. - * in "next-block execution" mode, _p_'s Tendermint will ignore the values for `ResponsePrepareProposal.app_hash`, - `ResponsePrepareProposal.validator_updates`, and `ResponsePrepareProposal.consensus_param_updates`. - * in both modes, the Application can manipulate transactions + * in next-block execution mode, _p_'s Tendermint will ignore the values for + `ResponsePrepareProposal.app_hash`, `ResponsePrepareProposal.tx_results`, + `ResponsePrepareProposal.validator_updates`, and + `ResponsePrepareProposal.consensus_param_updates`. + * in both modes, the Application can manipulate transactions: * leave transactions untouched - `TxAction = UNMODIFIED` - * add new transactions directly to the proposal - `TxAction = ADDED` - * remove transactions (invalid) from the proposal and from the mempool - `TxAction = REMOVED` + * add new transactions (not present initially) to the proposal - `TxAction = ADDED` + * remove (invalid) transactions from the proposal and from the mempool - `TxAction = REMOVED` * remove transactions from the proposal but not from the mempool (effectively _delaying_ them) - the - Application removes the transaction from the list - * modify transactions (e.g. aggregate them) - `TxAction = ADDED` followed by `TxAction = REMOVED`. As explained above, this compromises client traceability, unless it is implemented at the Application level. + Application does not include the transaction in `ResponsePrepareProposal.tx_records` + * modify transactions (e.g. aggregate them) - `TxAction = ADDED` followed by + `TxAction = REMOVED`. As explained above, this compromises client traceability, unless + it is implemented at the Application level. * reorder transactions - the Application reorders transactions in the list -4. If the block is modified, the Application sets `ResponsePrepareProposal.modified` to true, - and includes the modified block in the return parameters (see the rules in section _Usage_). - The Application returns from the call. +4. The Application includes the transaction list (whether modified or not) in the return parameters + (see the rules in section _Usage_), and returns from the call. 5. _p_'s Tendermint uses the (possibly) modified block as _p_'s proposal in round _r_, height _h_. -Note that, if _p_ has a non-`nil` _validValue_, Tendermint will use it as proposal and will not call `RequestPrepareProposal`. +Note that, if _p_ has a non-`nil` _validValue_ in round _r_, height _h_, Tendermint will use it as +proposal and will not call `RequestPrepareProposal`. ### ProcessProposal @@ -395,16 +455,16 @@ Note that, if _p_ has a non-`nil` _validValue_, Tendermint will use it as propos * **Request**: - | Name | Type | Description | Field Number | - |----------------------|---------------------------------------------|----------------------------------------------------------------------------------------------------------------|--------------| - | txs | repeated bytes | List of transactions that have been picked as part of the proposed block. | 1 | - | proposed_last_commit | [CommitInfo](#commitinfo) | Info about the last commit, obtained from the information in the proposed block. | 2 | - | byzantine_validators | repeated [Misbehavior](#misbehavior) | List of information about validators that acted incorrectly. | 3 | - | hash | bytes | The block header's hash of the proposed block. | 4 | - | height | int64 | The height of the proposed block. | 5 | - | time | [google.protobuf.Timestamp](https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#google.protobuf.Timestamp) | Timestamp included in the proposed block. | 6 | - | next_validators_hash | bytes | Merkle root of the next validator set. | 7 | - | proposer_address | bytes | [Address](../core/data_structures.md#address) of the validator that created the proposal. | 8 | + | Name | Type | Description | Field Number | + |----------------------|-------------------------------------------------|-------------------------------------------------------------------------------------------|--------------| + | txs | repeated bytes | List of transactions of the proposed block. | 1 | + | proposed_last_commit | [CommitInfo](#commitinfo) | Info about the last commit, obtained from the information in the proposed block. | 2 | + | misbehavior | repeated [Misbehavior](#misbehavior) | List of information about validators that misbehaved. | 3 | + | hash | bytes | The hash of the proposed block. | 4 | + | height | int64 | The height of the proposed block. | 5 | + | time | [google.protobuf.Timestamp][protobuf-timestamp] | Timestamp of the proposed block. | 6 | + | next_validators_hash | bytes | Merkle root of the next validator set. | 7 | + | proposer_address | bytes | [Address](../core/data_structures.md#address) of the validator that created the proposal. | 8 | * **Response**: @@ -414,14 +474,19 @@ Note that, if _p_ has a non-`nil` _validValue_, Tendermint will use it as propos | app_hash | bytes | The Merkle root hash of the application state. | 2 | | tx_results | repeated [ExecTxResult](#exectxresult) | List of structures containing the data resulting from executing the transactions. | 3 | | validator_updates | repeated [ValidatorUpdate](#validatorupdate) | Changes to validator set (set voting power to 0 to remove). | 4 | - | consensus_param_updates | [ConsensusParams](#consensusparams) | Changes to consensus-critical gas, size, and other parameters. | 5 | + | consensus_param_updates | [ConsensusParams](#consensusparams) | Changes to gas, size, and other consensus-related parameters. | 5 | * **Usage**: - * Contains fields from the proposed block. - * The Application may fully execute the block as though it was handling `RequestFinalizeBlock`. - However, any resulting state changes must be kept as _candidate state_, - and the Application should be ready to backtrack/discard it in case the decided block is different. - * The height and timestamp values match the values from the header of the proposed block. + * Contains all information on the proposed block needed to fully execute it. + * The Application may fully execute the block as though it was handling + `RequestFinalizeBlock`. + * However, any resulting state changes must be kept as _candidate state_, + and the Application should be ready to discard it in case another block is decided. + * `RequestProcessProposal` is also called at the proposer of a round. The reason for this is to + inform the Application of the block header's hash, which cannot be done at `PrepareProposal` + time. In this case, the call to `RequestProcessProposal` occurs right after the call to + `RequestPrepareProposal`. + * The height and time values match the values from the header of the proposed block. * If `ResponseProcessProposal.status` is `REJECT`, Tendermint assumes the proposal received is not valid. * In same-block execution mode, the Application is required to fully execute the block and provide values @@ -430,35 +495,39 @@ Note that, if _p_ has a non-`nil` _validValue_, Tendermint will use it as propos so that Tendermint can then verify the hashes in the block's header are correct. If the hashes mismatch, Tendermint will reject the block even if `ResponseProcessProposal.status` was set to `ACCEPT`. - * In next-block execution mode, the Application should *not* provide values for parameters + * In next-block execution mode, the Application should _not_ provide values for parameters `ResponseProcessProposal.app_hash`, `ResponseProcessProposal.tx_results`, `ResponseProcessProposal.validator_updates`, and `ResponseProcessProposal.consensus_param_updates`. * The implementation of `ProcessProposal` MUST be deterministic. Moreover, the value of `ResponseProcessProposal.status` MUST **exclusively** depend on the parameters passed in the call to `RequestProcessProposal`, and the last committed Application state - (see [Requirements](abci++_app_requirements_002_draft.md) section). + (see [Requirements](./abci++_app_requirements.md) section). * Moreover, application implementors SHOULD always set `ResponseProcessProposal.status` to `ACCEPT`, unless they _really_ know what the potential liveness implications of returning `REJECT` are. -#### When does Tendermint call it? +#### When does Tendermint call `ProcessProposal`? When a validator _p_ enters Tendermint consensus round _r_, height _h_, in which _q_ is the proposer (possibly _p_ = _q_): 1. _p_ sets up timer `ProposeTimeout`. 2. If _p_ is the proposer, _p_ executes steps 1-6 in [PrepareProposal](#prepareproposal). -3. Upon reception of Proposal message (which contains the header) for round _r_, height _h_ from _q_, _p_'s Tendermint verifies the block header. -4. Upon reception of Proposal message, along with all the block parts, for round _r_, height _h_ from _q_, _p_'s Tendermint follows its algorithm - to check whether it should prevote for the block just received, or `nil` -5. If Tendermint should prevote for the block just received +3. Upon reception of Proposal message (which contains the header) for round _r_, height _h_ from + _q_, _p_'s Tendermint verifies the block header. +4. Upon reception of Proposal message, along with all the block parts, for round _r_, height _h_ + from _q_, _p_'s Tendermint follows its algorithm to check whether it should prevote for the + proposed block, or `nil`. +5. If Tendermint should prevote for the proposed block: 1. Tendermint calls `RequestProcessProposal` with the block. The call is synchronous. - 2. The Application checks/processes the proposed block, which is read-only, and returns true (_accept_) or false (_reject_) in `ResponseProcessProposal.accept`. + 2. The Application checks/processes the proposed block, which is read-only, and returns + `ACCEPT` or `REJECT` in the `ResponseProcessProposal.status` field. * The Application, depending on its needs, may call `ResponseProcessProposal` - * either after it has completely processed the block (the simpler case), - * or immediately (after doing some basic checks), and process the block asynchronously. In this case the Application will - not be able to reject the block, or force prevote/precommit `nil` afterwards. + * either after it has completely processed the block (immediate execution), + * or after doing some basic checks, and process the block asynchronously. In this case the + Application will not be able to reject the block, or force prevote/precommit `nil` + afterwards. 3. If the returned value is - * _accept_, Tendermint prevotes on this proposal for round _r_, height _h_. - * _reject_, Tendermint prevotes `nil`. + * `ACCEPT`: Tendermint prevotes on this proposal for round _r_, height _h_. + * `REJECT`: Tendermint prevotes `nil`. ### ExtendVote @@ -473,20 +542,21 @@ When a validator _p_ enters Tendermint consensus round _r_, height _h_, in which * **Response**: - | Name | Type | Description | Field Number | - |-------------------|-------|-----------------------------------------------|--------------| - | vote_extension | bytes | Optional information signed by by Tendermint. | 1 | + | Name | Type | Description | Field Number | + |-------------------|-------|---------------------------------------------------------|--------------| + | vote_extension | bytes | Information signed by by Tendermint. Can have 0 length. | 1 | * **Usage**: - * `ResponseExtendVote.vote_extension` is optional information that, if present, will be signed by Tendermint and - attached to the Precommit message. - * `RequestExtendVote.hash` corresponds to the hash of a proposed block that was made available to the application - in a previous call to `ProcessProposal` or `PrepareProposal` for the current height. + * `ResponseExtendVote.vote_extension` is application-generated information that will be signed + by Tendermint and attached to the Precommit message. + * The Application may choose to use an empty vote extension (0 length). + * `RequestExtendVote.hash` corresponds to the hash of a proposed block that was made available + to the Application in a previous call to `ProcessProposal` for the current height. * `ResponseExtendVote.vote_extension` will only be attached to a non-`nil` Precommit message. If Tendermint is to precommit `nil`, it will not call `RequestExtendVote`. * The Application logic that creates the extension can be non-deterministic. -#### When does Tendermint call it? +#### When does Tendermint call `ExtendVote`? When a validator _p_ is in Tendermint consensus state _prevote_ of round _r_, height _h_, in which _q_ is the proposer; and _p_ has received @@ -497,7 +567,7 @@ then _p_'s Tendermint locks _v_ and sends a Precommit message in the following 1. _p_'s Tendermint sets _lockedValue_ and _validValue_ to _v_, and sets _lockedRound_ and _validRound_ to _r_ 2. _p_'s Tendermint calls `RequestExtendVote` with _id(v)_ (`RequestExtendVote.hash`). The call is synchronous. -3. The Application optionally returns an array of bytes, `ResponseExtendVote.extension`, which is not interpreted by Tendermint. +3. The Application returns an array of bytes, `ResponseExtendVote.extension`, which is not interpreted by Tendermint. 4. _p_'s Tendermint includes `ResponseExtendVote.extension` in a field of type [CanonicalVoteExtension](#canonicalvoteextension), it then populates the other fields in [CanonicalVoteExtension](#canonicalvoteextension), and signs the populated data structure. @@ -516,12 +586,12 @@ a [CanonicalVoteExtension](#canonicalvoteextension) field in the `precommit nil` * **Request**: - | Name | Type | Description | Field Number | - |-------------------|-------|------------------------------------------------------------------------------------------|--------------| - | hash | bytes | The header hash of the propsed block that the vote extension refers to. | 1 | - | validator_address | bytes | [Address](../core/data_structures.md#address) of the validator that signed the extension | 2 | - | height | int64 | Height of the block (for sanity check). | 3 | - | vote_extension | bytes | Application-specific information signed by Tendermint. Can have 0 length | 4 | + | Name | Type | Description | Field Number | + |-------------------|-------|-------------------------------------------------------------------------------------------|--------------| + | hash | bytes | The hash of the proposed block that the vote extension refers to. | 1 | + | validator_address | bytes | [Address](../core/data_structures.md#address) of the validator that signed the extension. | 2 | + | height | int64 | Height of the block (for sanity check). | 3 | + | vote_extension | bytes | Application-specific information signed by Tendermint. Can have 0 length. | 4 | * **Response**: @@ -530,33 +600,38 @@ a [CanonicalVoteExtension](#canonicalvoteextension) field in the `precommit nil` | status | [VerifyStatus](#verifystatus) | `enum` signaling if the application accepts the vote extension | 1 | * **Usage**: - * `RequestVerifyVoteExtension.vote_extension` can be an empty byte array. The Application's interpretation of it should be + * `RequestVerifyVoteExtension.vote_extension` can be an empty byte array. The Application's + interpretation of it should be that the Application running at the process that sent the vote chose not to extend it. Tendermint will always call `RequestVerifyVoteExtension`, even for 0 length vote extensions. + * `RequestVerifyVoteExtension` is not called for precommit votes sent by the local process. + * `RequestVerifyVoteExtension.hash` refers to a proposed block. There is not guarantee that + this proposed block has previously been exposed to the Application via `ProcessProposal`. * If `ResponseVerifyVoteExtension.status` is `REJECT`, Tendermint will reject the whole received vote. - See the [Requirements](abci++_app_requirements_002_draft.md) section to understand the potential + See the [Requirements](./abci++_app_requirements.md) section to understand the potential liveness implications of this. * The implementation of `VerifyVoteExtension` MUST be deterministic. Moreover, the value of `ResponseVerifyVoteExtension.status` MUST **exclusively** depend on the parameters passed in the call to `RequestVerifyVoteExtension`, and the last committed Application state - (see [Requirements](abci++_app_requirements_002_draft.md) section). + (see [Requirements](./abci++_app_requirements.md) section). * Moreover, application implementers SHOULD always set `ResponseVerifyVoteExtension.status` to `ACCEPT`, unless they _really_ know what the potential liveness implications of returning `REJECT` are. -#### When does Tendermint call it? +#### When does Tendermint call `VerifyVoteExtension`? -When a validator _p_ is in Tendermint consensus round _r_, height _h_, state _prevote_ (**TODO** discuss: I think I must remove the state -from this condition, but not sure), and _p_ receives a Precommit message for round _r_, height _h_ from _q_: +When a node _p_ is in Tendermint consensus round _r_, height _h_, and _p_ receives a Precommit +message for round _r_, height _h_ from validator _q_ (_q_ ≠ _p_): -1. If the Precommit message does not contain a vote extension with a valid signature, Tendermint discards the message as invalid. +1. If the Precommit message does not contain a vote extension with a valid signature, Tendermint + discards the Precommit message as invalid. * a 0-length vote extension is valid as long as its accompanying signature is also valid. 2. Else, _p_'s Tendermint calls `RequestVerifyVoteExtension`. -3. The Application returns _accept_ or _reject_ via `ResponseVerifyVoteExtension.status`. +3. The Application returns `ACCEPT` or `REJECT` via `ResponseVerifyVoteExtension.status`. 4. If the Application returns - * _accept_, _p_'s Tendermint will keep the received vote, together with its corresponding + * `ACCEPT`, _p_'s Tendermint will keep the received vote, together with its corresponding vote extension in its internal data structures. It will be used to populate the [ExtendedCommitInfo](#extendedcommitinfo) structure in calls to `RequestPrepareProposal`, in rounds of height _h + 1_ where _p_ is the proposer. - * _reject_, _p_'s Tendermint will deem the Precommit message invalid and discard it. + * `REJECT`, _p_'s Tendermint will deem the Precommit message invalid and discard it. ### FinalizeBlock @@ -564,38 +639,38 @@ from this condition, but not sure), and _p_ receives a Precommit message for rou * **Request**: - | Name | Type | Description | Field Number | - |----------------------|---------------------------------------------|------------------------------------------------------------------------------------------|--------------| - | txs | repeated bytes | List of transactions committed as part of the block. | 1 | - | decided_last_commit | [CommitInfo](#commitinfo) | Info about the last commit, obtained from the block that was just decided. | 2 | - | byzantine_validators | repeated [Misbehavior](#misbehavior) | List of information about validators that acted incorrectly. | 3 | - | hash | bytes | The block header's hash. Present for convenience (can be derived from the block header). | 4 | - | height | int64 | The height of the finalized block. | 5 | - | time | [google.protobuf.Timestamp](https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#google.protobuf.Timestamp) | Timestamp included in the finalized block. | 6 | - | next_validators_hash | bytes | Merkle root of the next validator set. | 7 | - | proposer_address | bytes | [Address](../core/data_structures.md#address) of the validator that created the proposal.| 8 | + | Name | Type | Description | Field Number | + |----------------------|-------------------------------------------------|-------------------------------------------------------------------------------------------|--------------| + | txs | repeated bytes | List of transactions committed as part of the block. | 1 | + | decided_last_commit | [CommitInfo](#commitinfo) | Info about the last commit, obtained from the block that was just decided. | 2 | + | misbehavior | repeated [Misbehavior](#misbehavior) | List of information about validators that misbehaved. | 3 | + | hash | bytes | The block's hash. | 4 | + | height | int64 | The height of the finalized block. | 5 | + | time | [google.protobuf.Timestamp][protobuf-timestamp] | Timestamp of the finalized block. | 6 | + | next_validators_hash | bytes | Merkle root of the next validator set. | 7 | + | proposer_address | bytes | [Address](../core/data_structures.md#address) of the validator that created the proposal. | 8 | * **Response**: | Name | Type | Description | Field Number | |-------------------------|-------------------------------------------------------------|----------------------------------------------------------------------------------|--------------| - | events | repeated [Event](abci++_basic_concepts_002_draft.md#events) | Type & Key-Value events for indexing | 1 | + | events | repeated [Event](abci++_basic_concepts.md#events) | Type & Key-Value events for indexing | 1 | | tx_results | repeated [ExecTxResult](#exectxresult) | List of structures containing the data resulting from executing the transactions | 2 | | validator_updates | repeated [ValidatorUpdate](#validatorupdate) | Changes to validator set (set voting power to 0 to remove). | 3 | - | consensus_param_updates | [ConsensusParams](#consensusparams) | Changes to consensus-critical gas, size, and other parameters. | 4 | + | consensus_param_updates | [ConsensusParams](#consensusparams) | Changes to gas, size, and other consensus-related parameters. | 4 | | app_hash | bytes | The Merkle root hash of the application state. | 5 | - | retain_height | int64 | Blocks below this height may be removed. Defaults to `0` (retain all). | 6 | * **Usage**: * Contains the fields of the newly decided block. * This method is equivalent to the call sequence `BeginBlock`, [`DeliverTx`], - `EndBlock`, `Commit` in the previous version of ABCI. - * The height and timestamp values match the values from the header of the proposed block. - * The Application can use `RequestFinalizeBlock.decided_last_commit` and `RequestFinalizeBlock.byzantine_validators` + and `EndBlock` in the previous version of ABCI. + * The height and time values match the values from the header of the proposed block. + * The Application can use `RequestFinalizeBlock.decided_last_commit` and `RequestFinalizeBlock.misbehavior` to determine rewards and punishments for the validators. - * The application must execute the transactions in full, in the order they appear in `RequestFinalizeBlock.txs`, - before returning control to Tendermint. Alternatively, it can commit the candidate state corresponding to the same block - previously executed via `PrepareProposal` or `ProcessProposal`. + * The Application executes the transactions in `RequestFinalizeBlock.txs` deterministically, + according to the rules set up by the Application, before returning control to Tendermint. + Alternatively, it can commit the candidate state corresponding to the same block previously + executed via `PrepareProposal` or `ProcessProposal`. * `ResponseFinalizeBlock.tx_results[i].Code == 0` only if the _i_-th transaction is fully valid. * In next-block execution mode, the Application must provide values for `ResponseFinalizeBlock.app_hash`, `ResponseFinalizeBlock.tx_results`, `ResponseFinalizeBlock.validator_updates`, and @@ -605,16 +680,18 @@ from this condition, but not sure), and _p_ receives a Precommit message for rou the current values. * `ResponseFinalizeBlock.validator_updates`, triggered by block `H`, affect validation for blocks `H+1`, `H+2`, and `H+3`. Heights following a validator update are affected in the following way: - - Height `H+1`: `NextValidatorsHash` includes the new `validator_updates` value. - - Height `H+2`: The validator set change takes effect and `ValidatorsHash` is updated. - - Height `H+3`: `decided_last_commit` now includes the altered validator set. + * Height `H+1`: `NextValidatorsHash` includes the new `validator_updates` value. + * Height `H+2`: The validator set change takes effect and `ValidatorsHash` is updated. + * Height `H+3`: `*_last_commit` fields in `PrepareProposal`, `ProcessProposal`, and + `FinalizeBlock` now include the altered validator set. * `ResponseFinalizeBlock.consensus_param_updates` returned for block `H` apply to the consensus params for block `H+1`. For more information on the consensus parameters, - see the [application spec entry on consensus parameters](../abci/apps.md#consensus-parameters). + see the [consensus parameters](./abci%2B%2B_app_requirements.md#consensus-parameters) + section. + * In same-block execution mode, Tendermint will log an error and ignore values for `ResponseFinalizeBlock.app_hash`, `ResponseFinalizeBlock.tx_results`, `ResponseFinalizeBlock.validator_updates`, and `ResponsePrepareProposal.consensus_param_updates`, as those must have been provided by `PrepareProposal`. - * Application is expected to persist its state at the end of this call, before calling `ResponseFinalizeBlock`. * `ResponseFinalizeBlock.app_hash` contains an (optional) Merkle root hash of the application state. * `ResponseFinalizeBlock.app_hash` is included * [in next-block execution mode] as the `Header.AppHash` in the next block. @@ -626,11 +703,7 @@ from this condition, but not sure), and _p_ receives a Precommit message for rou of `RequestFinalizeBlock` and the previous committed state. * Later calls to `Query` can return proofs about the application state anchored in this Merkle root hash. - * Use `ResponseFinalizeBlock.retain_height` with caution! If all nodes in the network remove historical - blocks then this data is permanently lost, and no new nodes will be able to join the network and - bootstrap. Historical blocks may also be required for other purposes, e.g. auditing, replay of - non-persisted heights, light client verification, and so on. - * Just as `ProcessProposal`, the implementation of `FinalizeBlock` MUST be deterministic, since it is + * The implementation of `FinalizeBlock` MUST be deterministic, since it is making the Application's state evolve in the context of state machine replication. * Currently, Tendermint will fill up all fields in `RequestFinalizeBlock`, even if they were already passed on to the Application via `RequestPrepareProposal` or `RequestProcessProposal`. @@ -638,9 +711,9 @@ from this condition, but not sure), and _p_ receives a Precommit message for rou (rather than executing the whole block). In this case the Application disregards all parameters in `RequestFinalizeBlock` except `RequestFinalizeBlock.hash`. -#### When does Tendermint call it? +#### When does Tendermint call `FinalizeBlock`? -When a validator _p_ is in Tendermint consensus height _h_, and _p_ receives +When a node _p_ is in Tendermint consensus height _h_, and _p_ receives * the Proposal message with block _v_ for a round _r_, along with all its block parts, from _q_, which is the proposer of round _r_, height _h_, @@ -649,16 +722,19 @@ When a validator _p_ is in Tendermint consensus height _h_, and _p_ receives then _p_'s Tendermint decides block _v_ and finalizes consensus for height _h_ in the following way -1. _p_'s Tendermint persists _v_ as decision for height _h_. -2. _p_'s Tendermint locks the mempool -- no calls to checkTx on new transactions. -3. _p_'s Tendermint calls `RequestFinalizeBlock` with _id(v)_. The call is synchronous. -4. _p_'s Application processes block _v_, received in a previous call to `RequestProcessProposal`. -5. _p_'s Application commits and persists the state resulting from processing the block. -6. _p_'s Application calculates and returns the _AppHash_, along with an array of arrays of bytes representing the output of each of the transactions -7. _p_'s Tendermint hashes the array of transaction outputs and stores it in _ResultHash_ -8. _p_'s Tendermint persists _AppHash_ and _ResultHash_ -9. _p_'s Tendermint unlocks the mempool -- newly received transactions can now be checked. -10. _p_'s starts consensus for a new height _h+1_, round 0 +1. _p_'s Tendermint persists _v_ as the decision for height _h_. +2. _p_'s Tendermint calls `RequestFinalizeBlock` with _v_'s data. The call is synchronous. +3. _p_'s Application executes block _v_. +4. _p_'s Application calculates and returns the _AppHash_, along with a list containing + the outputs of each of the transactions executed. +5. _p_'s Tendermint hashes all the transaction outputs and stores it in _ResultHash_. +6. _p_'s Tendermint persists the transaction outputs, _AppHash_, and _ResultsHash_. +7. _p_'s Tendermint locks the mempool — no calls to `CheckTx` on new transactions. +8. _p_'s Tendermint calls `RequestCommit` to instruct the Application to persist its state. +9. _p_'s Tendermint, optionally, re-checks all outstanding transactions in the mempool + against the newly persisted Application state. +10. _p_'s Tendermint unlocks the mempool — newly received transactions can now be checked. +11. _p_'s starts consensus for height _h+1_, round 0 ## Data Types existing in ABCI @@ -696,13 +772,13 @@ Most of the data structures used in ABCI are shared [common data structures](../ * **Fields**: - | Name | Type | Description | Field Number | - |--------------------|--------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------|--------------| - | type | [MisbehaviorType](#misbehaviortype) | Type of the misbehavior. An enum of possible misbehaviors. | 1 | - | validator | [Validator](#validator) | The offending validator | 2 | - | height | int64 | Height when the offense occurred | 3 | - | time | [google.protobuf.Timestamp](https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#google.protobuf.Timestamp) | Time of the block that was committed at the height that the offense occurred | 4 | - | total_voting_power | int64 | Total voting power of the validator set at height `Height` | 5 | + | Name | Type | Description | Field Number | + |--------------------|-------------------------------------------------|------------------------------------------------------------------------------|--------------| + | type | [MisbehaviorType](#misbehaviortype) | Type of the misbehavior. An enum of possible misbehaviors. | 1 | + | validator | [Validator](#validator) | The offending validator | 2 | + | height | int64 | Height when the offense occurred | 3 | + | time | [google.protobuf.Timestamp][protobuf-timestamp] | Timestamp of the block that was committed at height `height` | 4 | + | total_voting_power | int64 | Total voting power of the validator set at height `height` | 5 | #### MisbehaviorType @@ -823,7 +899,7 @@ Most of the data structures used in ABCI are shared [common data structures](../ | info | string | Additional information. **May be non-deterministic.** | 4 | | gas_wanted | int64 | Amount of gas requested for transaction. | 5 | | gas_used | int64 | Amount of gas consumed by transaction. | 6 | - | events | repeated [Event](abci++_basic_concepts_002_draft.md#events) | Type & Key-Value events for indexing transactions (e.g. by account). | 7 | + | events | repeated [Event](abci++_basic_concepts.md#events) | Type & Key-Value events for indexing transactions (e.g. by account). | 7 | | codespace | string | Namespace for the `code`. | 8 | ### TxAction @@ -842,7 +918,7 @@ enum TxAction { * If `Action` is `UNMODIFIED`, Tendermint includes the transaction in the proposal. Nothing to do on the mempool. * If `Action` is `ADDED`, Tendermint includes the transaction in the proposal. The transaction is _not_ added to the mempool. * If `Action` is `REMOVED`, Tendermint excludes the transaction from the proposal. The transaction is also removed from the mempool if it exists, - similar to `CheckTx` returning _false_. + similar to `CheckTx` returning an error code. ### TxRecord @@ -905,3 +981,5 @@ enum VerifyStatus { * Tendermint is to sign the whole data structure and attach it to a Precommit message * Upon reception, Tendermint validates the sender's signature and sanity-checks the values of `height`, `round`, and `chain_id`. Then it sends `extension` to the Application via `RequestVerifyVoteExtension` for verification. + +[protobuf-timestamp]: https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#google.protobuf.Timestamp diff --git a/spec/abci++/abci++_tmint_expected_behavior_002_draft.md b/spec/abci++/abci++_tmint_expected_behavior.md similarity index 66% rename from spec/abci++/abci++_tmint_expected_behavior_002_draft.md rename to spec/abci++/abci++_tmint_expected_behavior.md index 778689450..8df5e6c84 100644 --- a/spec/abci++/abci++_tmint_expected_behavior_002_draft.md +++ b/spec/abci++/abci++_tmint_expected_behavior.md @@ -11,16 +11,20 @@ This section describes what the Application can expect from Tendermint. The Tendermint consensus algorithm is designed to protect safety under any network conditions, as long as less than 1/3 of validators' voting power is byzantine. Most of the time, though, the network will behave -synchronously and there will be no byzantine process. In these frequent, benign conditions: +synchronously, no process will fall behind, and there will be no byzantine process. The following describes +what will happen during a block height _h_ in these frequent, benign conditions: -* Tendermint will decide in round 0; +* Tendermint will decide in round 0, for height _h_; * `PrepareProposal` will be called exactly once at the proposer process of round 0, height _h_; -* `ProcessProposal` will be called exactly once at all processes except the proposer of round 0, and +* `ProcessProposal` will be called exactly once at all processes, and will return _accept_ in its `Response*`; -* `ExtendVote` will be called exactly once at all processes -* `VerifyVoteExtension` will be called _n-1_ times at each validator process, where _n_ is the number of validators; and -* `FinalizeBlock` will be finally called at all processes at the end of height _h_, conveying the same prepared - block that all calls to `PrepareProposal` and `ProcessProposal` had previously reported for height _h_. +* `ExtendVote` will be called exactly once at all processes; +* `VerifyVoteExtension` will be called exactly _n-1_ times at each validator process, where _n_ is + the number of validators, and will always return _accept_ in its `Response*`; +* `FinalizeBlock` will be called exactly once at all processes, conveying the same prepared + block that all calls to `PrepareProposal` and `ProcessProposal` had previously reported for + height _h_; and +* `Commit` will finally be called exactly once at all processes at the end of height _h_. However, the Application logic must be ready to cope with any possible run of Tendermint for a given height, including bad periods (byzantine proposers, network being asynchronous). @@ -28,7 +32,7 @@ In these cases, the sequence of calls to ABCI++ methods may not be so straighfor the Application should still be able to handle them, e.g., without crashing. The purpose of this section is to define what these sequences look like an a precise way. -As mentioned in the [Basic Concepts](abci++_basic_concepts_002_draft.md) section, Tendermint +As mentioned in the [Basic Concepts](./abci%2B%2B_basic_concepts.md) section, Tendermint acts as a client of ABCI++ and the Application acts as a server. Thus, it is up to Tendermint to determine when and in which order the different ABCI++ methods will be called. A well-written Application design should consider _any_ of these possible sequences. @@ -46,18 +50,15 @@ state-sync = *state-sync-attempt success-sync info state-sync-attempt = offer-snapshot *apply-chunk success-sync = offer-snapshot 1*apply-chunk -recovery = info *consensus-replay consensus-exec -consensus-replay = decide +recovery = info consensus-exec consensus-exec = (inf)consensus-height -consensus-height = *consensus-round decide +consensus-height = *consensus-round decide commit consensus-round = proposer / non-proposer -proposer = prepare-proposal extend-proposer -extend-proposer = *got-vote [extend-vote] *got-vote - -non-proposer = *got-vote [extend-non-proposer] *got-vote -extend-non-proposer = process-proposal *got-vote [extend-vote] +proposer = *got-vote prepare-proposal *got-vote process-proposal [extend] +extend = *got-vote extend-vote *got-vote +non-proposer = *got-vote [process-proposal] [extend] init-chain = %s"" offer-snapshot = %s"" @@ -68,12 +69,10 @@ process-proposal = %s"" extend-vote = %s"" got-vote = %s"" decide = %s"" +commit = %s"" ``` ->**TODO** Still hesitating... introduce _n_ as total number of validators, so that we can bound the occurrences of ->`got-vote` in a round. - -We have kept some of the ABCI++ methods out of the grammar, in order to keep it as clear and concise as possible. +We have kept some ABCI methods out of the grammar, in order to keep it as clear and concise as possible. A common reason for keeping all these methods out is that they all can be called at any point in a sequence defined by the grammar above. Other reasons depend on the method in question: @@ -115,7 +114,7 @@ Let us now examine the grammar line by line, providing further details. * In _state-sync_ mode, Tendermint makes one or more attempts at synchronizing the Application's state. At the beginning of each attempt, it offers the Application a snapshot found at another process. - If the Application accepts the snapshop, at sequence of calls to `ApplySnapshotChunk` method follow + If the Application accepts the snapshot, a sequence of calls to `ApplySnapshotChunk` method follow to provide the Application with all the snapshots needed, in order to reconstruct the state locally. A successful attempt must provide at least one chunk via `ApplySnapshotChunk`. At the end of a successful attempt, Tendermint calls `Info` to make sure the recontructed state's @@ -128,12 +127,10 @@ Let us now examine the grammar line by line, providing further details. >``` * In recovery mode, Tendermint first calls `Info` to know from which height it needs to replay decisions - to the Application. To replay a decision, Tendermint simply calls `FinalizeBlock` with the decided - block at that height. After this, Tendermint enters nomal consensus execution. + to the Application. After this, Tendermint enters nomal consensus execution. >```abnf ->recovery = info *consensus-replay consensus-exec ->consensus-replay = decide +>recovery = info consensus-exec >``` * The non-terminal `consensus-exec` is a key point in this grammar. It is an infinite sequence of @@ -145,33 +142,36 @@ Let us now examine the grammar line by line, providing further details. >consensus-exec = (inf)consensus-height >``` -* A consensus height consists of zero or more rounds before deciding via a call to `FinalizeBlock`. - In each round, the sequence of method calls depends on whether the local process is the proposer or not. +* A consensus height consists of zero or more rounds before deciding and executing via a call to + `FinalizeBlock`, followed by a call to `Commit`. In each round, the sequence of method calls + depends on whether the local process is the proposer or not. Note that, if a height contains zero + rounds, this means the process is replaying an already decided value (catch-up mode). >```abnf ->consensus-height = *consensus-round decide +>consensus-height = *consensus-round decide commit >consensus-round = proposer / non-proposer >``` -* If the local process is the proposer of the current round, Tendermint starts by calling `PrepareProposal`. - No calls to methods related to vote extensions (`ExtendVote`, `VerifyVoteExtension`) can be called - in the present round before `PrepareProposal`. Once `PrepareProposal` is called, calls to - `ExtendVote` and `VerifyVoteExtension` can come in any order, although the former will be called - at most once in this round. +* For every round, if the local process is the proposer of the current round, Tendermint starts by + calling `PrepareProposal`, followed by `ProcessProposal`. Then, optionally, the Application is + asked to extend its vote for that round. Calls to `VerifyVoteExtension` can come at any time: the + local process may be slightly late in the current round, or votes may come from a future round + of this height. >```abnf ->proposer = prepare-proposal extend-proposer ->extend-proposer = *got-vote [extend-vote] *got-vote +>proposer = *got-vote prepare-proposal *got-vote process-proposal [extend] +>extend = *got-vote extend-vote *got-vote >``` -* If the local process is not the proposer of the current round, Tendermint will call `ProcessProposal` - at most once. At most one call to `ExtendVote` can occur only after `ProcessProposal` is called. - A number of calls to `VerifyVoteExtension` can occur in any order with respect to `ProcessProposal` - and `ExtendVote` throughout the round. +* Also for every round, if the local process is _not_ the proposer of the current round, Tendermint + will call `ProcessProposal` at most once. At most one call to `ExtendVote` may occur only after + `ProcessProposal` is called. A number of calls to `VerifyVoteExtension` can occur in any order + with respect to `ProcessProposal` and `ExtendVote` throughout the round. The reasons are the same + as above, namely, the process running slightly late in the current round, or votes from future + rounds of this height received. >```abnf ->non-proposer = *got-vote [extend-non-proposer] *got-vote ->extend-non-proposer = process-proposal *got-vote [extend-vote] +>non-proposer = *got-vote [process-proposal] [extend] >``` * Finally, the grammar describes all its terminal symbols, which denote the different ABCI++ method calls that @@ -187,6 +187,7 @@ Let us now examine the grammar line by line, providing further details. >extend-vote = %s"" >got-vote = %s"" >decide = %s"" +>commit = %s"" >``` ## Adapting existing Applications that use ABCI @@ -202,17 +203,21 @@ to undergo any changes in their implementation. As for the new methods: -* `PrepareProposal` must create a list of [TxRecord](./abci++_methods_002_draft.md#txrecord) each containing a - transaction passed in `RequestPrepareProposal.txs`, in the same other. The field `action` must be set to `UNMODIFIED` - for all [TxRecord](./abci++_methods_002_draft.md#txrecord) elements in the list. +* `PrepareProposal` must create a list of [TxRecord](./abci++_methods.md#txrecord) each containing + a transaction passed in `RequestPrepareProposal.txs`, in the same other. The field `action` must + be set to `UNMODIFIED` for all [TxRecord](./abci++_methods.md#txrecord) elements in the list. The Application must check whether the size of all transactions exceeds the byte limit - (`RequestPrepareProposal.max_tx_bytes`). If so, the Application must remove transactions at the end of the list - until the total byte size is at or below the limit. + (`RequestPrepareProposal.max_tx_bytes`). If so, the Application must remove transactions at the + end of the list until the total byte size is at or below the limit. * `ProcessProposal` must set `ResponseProcessProposal.accept` to _true_ and return. * `ExtendVote` is to set `ResponseExtendVote.extension` to an empty byte array and return. -* `VerifyVoteExtension` must set `ResponseVerifyVoteExtension.accept` to _true_ if the extension is an empty byte array - and _false_ otherwise, then return. -* `FinalizeBlock` is to coalesce the implementation of methods `BeginBlock`, `DeliverTx`, `EndBlock`, and `Commit`. - Legacy applications looking to reuse old code that implemented `DeliverTx` should wrap the legacy - `DeliverTx` logic in a loop that executes one transaction iteration per +* `VerifyVoteExtension` must set `ResponseVerifyVoteExtension.accept` to _true_ if the extension is + an empty byte array and _false_ otherwise, then return. +* `FinalizeBlock` is to coalesce the implementation of methods `BeginBlock`, `DeliverTx`, and + `EndBlock`. Legacy applications looking to reuse old code that implemented `DeliverTx` should + wrap the legacy `DeliverTx` logic in a loop that executes one transaction iteration per transaction in `RequestFinalizeBlock.tx`. + +Finally, `Commit`, which is kept in ABCI++, no longer returns the `AppHash`. It is now up to +`FinalizeBlock` to do so. Thus, a slight refactoring of the old `Commit` implementation will be +needed to move the return of `AppHash` to `FinalizeBlock`. diff --git a/spec/abci++/v0.md b/spec/abci++/v0.md deleted file mode 100644 index 163b3f7cb..000000000 --- a/spec/abci++/v0.md +++ /dev/null @@ -1,156 +0,0 @@ -# Tendermint v0 Markdown pseudocode - -This translates the latex code for Tendermint consensus from the Tendermint paper into markdown. - -### Initialization - -```go -h_p ← 0 -round_p ← 0 -step_p is one of {propose, prevote, precommit} -decision_p ← Vector() -lockedRound_p ← -1 -lockedValue_p ← nil -validValue_p ← nil -validRound_p ← -1 -``` - -### StartRound(round) - -```go -function startRound(round) { - round_p ← round - step_p ← propose - if proposer(h_p, round_p) = p { - if validValue_p != nil { - proposal ← validValue_p - } else { - proposal ← getValue() - } - broadcast ⟨PROPOSAL, h_p, round_p, proposal, validRound_p⟩ - } else { - schedule OnTimeoutPropose(h_p,round_p) to be executed after timeoutPropose(round_p) - } -} -``` - -### ReceiveProposal - -In the case where the local node is not locked on any round, the following is ran: - -```go -upon ⟨PROPOSAL, h_p, round_p, v, −1) from proposer(h_p, round_p) while step_p = propose do { - if valid(v) ∧ (lockedRound_p = −1 ∨ lockedValue_p = v) { - broadcast ⟨PREVOTE, h_p, round_p, id(v)⟩ - } else { - broadcast ⟨PREVOTE, h_p, round_p, nil⟩ - } - step_p ← prevote -} -``` - -In the case where the node is locked on a round, the following is ran: - -```go -upon ⟨PROPOSAL, h_p, round_p, v, vr⟩ from proposer(h_p, round_p) - AND 2f + 1 ⟨PREVOTE, h_p, vr, id(v)⟩ - while step_p = propose ∧ (vr ≥ 0 ∧ vr < round_p) do { - if valid(v) ∧ (lockedRound_p ≤ vr ∨ lockedValue_p = v) { - broadcast ⟨PREVOTE, h_p, round_p, id(v)⟩ - } else { - broadcast ⟨PREVOTE, h_p, round_p, nil⟩ - } - step_p ← prevote -} -``` - -### Prevote timeout - -Upon receiving 2f + 1 prevotes, setup a timeout. - -```go -upon 2f + 1 ⟨PREVOTE, h_p, vr, *⟩ with step_p = prevote for the first time, do { - schedule OnTimeoutPrevote(h_p, round_p) to be executed after timeoutPrevote(round_p) -} -``` - -with OnTimeoutPrevote defined as: - -```go -function OnTimeoutPrevote(height, round) { - if (height = h_p && round = round_p && step_p = prevote) { - broadcast ⟨PRECOMMIT, h_p, round_p, nil⟩ - step_p ← precommit - } -} -``` - -### Receiving enough prevotes to precommit - -The following code is ran upon receiving 2f + 1 prevotes for the same block - -```go -upon ⟨PROPOSAL, h_p, round_p, v, *⟩ - from proposer(h_p, round_p) - AND 2f + 1 ⟨PREVOTE, h_p, round_p, id(v)⟩ - while valid(v) ∧ step_p >= prevote for the first time do { - if (step_p = prevote) { - lockedValue_p ← v - lockedRound_p ← round_p - broadcast ⟨PRECOMMIT, h_p, round_p, id(v)⟩ - step_p ← precommit - } - validValue_p ← v - validRound_p ← round_p -} -``` - -And upon receiving 2f + 1 prevotes for nil: - -```go -upon 2f + 1 ⟨PREVOTE, h_p, round_p, nil⟩ - while step_p = prevote do { - broadcast ⟨PRECOMMIT, h_p, round_p, nil⟩ - step_p ← precommit -} -``` - -### Precommit timeout - -Upon receiving 2f + 1 precommits, setup a timeout. - -```go -upon 2f + 1 ⟨PRECOMMIT, h_p, vr, *⟩ for the first time, do { - schedule OnTimeoutPrecommit(h_p, round_p) to be executed after timeoutPrecommit(round_p) -} -``` - -with OnTimeoutPrecommit defined as: - -```go -function OnTimeoutPrecommit(height, round) { - if (height = h_p && round = round_p) { - StartRound(round_p + 1) - } -} -``` - -### Upon Receiving 2f + 1 precommits - -The following code is ran upon receiving 2f + 1 precommits for the same block - -```go -upon ⟨PROPOSAL, h_p, r, v, *⟩ - from proposer(h_p, r) - AND 2f + 1 ⟨ PRECOMMIT, h_p, r, id(v)⟩ - while decision_p[h_p] = nil do { - if (valid(v)) { - decision_p[h_p] ← v - h_p ← h_p + 1 - reset lockedRound_p, lockedValue_p,validRound_p and validValue_p to initial values - StartRound(0) - } -} -``` - -If we don't see 2f + 1 precommits for the same block, we wait until we get 2f + 1 precommits, and the timeout occurs. \ No newline at end of file diff --git a/spec/abci++/v1.md b/spec/abci++/v1.md deleted file mode 100644 index 96dc8e674..000000000 --- a/spec/abci++/v1.md +++ /dev/null @@ -1,162 +0,0 @@ -# Tendermint v1 Markdown pseudocode - -This adds hooks for the existing ABCI to the prior pseudocode - -### Initialization - -```go -h_p ← 0 -round_p ← 0 -step_p is one of {propose, prevote, precommit} -decision_p ← Vector() -lockedValue_p ← nil -validValue_p ← nil -validRound_p ← -1 -``` - -### StartRound(round) - -```go -function startRound(round) { - round_p ← round - step_p ← propose - if proposer(h_p, round_p) = p { - if validValue_p != nil { - proposal ← validValue_p - } else { - txdata ← mempool.GetBlock() - // getBlockProposal fills in header - proposal ← getBlockProposal(txdata) - } - broadcast ⟨PROPOSAL, h_p, round_p, proposal, validRound_p⟩ - } else { - schedule OnTimeoutPropose(h_p,round_p) to be executed after timeoutPropose(round_p) - } -} -``` - -### ReceiveProposal - -In the case where the local node is not locked on any round, the following is ran: - -```go -upon ⟨PROPOSAL, h_p, round_p, v, −1) from proposer(h_p, round_p) while step_p = propose do { - if valid(v) ∧ (lockedRound_p = −1 ∨ lockedValue_p = v) { - broadcast ⟨PREVOTE, h_p, round_p, id(v)⟩ - } else { - broadcast ⟨PREVOTE, h_p, round_p, nil⟩ - } - step_p ← prevote -} -``` - -In the case where the node is locked on a round, the following is ran: - -```go -upon ⟨PROPOSAL, h_p, round_p, v, vr⟩ - from proposer(h_p, round_p) - AND 2f + 1 ⟨PREVOTE, h_p, vr, id(v)⟩ - while step_p = propose ∧ (vr ≥ 0 ∧ vr < round_p) do { - if valid(v) ∧ (lockedRound_p ≤ vr ∨ lockedValue_p = v) { - broadcast ⟨PREVOTE, h_p, round_p, id(v)⟩ - } else { - broadcast ⟨PREVOTE, h_p, round_p, nil⟩ - } - step_p ← prevote -} -``` - -### Prevote timeout - -Upon receiving 2f + 1 prevotes, setup a timeout. - -```go -upon 2f + 1 ⟨PREVOTE, h_p, vr, -1⟩ - with step_p = prevote for the first time, do { - schedule OnTimeoutPrevote(h_p, round_p) to be executed after timeoutPrevote(round_p) -} -``` - -with OnTimeoutPrevote defined as: - -```go -function OnTimeoutPrevote(height, round) { - if (height = h_p && round = round_p && step_p = prevote) { - broadcast ⟨PRECOMMIT, h_p, round_p, nil⟩ - step_p ← precommit - } -} -``` - -### Receiving enough prevotes to precommit - -The following code is ran upon receiving 2f + 1 prevotes for the same block - -```go -upon ⟨PROPOSAL, h_p, round_p, v, *⟩ - from proposer(h_p, round_p) - AND 2f + 1 ⟨PREVOTE, h_p, vr, id(v)⟩ - while valid(v) ∧ step_p >= prevote for the first time do { - if (step_p = prevote) { - lockedValue_p ← v - lockedRound_p ← round_p - broadcast ⟨PRECOMMIT, h_p, round_p, id(v)⟩ - step_p ← precommit - } - validValue_p ← v - validRound_p ← round_p -} -``` - -And upon receiving 2f + 1 prevotes for nil: - -```go -upon 2f + 1 ⟨PREVOTE, h_p, round_p, nil⟩ - while step_p = prevote do { - broadcast ⟨PRECOMMIT, h_p, round_p, nil⟩ - step_p ← precommit -} -``` - -### Precommit timeout - -Upon receiving 2f + 1 precommits, setup a timeout. - -```go -upon 2f + 1 ⟨PRECOMMIT, h_p, vr, *⟩ for the first time, do { - schedule OnTimeoutPrecommit(h_p, round_p) to be executed after timeoutPrecommit(round_p) -} -``` - -with OnTimeoutPrecommit defined as: - -```go -function OnTimeoutPrecommit(height, round) { - if (height = h_p && round = round_p) { - StartRound(round_p + 1) - } -} -``` - -### Upon Receiving 2f + 1 precommits - -The following code is ran upon receiving 2f + 1 precommits for the same block - -```go -upon ⟨PROPOSAL, h_p, r, v, *⟩ - from proposer(h_p, r) - AND 2f + 1 ⟨ PRECOMMIT, h_p, r, id(v)⟩ - while decision_p[h_p] = nil do { - if (valid(v)) { - decision_p[h_p] ← v - h_p ← h_p + 1 - reset lockedRound_p, lockedValue_p,validRound_p and validValue_p to initial values - ABCI.BeginBlock(v.header) - ABCI.DeliverTxs(v.data) - ABCI.EndBlock() - StartRound(0) - } -} -``` - -If we don't see 2f + 1 precommits for the same block, we wait until we get 2f + 1 precommits, and the timeout occurs. \ No newline at end of file diff --git a/spec/abci++/v2.md b/spec/abci++/v2.md deleted file mode 100644 index 1abd8ec67..000000000 --- a/spec/abci++/v2.md +++ /dev/null @@ -1,180 +0,0 @@ -# Tendermint v2 Markdown pseudocode - -This adds a single-threaded implementation of ABCI++, -with no optimization for splitting out verifying the header and verifying the proposal. - -### Initialization - -```go -h_p ← 0 -round_p ← 0 -step_p is one of {propose, prevote, precommit} -decision_p ← Vector() -lockedValue_p ← nil -validValue_p ← nil -validRound_p ← -1 -``` - -### StartRound(round) - -```go -function startRound(round) { - round_p ← round - step_p ← propose - if proposer(h_p, round_p) = p { - if validValue_p != nil { - proposal ← validValue_p - } else { - txdata ← mempool.GetBlock() - // getUnpreparedBlockProposal takes tx data, and fills in the unprepared header data - unpreparedProposal ← getUnpreparedBlockProposal(txdata) - // ABCI++: the proposer may reorder/update transactions in `unpreparedProposal` - proposal ← ABCI.PrepareProposal(unpreparedProposal) - } - broadcast ⟨PROPOSAL, h_p, round_p, proposal, validRound_p⟩ - } else { - schedule OnTimeoutPropose(h_p,round_p) to be executed after timeoutPropose(round_p) - } -} -``` - -### ReceiveProposal - -In the case where the local node is not locked on any round, the following is ran: - -```go -upon ⟨PROPOSAL, h_p, round_p, v, −1) from proposer(h_p, round_p) while step_p = propose do { - if valid(v) ∧ ABCI.ProcessProposal(h_p, v).accept ∧ (lockedRound_p = −1 ∨ lockedValue_p = v) { - broadcast ⟨PREVOTE, h_p, round_p, id(v)⟩ - } else { - broadcast ⟨PREVOTE, h_p, round_p, nil⟩ - // Include any slashing evidence that may be sent in the process proposal response - for evidence in ABCI.ProcessProposal(h_p, v).evidence_list { - broadcast ⟨EVIDENCE, evidence⟩ - } - } - step_p ← prevote -} -``` - -In the case where the node is locked on a round, the following is ran: - -```go -upon ⟨PROPOSAL, h_p, round_p, v, vr⟩ - from proposer(h_p, round_p) - AND 2f + 1 ⟨PREVOTE, h_p, vr, id(v)⟩ - while step_p = propose ∧ (vr ≥ 0 ∧ vr < round_p) do { - if valid(v) ∧ ABCI.ProcessProposal(h_p, v).accept ∧ (lockedRound_p ≤ vr ∨ lockedValue_p = v) { - broadcast ⟨PREVOTE, h_p, round_p, id(v)⟩ - } else { - broadcast ⟨PREVOTE, h_p, round_p, nil⟩ - // Include any slashing evidence that may be sent in the process proposal response - for evidence in ABCI.ProcessProposal(h_p, v).evidence_list { - broadcast ⟨EVIDENCE, evidence⟩ - } - } - step_p ← prevote -} -``` - -### Prevote timeout - -Upon receiving 2f + 1 prevotes, setup a timeout. - -```go -upon 2f + 1 ⟨PREVOTE, h_p, vr, -1⟩ - with step_p = prevote for the first time, do { - schedule OnTimeoutPrevote(h_p, round_p) to be executed after timeoutPrevote(round_p) -} -``` - -with OnTimeoutPrevote defined as: - -```go -function OnTimeoutPrevote(height, round) { - if (height = h_p && round = round_p && step_p = prevote) { - precommit_extension ← ABCI.ExtendVote(h_p, round_p, nil) - broadcast ⟨PRECOMMIT, h_p, round_p, nil, precommit_extension⟩ - step_p ← precommit - } -} -``` - -### Receiving enough prevotes to precommit - -The following code is ran upon receiving 2f + 1 prevotes for the same block - -```go -upon ⟨PROPOSAL, h_p, round_p, v, *⟩ - from proposer(h_p, round_p) - AND 2f + 1 ⟨PREVOTE, h_p, vr, id(v)⟩ - while valid(v) ∧ step_p >= prevote for the first time do { - if (step_p = prevote) { - lockedValue_p ← v - lockedRound_p ← round_p - precommit_extension ← ABCI.ExtendVote(h_p, round_p, id(v)) - broadcast ⟨PRECOMMIT, h_p, round_p, id(v), precommit_extension⟩ - step_p ← precommit - } - validValue_p ← v - validRound_p ← round_p -} -``` - -And upon receiving 2f + 1 prevotes for nil: - -```go -upon 2f + 1 ⟨PREVOTE, h_p, round_p, nil⟩ - while step_p = prevote do { - precommit_extension ← ABCI.ExtendVote(h_p, round_p, nil) - broadcast ⟨PRECOMMIT, h_p, round_p, nil, precommit_extension⟩ - step_p ← precommit -} -``` - -### Upon receiving a precommit - -Upon receiving a precommit `precommit`, we ensure that `ABCI.VerifyVoteExtension(precommit.precommit_extension) = true` -before accepting the precommit. This is akin to how we check the signature on precommits normally, hence its not wrapped -in the syntax of methods from the paper. - -### Precommit timeout - -Upon receiving 2f + 1 precommits, setup a timeout. - -```go -upon 2f + 1 ⟨PRECOMMIT, h_p, vr, *⟩ for the first time, do { - schedule OnTimeoutPrecommit(h_p, round_p) to be executed after timeoutPrecommit(round_p) -} -``` - -with OnTimeoutPrecommit defined as: - -```go -function OnTimeoutPrecommit(height, round) { - if (height = h_p && round = round_p) { - StartRound(round_p + 1) - } -} -``` - -### Upon Receiving 2f + 1 precommits - -The following code is ran upon receiving 2f + 1 precommits for the same block - -```go -upon ⟨PROPOSAL, h_p, r, v, *⟩ - from proposer(h_p, r) - AND 2f + 1 ⟨ PRECOMMIT, h_p, r, id(v)⟩ - while decision_p[h_p] = nil do { - if (valid(v)) { - decision_p[h_p] ← v - h_p ← h_p + 1 - reset lockedRound_p, lockedValue_p,validRound_p and validValue_p to initial values - ABCI.FinalizeBlock(id(v)) - StartRound(0) - } -} -``` - -If we don't see 2f + 1 precommits for the same block, we wait until we get 2f + 1 precommits, and the timeout occurs. diff --git a/spec/abci++/v3.md b/spec/abci++/v3.md deleted file mode 100644 index ed4c720b4..000000000 --- a/spec/abci++/v3.md +++ /dev/null @@ -1,201 +0,0 @@ -# Tendermint v3 Markdown pseudocode - -This is a single-threaded implementation of ABCI++, -with an optimization for the ProcessProposal phase. -Namely, processing of the header and the block data is separated into two different functions. - -### Initialization - -```go -h_p ← 0 -round_p ← 0 -step_p is one of {propose, prevote, precommit} -decision_p ← Vector() -lockedValue_p ← nil -validValue_p ← nil -validRound_p ← -1 -``` - -### StartRound(round) - -```go -function startRound(round) { - round_p ← round - step_p ← propose - if proposer(h_p, round_p) = p { - if validValue_p != nil { - proposal ← validValue_p - } else { - txdata ← mempool.GetBlock() - // getUnpreparedBlockProposal fills in header - unpreparedProposal ← getUnpreparedBlockProposal(txdata) - proposal ← ABCI.PrepareProposal(unpreparedProposal) - } - broadcast ⟨PROPOSAL, h_p, round_p, proposal, validRound_p⟩ - } else { - schedule OnTimeoutPropose(h_p,round_p) to be executed after timeoutPropose(round_p) - } -} -``` - -### ReceiveProposal - -In the case where the local node is not locked on any round, the following is ran: - -```go -upon ⟨PROPOSAL, h_p, round_p, v_header, −1) from proposer(h_p, round_p) while step_p = propose do { - prevote_nil ← false - // valid is Tendermints validation, ABCI.VerifyHeader is the applications - if valid(v_header) ∧ ABCI.VerifyHeader(h_p, v_header) ∧ (lockedRound_p = −1 ∨ lockedValue_p = id(v_header)) { - wait to receive proposal v corresponding to v_header - // We split up the app's header verification from the remainder of its processing of the proposal - if ABCI.ProcessProposal(h_p, v).accept { - broadcast ⟨PREVOTE, h_p, round_p, id(v)⟩ - } else { - prevote_nil ← true - // Include any slashing evidence that may be sent in the process proposal response - for evidence in ABCI.ProcessProposal(h_p, v).evidence_list { - broadcast ⟨EVIDENCE, evidence⟩ - } - } - } else { - prevote_nil ← true - } - if prevote_nil { - broadcast ⟨PREVOTE, h_p, round_p, nil⟩ - } - step_p ← prevote -} -``` - -In the case where the node is locked on a round, the following is ran: - -```go -upon ⟨PROPOSAL, h_p, round_p, v_header, vr⟩ - from proposer(h_p, round_p) - AND 2f + 1 ⟨PREVOTE, h_p, vr, id(v_header)⟩ - while step_p = propose ∧ (vr ≥ 0 ∧ vr < round_p) do { - prevote_nil ← false - if valid(v) ∧ ABCI.VerifyHeader(h_p, v.header) ∧ (lockedRound_p ≤ vr ∨ lockedValue_p = v) { - wait to receive proposal v corresponding to v_header - // We split up the app's header verification from the remainder of its processing of the proposal - if ABCI.ProcessProposal(h_p, v).accept { - broadcast ⟨PREVOTE, h_p, round_p, id(v)⟩ - } else { - prevote_nil ← true - // Include any slashing evidence that may be sent in the process proposal response - for evidence in ABCI.ProcessProposal(h_p, v).evidence_list { - broadcast ⟨EVIDENCE, evidence⟩ - } - } - } else { - prevote_nil ← true - } - if prevote_nil { - broadcast ⟨PREVOTE, h_p, round_p, nil⟩ - } - step_p ← prevote -} -``` - -### Prevote timeout - -Upon receiving 2f + 1 prevotes, setup a timeout. - -```go -upon 2f + 1 ⟨PREVOTE, h_p, vr, -1⟩ - with step_p = prevote for the first time, do { - schedule OnTimeoutPrevote(h_p, round_p) to be executed after timeoutPrevote(round_p) -} -``` - -with OnTimeoutPrevote defined as: - -```go -function OnTimeoutPrevote(height, round) { - if (height = h_p && round = round_p && step_p = prevote) { - precommit_extension ← ABCI.ExtendVote(h_p, round_p, nil) - broadcast ⟨PRECOMMIT, h_p, round_p, nil, precommit_extension⟩ - step_p ← precommit - } -} -``` - -### Receiving enough prevotes to precommit - -The following code is ran upon receiving 2f + 1 prevotes for the same block - -```go -upon ⟨PROPOSAL, h_p, round_p, v, *⟩ - from proposer(h_p, round_p) - AND 2f + 1 ⟨PREVOTE, h_p, vr, id(v)⟩ - while valid(v) ∧ step_p >= prevote for the first time do { - if (step_p = prevote) { - lockedValue_p ← v - lockedRound_p ← round_p - precommit_extension ← ABCI.ExtendVote(h_p, round_p, id(v)) - broadcast ⟨PRECOMMIT, h_p, round_p, id(v), precommit_extension⟩ - step_p ← precommit - } - validValue_p ← v - validRound_p ← round_p -} -``` - -And upon receiving 2f + 1 prevotes for nil: - -```go -upon 2f + 1 ⟨PREVOTE, h_p, round_p, nil⟩ - while step_p = prevote do { - precommit_extension ← ABCI.ExtendVote(h_p, round_p, nil) - broadcast ⟨PRECOMMIT, h_p, round_p, nil, precommit_extension⟩ - step_p ← precommit -} -``` - -### Upon receiving a precommit - -Upon receiving a precommit `precommit`, we ensure that `ABCI.VerifyVoteExtension(precommit.precommit_extension) = true` -before accepting the precommit. This is akin to how we check the signature on precommits normally, hence its not wrapped -in the syntax of methods from the paper. - -### Precommit timeout - -Upon receiving 2f + 1 precommits, setup a timeout. - -```go -upon 2f + 1 ⟨PRECOMMIT, h_p, vr, *⟩ for the first time, do { - schedule OnTimeoutPrecommit(h_p, round_p) to be executed after timeoutPrecommit(round_p) -} -``` - -with OnTimeoutPrecommit defined as: - -```go -function OnTimeoutPrecommit(height, round) { - if (height = h_p && round = round_p) { - StartRound(round_p + 1) - } -} -``` - -### Upon Receiving 2f + 1 precommits - -The following code is ran upon receiving 2f + 1 precommits for the same block - -```go -upon ⟨PROPOSAL, h_p, r, v, *⟩ - from proposer(h_p, r) - AND 2f + 1 ⟨ PRECOMMIT, h_p, r, id(v)⟩ - while decision_p[h_p] = nil do { - if (valid(v)) { - decision_p[h_p] ← v - h_p ← h_p + 1 - reset lockedRound_p, lockedValue_p,validRound_p and validValue_p to initial values - ABCI.FinalizeBlock(id(v)) - StartRound(0) - } -} -``` - -If we don't see 2f + 1 precommits for the same block, we wait until we get 2f + 1 precommits, and the timeout occurs. \ No newline at end of file diff --git a/spec/abci++/v4.md b/spec/abci++/v4.md deleted file mode 100644 index d211fd87f..000000000 --- a/spec/abci++/v4.md +++ /dev/null @@ -1,199 +0,0 @@ -# Tendermint v4 Markdown pseudocode - -This is a multi-threaded implementation of ABCI++, -where ProcessProposal starts when the proposal is received, but ends before precommitting. - -### Initialization - -```go -h_p ← 0 -round_p ← 0 -step_p is one of {propose, prevote, precommit} -decision_p ← Vector() -lockedValue_p ← nil -validValue_p ← nil -validRound_p ← -1 -``` - -### StartRound(round) - -```go -function startRound(round) { - round_p ← round - step_p ← propose - if proposer(h_p, round_p) = p { - if validValue_p != nil { - proposal ← validValue_p - } else { - txdata ← mempool.GetBlock() - // getUnpreparedBlockProposal fills in header - unpreparedProposal ← getUnpreparedBlockProposal(txdata) - proposal ← ABCI.PrepareProposal(unpreparedProposal) - } - broadcast ⟨PROPOSAL, h_p, round_p, proposal, validRound_p⟩ - } else { - schedule OnTimeoutPropose(h_p,round_p) to be executed after timeoutPropose(round_p) - } -} -``` - -### ReceiveProposal - -In the case where the local node is not locked on any round, the following is ran: - -```go -upon ⟨PROPOSAL, h_p, round_p, v, −1) from proposer(h_p, round_p) while step_p = propose do { - if valid(v) ∧ ABCI.VerifyHeader(h_p, v.header) ∧ (lockedRound_p = −1 ∨ lockedValue_p = v) { - // We fork process proposal into a parallel process - Fork ABCI.ProcessProposal(h_p, v) - broadcast ⟨PREVOTE, h_p, round_p, id(v)⟩ - } else { - broadcast ⟨PREVOTE, h_p, round_p, nil⟩ - } - step_p ← prevote -} -``` - -In the case where the node is locked on a round, the following is ran: - -```go -upon ⟨PROPOSAL, h_p, round_p, v, vr⟩ - from proposer(h_p, round_p) - AND 2f + 1 ⟨PREVOTE, h_p, vr, id(v)⟩ - while step_p = propose ∧ (vr ≥ 0 ∧ vr < round_p) do { - if valid(v) ∧ ABCI.VerifyHeader(h_p, v.header) ∧ (lockedRound_p ≤ vr ∨ lockedValue_p = v) { - // We fork process proposal into a parallel process - Fork ABCI.ProcessProposal(h_p, v) - broadcast ⟨PREVOTE, h_p, round_p, id(v)⟩ - } else { - broadcast ⟨PREVOTE, h_p, round_p, nil⟩ - } - step_p ← prevote -} -``` - -### Prevote timeout - -Upon receiving 2f + 1 prevotes, setup a timeout. - -```go -upon 2f + 1 ⟨PREVOTE, h_p, vr, -1⟩ - with step_p = prevote for the first time, do { - schedule OnTimeoutPrevote(h_p, round_p) to be executed after timeoutPrevote(round_p) -} -``` - -with OnTimeoutPrevote defined as: - -```go -def OnTimeoutPrevote(height, round) { - if (height = h_p && round = round_p && step_p = prevote) { - // Join the ProcessProposal, and output any evidence in case it has some. - processProposalOutput ← Join ABCI.ProcessProposal(h_p, v) - for evidence in processProposalOutput.evidence_list { - broadcast ⟨EVIDENCE, evidence⟩ - } - - precommit_extension ← ABCI.ExtendVote(h_p, round_p, nil) - broadcast ⟨PRECOMMIT, h_p, round_p, nil, precommit_extension⟩ - step_p ← precommit - } -} -``` - -### Receiving enough prevotes to precommit - -The following code is ran upon receiving 2f + 1 prevotes for the same block - -```go -upon ⟨PROPOSAL, h_p, round_p, v, *⟩ - from proposer(h_p, round_p) - AND 2f + 1 ⟨PREVOTE, h_p, vr, id(v)⟩ -while valid(v) ∧ step_p >= prevote for the first time do { - if (step_p = prevote) { - lockedValue_p ← v - lockedRound_p ← round_p - processProposalOutput ← Join ABCI.ProcessProposal(h_p, v) - // If the proposal is valid precommit as before. - // If it was invalid, precommit nil. - // Note that ABCI.ProcessProposal(h_p, v).accept is deterministic for all honest nodes. - precommit_value ← nil - if processProposalOutput.accept { - precommit_value ← id(v) - } - precommit_extension ← ABCI.ExtendVote(h_p, round_p, precommit_value) - broadcast ⟨PRECOMMIT, h_p, round_p, precommit_value, precommit_extension⟩ - for evidence in processProposalOutput.evidence_list { - broadcast ⟨EVIDENCE, evidence⟩ - } - - step_p ← precommit - } - validValue_p ← v - validRound_p ← round_p -} -``` - -And upon receiving 2f + 1 prevotes for nil: - -```go -upon 2f + 1 ⟨PREVOTE, h_p, round_p, nil⟩ - while step_p = prevote do { - // Join ABCI.ProcessProposal, and broadcast any evidence if it exists. - processProposalOutput ← Join ABCI.ProcessProposal(h_p, v) - for evidence in processProposalOutput.evidence_list { - broadcast ⟨EVIDENCE, evidence⟩ - } - - precommit_extension ← ABCI.ExtendVote(h_p, round_p, nil) - broadcast ⟨PRECOMMIT, h_p, round_p, nil, precommit_extension⟩ - step_p ← precommit -} -``` - -### Upon receiving a precommit - -Upon receiving a precommit `precommit`, we ensure that `ABCI.VerifyVoteExtension(precommit.precommit_extension) = true` -before accepting the precommit. This is akin to how we check the signature on precommits normally, hence its not wrapped -in the syntax of methods from the paper. - -### Precommit timeout - -Upon receiving 2f + 1 precommits, setup a timeout. - -```go -upon 2f + 1 ⟨PRECOMMIT, h_p, vr, *⟩ for the first time, do { - schedule OnTimeoutPrecommit(h_p, round_p) to be executed after timeoutPrecommit(round_p) -} -``` - -with OnTimeoutPrecommit defined as: - -```go -def OnTimeoutPrecommit(height, round) { - if (height = h_p && round = round_p) { - StartRound(round_p + 1) - } -} -``` - -### Upon Receiving 2f + 1 precommits - -The following code is ran upon receiving 2f + 1 precommits for the same block - -```go -upon ⟨PROPOSAL, h_p, r, v, *⟩ - from proposer(h_p, r) - AND 2f + 1 ⟨ PRECOMMIT, h_p, r, id(v)⟩ - while decision_p[h_p] = nil do { - if (valid(v)) { - decision_p[h_p] ← v - h_p ← h_p + 1 - reset lockedRound_p, lockedValue_p,validRound_p and validValue_p to initial values - ABCI.FinalizeBlock(id(v)) - StartRound(0) - } -} -``` - -If we don't see 2f + 1 precommits for the same block, we wait until we get 2f + 1 precommits, and the timeout occurs. \ No newline at end of file From d8de2d85c0a07d0c2b72591a7991861bf88c2775 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Jul 2022 12:12:15 +0200 Subject: [PATCH 54/61] build(deps): Bump github.com/libp2p/go-buffer-pool from 0.0.2 to 0.1.0 (#8932) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 691efb09f..f8fa79cfc 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/lib/pq v1.10.6 - github.com/libp2p/go-buffer-pool v0.0.2 + github.com/libp2p/go-buffer-pool v0.1.0 github.com/mroth/weightedrand v0.4.1 github.com/oasisprotocol/curve25519-voi v0.0.0-20210609091139-0a56a4bca00b github.com/ory/dockertest v3.3.5+incompatible diff --git a/go.sum b/go.sum index ca88c3e8f..f7b9149a2 100644 --- a/go.sum +++ b/go.sum @@ -717,8 +717,8 @@ github.com/lib/pq v1.9.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.6 h1:jbk+ZieJ0D7EVGJYpL9QTz7/YW6UHbmdnZWYyK5cdBs= github.com/lib/pq v1.10.6/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/libp2p/go-buffer-pool v0.0.2 h1:QNK2iAFa8gjAe1SPz6mHSMuCcjs+X1wlHzeOSqcmlfs= -github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM= +github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= +github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/lufeee/execinquery v1.0.0 h1:1XUTuLIVPDlFvUU3LXmmZwHDsolsxXnY67lzhpeqe0I= github.com/lufeee/execinquery v1.0.0/go.mod h1:EC7DrEKView09ocscGHC+apXMIaorh4xqSxS/dy8SbM= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= From 3cde9a0bbc04683aedb96871b0b45f8e2e650101 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A1n=20Vanzetto?= Date: Tue, 5 Jul 2022 16:08:58 +0300 Subject: [PATCH 55/61] abci-cli: Add `process_proposal` command to abci-cli (#8901) * Add `process_proposal` command to abci-cli * Added process proposal to the 'tutorial' examples * Added entry in CHANGELOG_PENDING.md * Allow empty blocks in PrepareProposal, ProcessProposal, and FinalizeBlock * Fix minimum arguments * Add tests for empty block * Updated abci-cli doc Co-authored-by: Sergio Mena Co-authored-by: Jasmina Malicevic --- CHANGELOG_PENDING.md | 1 + abci/cmd/abci-cli/abci-cli.go | 74 ++++++++++++++++++++++---------- abci/example/kvstore/kvstore.go | 2 +- abci/tests/server/client.go | 13 ++++++ abci/tests/test_cli/ex1.abci | 9 +++- abci/tests/test_cli/ex1.abci.out | 25 +++++++++++ docs/app-dev/abci-cli.md | 27 ++++++++++++ 7 files changed, 127 insertions(+), 24 deletions(-) diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 07e755501..13eeaef16 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -33,6 +33,7 @@ Special thanks to external contributors on this release: - [abci] \#8664 Move `app_hash` parameter from `Commit` to `FinalizeBlock`. (@sergio-mena) - [abci] \#8656 Added cli command for `PrepareProposal`. (@jmalicevic) - [sink/psql] \#8637 tx_results emitted from psql sink are now json encoded, previously they were protobuf encoded + - [abci] \#8901 Added cli command for `ProcessProposal`. (@hvanz) - P2P Protocol diff --git a/abci/cmd/abci-cli/abci-cli.go b/abci/cmd/abci-cli/abci-cli.go index 237493e0e..824bbb360 100644 --- a/abci/cmd/abci-cli/abci-cli.go +++ b/abci/cmd/abci-cli/abci-cli.go @@ -79,10 +79,11 @@ func RootCmmand(logger log.Logger) *cobra.Command { // Structure for data passed to print response. type response struct { // generic abci response - Data []byte - Code uint32 - Info string - Log string + Data []byte + Code uint32 + Info string + Log string + Status int32 Query *queryResponse } @@ -132,6 +133,7 @@ func addCommands(cmd *cobra.Command, logger log.Logger) { cmd.AddCommand(versionCmd) cmd.AddCommand(testCmd) cmd.AddCommand(prepareProposalCmd) + cmd.AddCommand(processProposalCmd) cmd.AddCommand(getQueryCmd()) // examples @@ -172,7 +174,7 @@ This command opens an interactive console for running any of the other commands without opening a new connection each time `, Args: cobra.ExactArgs(0), - ValidArgs: []string{"echo", "info", "query", "check_tx", "prepare_proposal", "finalize_block", "commit"}, + ValidArgs: []string{"echo", "info", "query", "check_tx", "prepare_proposal", "process_proposal", "finalize_block", "commit"}, RunE: cmdConsole, } @@ -195,7 +197,7 @@ var finalizeBlockCmd = &cobra.Command{ Use: "finalize_block", Short: "deliver a block of transactions to the application", Long: "deliver a block of transactions to the application", - Args: cobra.MinimumNArgs(1), + Args: cobra.MinimumNArgs(0), RunE: cmdFinalizeBlock, } @@ -230,10 +232,18 @@ var prepareProposalCmd = &cobra.Command{ Use: "prepare_proposal", Short: "prepare proposal", Long: "prepare proposal", - Args: cobra.MinimumNArgs(1), + Args: cobra.MinimumNArgs(0), RunE: cmdPrepareProposal, } +var processProposalCmd = &cobra.Command{ + Use: "process_proposal", + Short: "process proposal", + Long: "process proposal", + Args: cobra.MinimumNArgs(0), + RunE: cmdProcessProposal, +} + func getQueryCmd() *cobra.Command { cmd := &cobra.Command{ Use: "query", @@ -352,6 +362,11 @@ func cmdTest(cmd *cobra.Command, args []string) error { types.TxRecord_UNMODIFIED, }, nil) }, + func() error { + return servertest.ProcessProposal(ctx, client, [][]byte{ + {0x01}, + }, types.ResponseProcessProposal_ACCEPT) + }, }) } @@ -454,6 +469,8 @@ func muxOnCommands(cmd *cobra.Command, pArgs []string) error { return cmdQuery(cmd, actualArgs) case "prepare_proposal": return cmdPrepareProposal(cmd, actualArgs) + case "process_proposal": + return cmdProcessProposal(cmd, actualArgs) default: return cmdUnimplemented(cmd, pArgs) } @@ -517,13 +534,6 @@ const codeBad uint32 = 10 // Append new txs to application func cmdFinalizeBlock(cmd *cobra.Command, args []string) error { - if len(args) == 0 { - printResponse(cmd, args, response{ - Code: codeBad, - Log: "Must provide at least one transaction", - }) - return nil - } txs := make([][]byte, len(args)) for i, arg := range args { txBytes, err := stringOrHexToBytes(arg) @@ -633,15 +643,8 @@ func inTxArray(txByteArray [][]byte, tx []byte) bool { } return false } + func cmdPrepareProposal(cmd *cobra.Command, args []string) error { - if len(args) == 0 { - printResponse(cmd, args, response{ - Code: codeBad, - Info: "Must provide at least one transaction", - Log: "Must provide at least one transaction", - }) - return nil - } txsBytesArray := make([][]byte, len(args)) for i, arg := range args { @@ -682,6 +685,30 @@ func cmdPrepareProposal(cmd *cobra.Command, args []string) error { return nil } +func cmdProcessProposal(cmd *cobra.Command, args []string) error { + txsBytesArray := make([][]byte, len(args)) + + for i, arg := range args { + txBytes, err := stringOrHexToBytes(arg) + if err != nil { + return err + } + txsBytesArray[i] = txBytes + } + + res, err := client.ProcessProposal(cmd.Context(), &types.RequestProcessProposal{ + Txs: txsBytesArray, + }) + if err != nil { + return err + } + + printResponse(cmd, args, response{ + Status: int32(res.Status), + }) + return nil +} + func makeKVStoreCmd(logger log.Logger) func(*cobra.Command, []string) error { return func(cmd *cobra.Command, args []string) error { // Create the application - in memory or persisted to disk @@ -739,6 +766,9 @@ func printResponse(cmd *cobra.Command, args []string, rsps ...response) { if rsp.Log != "" { fmt.Printf("-> log: %s\n", rsp.Log) } + if cmd.Use == "process_proposal" { + fmt.Printf("-> status: %s\n", types.ResponseProcessProposal_ProposalStatus_name[rsp.Status]) + } if rsp.Query != nil { fmt.Printf("-> height: %d\n", rsp.Query.Height) diff --git a/abci/example/kvstore/kvstore.go b/abci/example/kvstore/kvstore.go index c9a2a148c..c1005eb86 100644 --- a/abci/example/kvstore/kvstore.go +++ b/abci/example/kvstore/kvstore.go @@ -293,7 +293,7 @@ func (app *Application) PrepareProposal(_ context.Context, req *types.RequestPre func (*Application) ProcessProposal(_ context.Context, req *types.RequestProcessProposal) (*types.ResponseProcessProposal, error) { for _, tx := range req.Txs { - if len(tx) == 0 { + if len(tx) == 0 || isPrepareTx(tx) { return &types.ResponseProcessProposal{Status: types.ResponseProcessProposal_REJECT}, nil } } diff --git a/abci/tests/server/client.go b/abci/tests/server/client.go index 7762c8d03..eb5649d9e 100644 --- a/abci/tests/server/client.go +++ b/abci/tests/server/client.go @@ -83,6 +83,19 @@ func PrepareProposal(ctx context.Context, client abciclient.Client, txBytes [][] fmt.Println("Passed test: PrepareProposal") return nil } + +func ProcessProposal(ctx context.Context, client abciclient.Client, txBytes [][]byte, statusExp types.ResponseProcessProposal_ProposalStatus) error { + res, _ := client.ProcessProposal(ctx, &types.RequestProcessProposal{Txs: txBytes}) + if res.Status != statusExp { + fmt.Println("Failed test: ProcessProposal") + fmt.Printf("ProcessProposal response status was unexpected. Got %v expected %v.", + res.Status, statusExp) + return errors.New("ProcessProposal error") + } + fmt.Println("Passed test: ProcessProposal") + return nil +} + func CheckTx(ctx context.Context, client abciclient.Client, txBytes []byte, codeExp uint32, dataExp []byte) error { res, _ := client.CheckTx(ctx, &types.RequestCheckTx{Tx: txBytes}) code, data := res.Code, res.Data diff --git a/abci/tests/test_cli/ex1.abci b/abci/tests/test_cli/ex1.abci index dc9e213ec..eba06028a 100644 --- a/abci/tests/test_cli/ex1.abci +++ b/abci/tests/test_cli/ex1.abci @@ -1,6 +1,7 @@ echo hello info prepare_proposal "abc" +process_proposal "abc" finalize_block "abc" commit info @@ -8,4 +9,10 @@ query "abc" finalize_block "def=xyz" "ghi=123" commit query "def" -prepare_proposal "preparedef" \ No newline at end of file +prepare_proposal "preparedef" +process_proposal "def" +process_proposal "preparedef" +prepare_proposal +process_proposal +finalize_block +commit \ No newline at end of file diff --git a/abci/tests/test_cli/ex1.abci.out b/abci/tests/test_cli/ex1.abci.out index f4f342dfb..fb3ababfa 100644 --- a/abci/tests/test_cli/ex1.abci.out +++ b/abci/tests/test_cli/ex1.abci.out @@ -12,6 +12,10 @@ -> code: OK -> log: Succeeded. Tx: abc action: UNMODIFIED +> process_proposal "abc" +-> code: OK +-> status: ACCEPT + > finalize_block "abc" -> code: OK -> code: OK @@ -58,3 +62,24 @@ -> code: OK -> log: Succeeded. Tx: preparedef action: REMOVED +> process_proposal "def" +-> code: OK +-> status: ACCEPT + +> process_proposal "preparedef" +-> code: OK +-> status: REJECT + +> prepare_proposal + +> process_proposal +-> code: OK +-> status: ACCEPT + +> finalize_block +-> code: OK +-> data.hex: 0x0600000000000000 + +> commit +-> code: OK + diff --git a/docs/app-dev/abci-cli.md b/docs/app-dev/abci-cli.md index 109e3b9fb..bb94a36ca 100644 --- a/docs/app-dev/abci-cli.md +++ b/docs/app-dev/abci-cli.md @@ -208,6 +208,33 @@ Try running these commands: -> key.hex: 646566 -> value: xyz -> value.hex: 78797A + +> prepare_proposal "preparedef" +-> code: OK +-> log: Succeeded. Tx: def action: ADDED +-> code: OK +-> log: Succeeded. Tx: preparedef action: REMOVED + +> process_proposal "def" +-> code: OK +-> status: ACCEPT + +> process_proposal "preparedef" +-> code: OK +-> status: REJECT + +> prepare_proposal + +> process_proposal +-> code: OK +-> status: ACCEPT + +> finalize_block +-> code: OK +-> data.hex: 0x0600000000000000 + +> commit +-> code: OK ``` Note that if we do `finalize_block "abc" ...` it will store `(abc, abc)`, but if From 2b5329ae47a092941c18b3663ebedf0d387abce7 Mon Sep 17 00:00:00 2001 From: Sergio Mena Date: Wed, 6 Jul 2022 00:02:29 +0200 Subject: [PATCH 56/61] Typos in spec (#8939) --- spec/abci++/abci++_basic_concepts.md | 4 ++-- spec/abci++/abci++_methods.md | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/spec/abci++/abci++_basic_concepts.md b/spec/abci++/abci++_basic_concepts.md index a467b623e..9b9167865 100644 --- a/spec/abci++/abci++_basic_concepts.md +++ b/spec/abci++/abci++_basic_concepts.md @@ -32,7 +32,7 @@ now better understood than when ABCI was first written. For example, many ideas scalability can be boiled down to "make the block proposers do work, so the network does not have to". This includes optimizations such as transaction level signature aggregation, state transition proofs, etc. Furthermore, many new security properties cannot be achieved in the current paradigm, -as the Application cannot require validators to do more than execute the transactions contained in +as the Application cannot require validators to do more than executing the transactions contained in finalized blocks. This includes features such as threshold cryptography, and guaranteed IBC connection attempts. @@ -312,7 +312,7 @@ Sources of non-determinism in applications may include: See [#56](https://github.com/tendermint/abci/issues/56) for the original discussion. -Note that some methods (`Query, CheckTx, FinalizeBlock`) return non-deterministic data in the form +Note that some methods (`Query, FinalizeBlock`) return non-deterministic data in the form of `Info` and `Log` fields. The `Log` is intended for the literal output from the Application's logger, while the `Info` is any additional info that should be returned. These are the only fields that are not included in block header computations, so we don't need agreement diff --git a/spec/abci++/abci++_methods.md b/spec/abci++/abci++_methods.md index 9d33652dd..4d1e47bef 100644 --- a/spec/abci++/abci++_methods.md +++ b/spec/abci++/abci++_methods.md @@ -669,7 +669,7 @@ message for round _r_, height _h_ from validator _q_ (_q_ ≠ _p_): to determine rewards and punishments for the validators. * The Application executes the transactions in `RequestFinalizeBlock.txs` deterministically, according to the rules set up by the Application, before returning control to Tendermint. - Alternatively, it can commit the candidate state corresponding to the same block previously + Alternatively, it can apply the candidate state corresponding to the same block previously executed via `PrepareProposal` or `ProcessProposal`. * `ResponseFinalizeBlock.tx_results[i].Code == 0` only if the _i_-th transaction is fully valid. * In next-block execution mode, the Application must provide values for `ResponseFinalizeBlock.app_hash`, @@ -830,8 +830,8 @@ Most of the data structures used in ABCI are shared [common data structures](../ | height | uint64 | The height at which the snapshot was taken (after commit). | 1 | | format | uint32 | An application-specific snapshot format, allowing applications to version their snapshot data format and make backwards-incompatible changes. Tendermint does not interpret this. | 2 | | chunks | uint32 | The number of chunks in the snapshot. Must be at least 1 (even if empty). | 3 | - | hash | bytes | TAn arbitrary snapshot hash. Must be equal only for identical snapshots across nodes. Tendermint does not interpret the hash, it only compares them. | 3 | - | metadata | bytes | Arbitrary application metadata, for example chunk hashes or other verification data. | 3 | + | hash | bytes | An arbitrary snapshot hash. Must be equal only for identical snapshots across nodes. Tendermint does not interpret the hash, it only compares them. | 4 | + | metadata | bytes | Arbitrary application metadata, for example chunk hashes or other verification data. | 5 | * **Usage**: * Used for state sync snapshots, see the [state sync section](../p2p/messages/state-sync.md) for details. From 70e7372c4ac518c740458f7873ba9b4409a44751 Mon Sep 17 00:00:00 2001 From: samricotta <37125168+samricotta@users.noreply.github.com> Date: Wed, 6 Jul 2022 14:30:45 +0200 Subject: [PATCH 57/61] added name to CO (#8947) Co-authored-by: Samantha Ricotta --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 950ed1dc2..05357c87c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -7,7 +7,7 @@ # global owners are only requested if there isn't a more specific # codeowner specified below. For this reason, the global codeowners # are often repeated in package-level definitions. -* @ebuchman @cmwaters @tychoish @williambanfield @creachadair @sergio-mena @jmalicevic @thanethomson @ancazamfir +* @ebuchman @cmwaters @tychoish @williambanfield @creachadair @sergio-mena @jmalicevic @thanethomson @ancazamfir @samricotta # Spec related changes can be approved by the protocol design team /spec @josef-widder @milosevic @cason @sergio-mena @jmalicevic From be6d74e6579bad8709def9716268f12f460df93d Mon Sep 17 00:00:00 2001 From: yihuang Date: Thu, 7 Jul 2022 02:05:48 +0800 Subject: [PATCH 58/61] Work around indexing problem for duplicate transactions (forward port: #8625) (#8945) Port the bug fix terra-money#76 to upstream. This is critical for ethermint json-rpc to work. fix: prevent duplicate tx index if it succeeded before fix: use CodeTypeOk instead of 0 fix: handle duplicate txs within the same block Co-authored-by: jess jesse@soob.co ref: #5281 Co-authored-by: M. J. Fromberger --- CHANGELOG_PENDING.md | 1 + internal/state/indexer/indexer_service.go | 52 +++++- .../state/indexer/indexer_service_test.go | 159 ++++++++++++++++++ 3 files changed, 211 insertions(+), 1 deletion(-) diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 13eeaef16..9936e1309 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -101,3 +101,4 @@ Special thanks to external contributors on this release: - [cli] \#8276 scmigrate: ensure target key is correctly renamed. (@creachadair) - [cli] \#8294 keymigrate: ensure block hash keys are correctly translated. (@creachadair) - [cli] \#8352 keymigrate: ensure transaction hash keys are correctly translated. (@creachadair) +- (indexer) \#8625 Fix overriding tx index of duplicated txs. diff --git a/internal/state/indexer/indexer_service.go b/internal/state/indexer/indexer_service.go index e73e4a3ba..d6db82806 100644 --- a/internal/state/indexer/indexer_service.go +++ b/internal/state/indexer/indexer_service.go @@ -4,6 +4,7 @@ import ( "context" "time" + abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/internal/eventbus" "github.com/tendermint/tendermint/internal/pubsub" "github.com/tendermint/tendermint/libs/log" @@ -96,7 +97,14 @@ func (is *Service) publish(msg pubsub.Message) error { if curr.Size() != 0 { start := time.Now() - err := sink.IndexTxEvents(curr.Ops) + + var err error + curr.Ops, err = DeduplicateBatch(curr.Ops, sink) + if err != nil { + is.logger.Error("failed to deduplicate batch", "height", is.currentBlock.height, "error", err) + } + + err = sink.IndexTxEvents(curr.Ops) if err != nil { is.logger.Error("failed to index block txs", "height", is.currentBlock.height, "err", err) @@ -169,3 +177,45 @@ func IndexingEnabled(sinks []EventSink) bool { return false } + +// DeduplicateBatch consider the case of duplicate txs. +// if the current one under investigation is NOT OK, then we need to check +// whether there's a previously indexed tx. +// SKIP the current tx if the previously indexed record is found and successful. +func DeduplicateBatch(ops []*abci.TxResult, sink EventSink) ([]*abci.TxResult, error) { + result := make([]*abci.TxResult, 0, len(ops)) + + // keep track of successful txs in this block in order to suppress latter ones being indexed. + var successfulTxsInThisBlock = make(map[string]struct{}) + + for _, txResult := range ops { + hash := types.Tx(txResult.Tx).Hash() + + if txResult.Result.IsOK() { + successfulTxsInThisBlock[string(hash)] = struct{}{} + } else { + // if it already appeared in current block and was successful, skip. + if _, found := successfulTxsInThisBlock[string(hash)]; found { + continue + } + + // check if this tx hash is already indexed + old, err := sink.GetTxByHash(hash) + + // if db op errored + // Not found is not an error + if err != nil { + return nil, err + } + + // if it's already indexed in an older block and was successful, skip. + if old != nil && old.Result.Code == abci.CodeTypeOK { + continue + } + } + + result = append(result, txResult) + } + + return result, nil +} diff --git a/internal/state/indexer/indexer_service_test.go b/internal/state/indexer/indexer_service_test.go index 6126ae259..6dc1bdf50 100644 --- a/internal/state/indexer/indexer_service_test.go +++ b/internal/state/indexer/indexer_service_test.go @@ -109,6 +109,165 @@ func TestIndexerServiceIndexesBlocks(t *testing.T) { assert.Nil(t, teardown(t, pool)) } +func TestTxIndexDuplicatedTx(t *testing.T) { + var mockTx = types.Tx("MOCK_TX_HASH") + + testCases := []struct { + name string + tx1 abci.TxResult + tx2 abci.TxResult + expSkip bool // do we expect the second tx to be skipped by tx indexer + }{ + {"skip, previously successful", + abci.TxResult{ + Height: 1, + Index: 0, + Tx: mockTx, + Result: abci.ExecTxResult{ + Code: abci.CodeTypeOK, + }, + }, + abci.TxResult{ + Height: 2, + Index: 0, + Tx: mockTx, + Result: abci.ExecTxResult{ + Code: abci.CodeTypeOK + 1, + }, + }, + true, + }, + {"not skip, previously unsuccessful", + abci.TxResult{ + Height: 1, + Index: 0, + Tx: mockTx, + Result: abci.ExecTxResult{ + Code: abci.CodeTypeOK + 1, + }, + }, + abci.TxResult{ + Height: 2, + Index: 0, + Tx: mockTx, + Result: abci.ExecTxResult{ + Code: abci.CodeTypeOK + 1, + }, + }, + false, + }, + {"not skip, both successful", + abci.TxResult{ + Height: 1, + Index: 0, + Tx: mockTx, + Result: abci.ExecTxResult{ + Code: abci.CodeTypeOK, + }, + }, + abci.TxResult{ + Height: 2, + Index: 0, + Tx: mockTx, + Result: abci.ExecTxResult{ + Code: abci.CodeTypeOK, + }, + }, + false, + }, + {"not skip, both unsuccessful", + abci.TxResult{ + Height: 1, + Index: 0, + Tx: mockTx, + Result: abci.ExecTxResult{ + Code: abci.CodeTypeOK + 1, + }, + }, + abci.TxResult{ + Height: 2, + Index: 0, + Tx: mockTx, + Result: abci.ExecTxResult{ + Code: abci.CodeTypeOK + 1, + }, + }, + false, + }, + {"skip, same block, previously successful", + abci.TxResult{ + Height: 1, + Index: 0, + Tx: mockTx, + Result: abci.ExecTxResult{ + Code: abci.CodeTypeOK, + }, + }, + abci.TxResult{ + Height: 1, + Index: 0, + Tx: mockTx, + Result: abci.ExecTxResult{ + Code: abci.CodeTypeOK + 1, + }, + }, + true, + }, + {"not skip, same block, previously unsuccessful", + abci.TxResult{ + Height: 1, + Index: 0, + Tx: mockTx, + Result: abci.ExecTxResult{ + Code: abci.CodeTypeOK + 1, + }, + }, + abci.TxResult{ + Height: 1, + Index: 0, + Tx: mockTx, + Result: abci.ExecTxResult{ + Code: abci.CodeTypeOK, + }, + }, + false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + sink := kv.NewEventSink(dbm.NewMemDB()) + + if tc.tx1.Height != tc.tx2.Height { + // index the first tx + err := sink.IndexTxEvents([]*abci.TxResult{&tc.tx1}) + require.NoError(t, err) + + // check if the second one should be skipped. + ops, err := indexer.DeduplicateBatch([]*abci.TxResult{&tc.tx2}, sink) + require.NoError(t, err) + + if tc.expSkip { + require.Empty(t, ops) + } else { + require.Equal(t, []*abci.TxResult{&tc.tx2}, ops) + } + } else { + // same block + ops := []*abci.TxResult{&tc.tx1, &tc.tx2} + ops, err := indexer.DeduplicateBatch(ops, sink) + require.NoError(t, err) + if tc.expSkip { + // the second one is skipped + require.Equal(t, []*abci.TxResult{&tc.tx1}, ops) + } else { + require.Equal(t, []*abci.TxResult{&tc.tx1, &tc.tx2}, ops) + } + } + }) + } +} + func readSchema() ([]*schema.Migration, error) { filename := "./sink/psql/schema.sql" contents, err := os.ReadFile(filename) From 27c523dccbe7f2f577868ee871b0042c1fef9a53 Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Wed, 6 Jul 2022 20:17:56 +0200 Subject: [PATCH 59/61] mempool: return error when mempool is full and inbound tx is rejected (#8942) Addresses: https://github.com/tendermint/tendermint/issues/8928 --- internal/mempool/mempool.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/mempool/mempool.go b/internal/mempool/mempool.go index caee97ff8..2398180fc 100644 --- a/internal/mempool/mempool.go +++ b/internal/mempool/mempool.go @@ -540,7 +540,7 @@ func (txmp *TxMempool) initTxCallback(wtx *WrappedTx, res *abci.ResponseCheckTx, ) if len(evictTxs) == 0 { // No room for the new incoming transaction so we just remove it from - // the cache. + // the cache and return an error to the user. txmp.cache.Remove(wtx.tx) txmp.logger.Error( "rejected incoming good transaction; mempool full", @@ -548,7 +548,7 @@ func (txmp *TxMempool) initTxCallback(wtx *WrappedTx, res *abci.ResponseCheckTx, "err", err.Error(), ) txmp.metrics.RejectedTxs.Add(1) - return nil + return err } // evict an existing transaction(s) From d1a16e8ff0393b80d45eb7299cac314570e48f8c Mon Sep 17 00:00:00 2001 From: Sam Kleinman Date: Thu, 7 Jul 2022 12:13:52 -0400 Subject: [PATCH 60/61] p2p: simpler priority queue (#8929) --- internal/p2p/channel.go | 4 ++ internal/p2p/pqueue.go | 14 +++-- internal/p2p/router.go | 13 ++++- internal/p2p/rqueue.go | 112 ++++++++++++++++++++++++++++++++++++ internal/p2p/rqueue_test.go | 47 +++++++++++++++ 5 files changed, 183 insertions(+), 7 deletions(-) create mode 100644 internal/p2p/rqueue.go create mode 100644 internal/p2p/rqueue_test.go diff --git a/internal/p2p/channel.go b/internal/p2p/channel.go index d6763543a..e33e7faa7 100644 --- a/internal/p2p/channel.go +++ b/internal/p2p/channel.go @@ -19,6 +19,10 @@ type Envelope struct { ChannelID ChannelID } +func (e Envelope) IsZero() bool { + return e.From == "" && e.To == "" && e.Message == nil +} + // Wrapper is a Protobuf message that can contain a variety of inner messages // (e.g. via oneof fields). If a Channel's message type implements Wrapper, the // Router will automatically wrap outbound messages and unwrap inbound messages, diff --git a/internal/p2p/pqueue.go b/internal/p2p/pqueue.go index 268daa8de..3cd1c897a 100644 --- a/internal/p2p/pqueue.go +++ b/internal/p2p/pqueue.go @@ -31,8 +31,16 @@ func (pq priorityQueue) get(i int) *pqEnvelope { return pq[i] } func (pq priorityQueue) Len() int { return len(pq) } func (pq priorityQueue) Less(i, j int) bool { - // if both elements have the same priority, prioritize based on most recent + // if both elements have the same priority, prioritize based + // on most recent and largest if pq[i].priority == pq[j].priority { + diff := pq[i].timestamp.Sub(pq[j].timestamp) + if diff < 0 { + diff *= -1 + } + if diff < 10*time.Millisecond { + return pq[i].size > pq[j].size + } return pq[i].timestamp.After(pq[j].timestamp) } @@ -272,12 +280,10 @@ func (s *pqScheduler) process(ctx context.Context) { } func (s *pqScheduler) push(pqEnv *pqEnvelope) { - chIDStr := strconv.Itoa(int(pqEnv.envelope.ChannelID)) - // enqueue the incoming Envelope heap.Push(s.pq, pqEnv) s.size += pqEnv.size - s.metrics.PeerQueueMsgSize.With("ch_id", chIDStr).Add(float64(pqEnv.size)) + s.metrics.PeerQueueMsgSize.With("ch_id", strconv.Itoa(int(pqEnv.envelope.ChannelID))).Add(float64(pqEnv.size)) // Update the cumulative sizes by adding the Envelope's size to every // priority less than or equal to it. diff --git a/internal/p2p/router.go b/internal/p2p/router.go index 8b7db9a03..0e55049c1 100644 --- a/internal/p2p/router.go +++ b/internal/p2p/router.go @@ -68,8 +68,9 @@ type RouterOptions struct { } const ( - queueTypeFifo = "fifo" - queueTypePriority = "priority" + queueTypeFifo = "fifo" + queueTypePriority = "priority" + queueTypeSimplePriority = "simple-priority" ) // Validate validates router options. @@ -77,7 +78,7 @@ func (o *RouterOptions) Validate() error { switch o.QueueType { case "": o.QueueType = queueTypeFifo - case queueTypeFifo, queueTypePriority: + case queueTypeFifo, queueTypePriority, queueTypeSimplePriority: // pass default: return fmt.Errorf("queue type %q is not supported", o.QueueType) @@ -227,6 +228,9 @@ func (r *Router) createQueueFactory(ctx context.Context) (func(int) queue, error return q }, nil + case queueTypeSimplePriority: + return func(size int) queue { return newSimplePriorityQueue(ctx, size, r.chDescs) }, nil + default: return nil, fmt.Errorf("cannot construct queue of type %q", r.options.QueueType) } @@ -304,6 +308,9 @@ func (r *Router) routeChannel( for { select { case envelope := <-outCh: + if envelope.IsZero() { + continue + } // Mark the envelope with the channel ID to allow sendPeer() to pass // it on to Transport.SendMessage(). envelope.ChannelID = chID diff --git a/internal/p2p/rqueue.go b/internal/p2p/rqueue.go new file mode 100644 index 000000000..8d6406864 --- /dev/null +++ b/internal/p2p/rqueue.go @@ -0,0 +1,112 @@ +package p2p + +import ( + "container/heap" + "context" + "sort" + "time" + + "github.com/gogo/protobuf/proto" +) + +type simpleQueue struct { + input chan Envelope + output chan Envelope + closeFn func() + closeCh <-chan struct{} + + maxSize int + chDescs []*ChannelDescriptor +} + +func newSimplePriorityQueue(ctx context.Context, size int, chDescs []*ChannelDescriptor) *simpleQueue { + if size%2 != 0 { + size++ + } + + ctx, cancel := context.WithCancel(ctx) + q := &simpleQueue{ + input: make(chan Envelope, size*2), + output: make(chan Envelope, size/2), + maxSize: size * size, + closeCh: ctx.Done(), + closeFn: cancel, + } + + go q.run(ctx) + return q +} + +func (q *simpleQueue) enqueue() chan<- Envelope { return q.input } +func (q *simpleQueue) dequeue() <-chan Envelope { return q.output } +func (q *simpleQueue) close() { q.closeFn() } +func (q *simpleQueue) closed() <-chan struct{} { return q.closeCh } + +func (q *simpleQueue) run(ctx context.Context) { + defer q.closeFn() + + var chPriorities = make(map[ChannelID]uint, len(q.chDescs)) + for _, chDesc := range q.chDescs { + chID := chDesc.ID + chPriorities[chID] = uint(chDesc.Priority) + } + + pq := make(priorityQueue, 0, q.maxSize) + heap.Init(&pq) + ticker := time.NewTicker(10 * time.Millisecond) + // must have a buffer of exactly one because both sides of + // this channel are used in this loop, and simply signals adds + // to the heap + signal := make(chan struct{}, 1) + for { + select { + case <-ctx.Done(): + return + case <-q.closeCh: + return + case e := <-q.input: + // enqueue the incoming Envelope + heap.Push(&pq, &pqEnvelope{ + envelope: e, + size: uint(proto.Size(e.Message)), + priority: chPriorities[e.ChannelID], + timestamp: time.Now().UTC(), + }) + + select { + case signal <- struct{}{}: + default: + if len(pq) > q.maxSize { + sort.Sort(pq) + pq = pq[:q.maxSize] + } + } + + case <-ticker.C: + if len(pq) > q.maxSize { + sort.Sort(pq) + pq = pq[:q.maxSize] + } + if len(pq) > 0 { + select { + case signal <- struct{}{}: + default: + } + } + case <-signal: + SEND: + for len(pq) > 0 { + select { + case <-ctx.Done(): + return + case <-q.closeCh: + return + case q.output <- heap.Pop(&pq).(*pqEnvelope).envelope: + continue SEND + default: + break SEND + } + } + } + } +} diff --git a/internal/p2p/rqueue_test.go b/internal/p2p/rqueue_test.go new file mode 100644 index 000000000..43c4066e5 --- /dev/null +++ b/internal/p2p/rqueue_test.go @@ -0,0 +1,47 @@ +package p2p + +import ( + "context" + "testing" + "time" +) + +func TestSimpleQueue(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // set up a small queue with very small buffers so we can + // watch it shed load, then send a bunch of messages to the + // queue, most of which we'll watch it drop. + sq := newSimplePriorityQueue(ctx, 1, nil) + for i := 0; i < 100; i++ { + sq.enqueue() <- Envelope{From: "merlin"} + } + + seen := 0 + +RETRY: + for seen <= 2 { + select { + case e := <-sq.dequeue(): + if e.From != "merlin" { + continue + } + seen++ + case <-time.After(10 * time.Millisecond): + break RETRY + } + } + // if we don't see any messages, then it's just broken. + if seen == 0 { + t.Errorf("seen %d messages, should have seen more than one", seen) + } + // ensure that load shedding happens: there can be at most 3 + // messages that we get out of this, one that was buffered + // plus 2 that were under the cap, everything else gets + // dropped. + if seen > 3 { + t.Errorf("saw %d messages, should have seen 5 or fewer", seen) + } + +} From 636320f9010be75c0ae804714ddce62b46f17930 Mon Sep 17 00:00:00 2001 From: Sam Kleinman Date: Thu, 7 Jul 2022 12:29:50 -0400 Subject: [PATCH 61/61] p2p: delete cruft (#8958) I think the decision in #8806 is that we shouldn't do this yet, so I think it's best to just drop this. --- internal/p2p/peermanager.go | 41 ------------------------------------- 1 file changed, 41 deletions(-) diff --git a/internal/p2p/peermanager.go b/internal/p2p/peermanager.go index 8230b262e..fabff390e 100644 --- a/internal/p2p/peermanager.go +++ b/internal/p2p/peermanager.go @@ -1419,47 +1419,6 @@ func (s *peerStore) Ranked() []*peerInfo { } sort.Slice(s.ranked, func(i, j int) bool { return s.ranked[i].Score() > s.ranked[j].Score() - // TODO: reevaluate more wholistic sorting, perhaps as follows: - - // // sort inactive peers after active peers - // if s.ranked[i].Inactive && !s.ranked[j].Inactive { - // return false - // } else if !s.ranked[i].Inactive && s.ranked[j].Inactive { - // return true - // } - - // iLastDialed, iLastDialSuccess := s.ranked[i].LastDialed() - // jLastDialed, jLastDialSuccess := s.ranked[j].LastDialed() - - // // sort peers who our most recent dialing attempt was - // // successful ahead of peers with recent dialing - // // failures - // switch { - // case iLastDialSuccess && jLastDialSuccess: - // // if both peers were (are?) successfully - // // connected, convey their score, but give the - // // one we dialed successfully most recently a bonus - - // iScore := s.ranked[i].Score() - // jScore := s.ranked[j].Score() - // if jLastDialed.Before(iLastDialed) { - // jScore++ - // } else { - // iScore++ - // } - - // return iScore > jScore - // case iLastDialSuccess: - // return true - // case jLastDialSuccess: - // return false - // default: - // // if both peers were not successful in their - // // most recent dialing attempt, fall back to - // // peer score. - - // return s.ranked[i].Score() > s.ranked[j].Score() - // } }) return s.ranked }