Merge from master and fix conflicts

Signed-off-by: Thane Thomson <connect@thanethomson.com>

scripts/confix/condiff/condiff.go

@@ -0,0 +1,152 @@
+// Program condiff performs a keyspace diff on two TOML documents.
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/creachadair/tomledit"
+	"github.com/creachadair/tomledit/parser"
+	"github.com/creachadair/tomledit/transform"
+)
+
+var (
+	doDesnake = flag.Bool("desnake", false, "Convert snake_case to kebab-case before comparing")
+)
+
+func init() {
+	flag.Usage = func() {
+		fmt.Fprintf(os.Stderr, `Usage: %[1]s [options] f1 f2
+
+Diff the keyspaces of the TOML documents in files f1 and f2.
+The output prints one line per key that differs:
+
+   -S name    -- section exists in f1 but not f2
+   +S name    -- section exists in f2 but not f1
+   -M name    -- mapping exists in f1 but not f2
+   +M name    -- mapping exists in f2 but not f1
+
+Comments, order, and values are ignored for comparison purposes.
+
+Options:
+`, filepath.Base(os.Args[0]))
+		flag.PrintDefaults()
+	}
+}
+
+func main() {
+	flag.Parse()
+
+	if flag.NArg() != 2 {
+		log.Fatalf("Usage: %[1]s <lhs> <rhs>", filepath.Base(os.Args[0]))
+	}
+	lhs := mustParse(flag.Arg(0))
+	rhs := mustParse(flag.Arg(1))
+	if *doDesnake {
+		log.Printf("Converting all names from snake_case to kebab-case")
+		fix := transform.SnakeToKebab()
+		_ = fix(context.Background(), lhs)
+		_ = fix(context.Background(), rhs)
+	}
+	diffDocs(os.Stdout, lhs, rhs)
+}
+
+func mustParse(path string) *tomledit.Document {
+	f, err := os.Open(path)
+	if err != nil {
+		log.Fatalf("Opening TOML input: %v", err)
+	}
+	defer f.Close()
+	doc, err := tomledit.Parse(f)
+	if err != nil {
+		log.Fatalf("Parsing %q: %v", path, err)
+	}
+	return doc
+}
+
+func allKeys(s *tomledit.Section) []string {
+	var keys []string
+	s.Scan(func(key parser.Key, _ *tomledit.Entry) bool {
+		keys = append(keys, key.String())
+		return true
+	})
+	return keys
+}
+
+const (
+	delSection = "-S"
+	delMapping = "-M"
+	addSection = "+S"
+	addMapping = "+M"
+
+	delMapSep = "\n" + delMapping + " "
+	addMapSep = "\n" + addMapping + " "
+)
+
+func diffDocs(w io.Writer, lhs, rhs *tomledit.Document) {
+	diffSections(w, lhs.Global, rhs.Global)
+	lsec, rsec := lhs.Sections, rhs.Sections
+	transform.SortSectionsByName(lsec)
+	transform.SortSectionsByName(rsec)
+
+	i, j := 0, 0
+	for i < len(lsec) && j < len(rsec) {
+		if lsec[i].Name.Before(rsec[j].Name) {
+			fmt.Fprintln(w, delSection, lsec[i].Name)
+			fmt.Fprintln(w, delMapping, strings.Join(allKeys(lsec[i]), delMapSep))
+			i++
+		} else if rsec[j].Name.Before(lsec[i].Name) {
+			fmt.Fprintln(w, addSection, rsec[j].Name)
+			fmt.Fprintln(w, addMapping, strings.Join(allKeys(rsec[j]), addMapSep))
+			j++
+		} else {
+			diffSections(w, lsec[i], rsec[j])
+			i++
+			j++
+		}
+	}
+	for ; i < len(lsec); i++ {
+		fmt.Fprintln(w, delSection, lsec[i].Name)
+		fmt.Fprintln(w, delMapping, strings.Join(allKeys(lsec[i]), delMapSep))
+	}
+	for ; j < len(rsec); j++ {
+		fmt.Fprintln(w, addSection, rsec[j].Name)
+		fmt.Fprintln(w, addMapping, strings.Join(allKeys(rsec[j]), addMapSep))
+	}
+}
+
+func diffSections(w io.Writer, lhs, rhs *tomledit.Section) {
+	diffKeys(w, allKeys(lhs), allKeys(rhs))
+}
+
+func diffKeys(w io.Writer, lhs, rhs []string) {
+	sort.Strings(lhs)
+	sort.Strings(rhs)
+
+	i, j := 0, 0
+	for i < len(lhs) && j < len(rhs) {
+		if lhs[i] < rhs[j] {
+			fmt.Fprintln(w, delMapping, lhs[i])
+			i++
+		} else if lhs[i] > rhs[j] {
+			fmt.Fprintln(w, addMapping, rhs[j])
+			j++
+		} else {
+			i++
+			j++
+		}
+	}
+	for ; i < len(lhs); i++ {
+		fmt.Fprintln(w, delMapping, lhs[i])
+	}
+	for ; j < len(rhs); j++ {
+		fmt.Fprintln(w, addMapping, rhs[j])
+	}
+}

scripts/confix/confix.go

@@ -85,21 +85,138 @@ var plan = transform.Plan{
 		T:    transform.SnakeToKebab(),
 	},
 	{
-		// Since https://github.com/tendermint/tendermint/pull/6896.
-		Desc:    "Rename [fastsync] to [blocksync]",
-		T:       transform.Rename(parser.Key{"fastsync"}, parser.Key{"blocksync"}),
+		// [fastsync]  renamed in https://github.com/tendermint/tendermint/pull/6896.
+		// [blocksync] removed in https://github.com/tendermint/tendermint/pull/7159.
+		Desc: "Remove [fastsync] and [blocksync] sections",
+		T: transform.Func(func(_ context.Context, doc *tomledit.Document) error {
+			doc.First("fast-sync").Remove()
+			transform.FindTable(doc, "fastsync").Remove()
+			transform.FindTable(doc, "blocksync").Remove()
+			return nil
+		}),
 		ErrorOK: true,
 	},
 	{
-		// Since https://github.com/tendermint/tendermint/pull/7159.
-		Desc: "Move top-level fast_sync key to blocksync.enable",
-		T: transform.MoveKey(
-			parser.Key{"fast-sync"},
-			parser.Key{"blocksync"},
-			parser.Key{"enable"},
+		// Since https://github.com/tendermint/tendermint/pull/6241.
+		//
+		// TODO(creachadair): backport into v0.35.x.
+		Desc: `Add top-level mode setting (default "full")`,
+		T: transform.EnsureKey(nil, &parser.KeyValue{
+			Block: parser.Comments{"Mode of Node: full | validator | seed"},
+			Name:  parser.Key{"mode"},
+			Value: parser.MustValue(`"full"`),
+		}),
+		ErrorOK: true,
+	},
+	{
+		// Since https://github.com/tendermint/tendermint/pull/7121.
+		Desc: "Remove gRPC settings from the [rpc] section",
+		T: transform.Func(func(_ context.Context, doc *tomledit.Document) error {
+			doc.First("rpc", "grpc-laddr").Remove()
+			doc.First("rpc", "grpc-max-open-connections").Remove()
+			return nil
+		}),
+	},
+	{
+		// Since https://github.com/tendermint/tendermint/pull/8217.
+		Desc: "Remove per-node consensus timeouts (converted to consensus parameters)",
+		T: transform.Remove(
+			parser.Key{"consensus", "skip-timeout-commit"},
+			parser.Key{"consensus", "timeout-commit"},
+			parser.Key{"consensus", "timeout-precommit"},
+			parser.Key{"consensus", "timeout-precommit-delta"},
+			parser.Key{"consensus", "timeout-prevote"},
+			parser.Key{"consensus", "timeout-prevote-delta"},
+			parser.Key{"consensus", "timeout-propose"},
+			parser.Key{"consensus", "timeout-propose-delta"},
 		),
 		ErrorOK: true,
 	},
+	{
+		// Since https://github.com/tendermint/tendermint/pull/6396.
+		//
+		// TODO(creachadair): backport into v0.35.x.
+		Desc:    "Remove vestigial mempool.wal-dir setting",
+		T:       transform.Remove(parser.Key{"mempool", "wal-dir"}),
+		ErrorOK: true,
+	},
+	{
+		// Since https://github.com/tendermint/tendermint/pull/6323.
+		//
+		// TODO(creachadair): backport into v0.35.x.
+		Desc: "Add new [p2p] queue-type setting",
+		T: transform.EnsureKey(parser.Key{"p2p"}, &parser.KeyValue{
+			Block: parser.Comments{"Select the p2p internal queue"},
+			Name:  parser.Key{"queue-type"},
+			Value: parser.MustValue(`"priority"`),
+		}),
+		ErrorOK: true,
+	},
+	{
+		// Since https://github.com/tendermint/tendermint/pull/6353.
+		//
+		// TODO(creachadair): backport into v0.35.x.
+		Desc: "Add [p2p] connection count and rate limit settings",
+		T: transform.Func(func(_ context.Context, doc *tomledit.Document) error {
+			tab := transform.FindTable(doc, "p2p")
+			if tab == nil {
+				return errors.New("p2p table not found")
+			}
+			transform.InsertMapping(tab.Section, &parser.KeyValue{
+				Block: parser.Comments{"Maximum number of connections (inbound and outbound)."},
+				Name:  parser.Key{"max-connections"},
+				Value: parser.MustValue("64"),
+			}, false)
+			transform.InsertMapping(tab.Section, &parser.KeyValue{
+				Block: parser.Comments{
+					"Rate limits the number of incoming connection attempts per IP address.",
+				},
+				Name:  parser.Key{"max-incoming-connection-attempts"},
+				Value: parser.MustValue("100"),
+			}, false)
+			return nil
+		}),
+	},
+	{
+		// Added "chunk-fetchers" https://github.com/tendermint/tendermint/pull/6566.
+		// This value was backported into v0.34.11 (modulo casing).
+		// Renamed to "fetchers"  https://github.com/tendermint/tendermint/pull/6587.
+		//
+		// TODO(creachadair): backport into v0.35.x.
+		Desc: "Rename statesync.chunk-fetchers to statesync.fetchers",
+		T: transform.Func(func(ctx context.Context, doc *tomledit.Document) error {
+			// If the key already exists, rename it preserving its value.
+			if found := doc.First("statesync", "chunk-fetchers"); found != nil {
+				found.KeyValue.Name = parser.Key{"fetchers"}
+				return nil
+			}
+
+			// Otherwise, add it.
+			return transform.EnsureKey(parser.Key{"statesync"}, &parser.KeyValue{
+				Block: parser.Comments{
+					"The number of concurrent chunk and block fetchers to run (default: 4).",
+				},
+				Name:  parser.Key{"fetchers"},
+				Value: parser.MustValue("4"),
+			})(ctx, doc)
+		}),
+	},
+	{
+		// Since https://github.com/tendermint/tendermint/pull/6807.
+		// Backported into v0.34.13 (modulo casing).
+		//
+		// TODO(creachadair): backport into v0.35.x.
+		Desc: "Add statesync.use-p2p setting",
+		T: transform.EnsureKey(parser.Key{"statesync"}, &parser.KeyValue{
+			Block: parser.Comments{
+				"# State sync uses light client verification to verify state. This can be done either through the",
+				"# P2P layer or RPC layer. Set this to true to use the P2P layer. If false (default), RPC layer",
+				"# will be used.",
+			},
+			Name:  parser.Key{"use-p2p"},
+			Value: parser.MustValue("false"),
+		}),
+	},
 	{
 		// Since https://github.com/tendermint/tendermint/pull/6462.
 		Desc: "Move priv-validator settings under [priv-validator]",
@@ -147,22 +264,6 @@ var plan = transform.Plan{
 			return nil
 		}),
 	},
-	{
-		// v1 removed: https://github.com/tendermint/tendermint/pull/5728
-		// v2 deprecated: https://github.com/tendermint/tendermint/pull/6730
-		Desc: `Set blocksync.version to "v0"`,
-		T: transform.Func(func(_ context.Context, doc *tomledit.Document) error {
-			v := doc.First("blocksync", "version")
-			if v == nil {
-				return nil // nothing to do
-			} else if !v.IsMapping() {
-				// This shouldn't happen, but is easier to debug than a panic.
-				return fmt.Errorf("blocksync.version is weird: %v", v)
-			}
-			v.Value.X = parser.MustValue(`"v0"`).X
-			return nil
-		}),
-	},
 }
 
 // ApplyFixes transforms doc and reports whether it succeeded.
@@ -248,8 +349,5 @@ func CheckValid(data []byte) error {
 		return fmt.Errorf("decoding config: %w", err)
 	}
 
-	// Stub in required value not stored in the config file, so that validation
-	// will not fail spuriously.
-	cfg.Mode = config.ModeValidator
 	return cfg.ValidateBasic()
 }

scripts/confix/testdata/diff-31-32.txt

@@ -0,0 +1,5 @@
++S fastsync
++M fastsync.version
++M mempool.max-tx-bytes
++M rpc.max-body-bytes
++M rpc.max-header-bytes

scripts/confix/testdata/diff-32-33.txt

@@ -0,0 +1,6 @@
++M p2p.persistent-peers-max-dial-period
++M p2p.unconditional-peer-ids
++M tx-index.index-all-keys
+-M tx-index.index-all-tags
++M tx-index.index-keys
+-M tx-index.index-tags

scripts/confix/testdata/diff-33-34.txt

@@ -0,0 +1,20 @@
+-M prof-laddr
++M consensus.double-sign-check-height
++M mempool.keep-invalid-txs-in-cache
++M mempool.max-batch-bytes
++M rpc.experimental-close-on-slow-client
++M rpc.experimental-subscription-buffer-size
++M rpc.experimental-websocket-write-buffer-size
++M rpc.pprof-laddr
++S statesync
++M statesync.enable
++M statesync.rpc-servers
++M statesync.trust-height
++M statesync.trust-hash
++M statesync.trust-period
++M statesync.discovery-time
++M statesync.temp-dir
++M statesync.chunk-request-timeout
++M statesync.chunk-fetchers
+-M tx-index.index-all-keys
+-M tx-index.index-keys

scripts/confix/testdata/diff-34-35.txt

@@ -0,0 +1,31 @@
+-M fast-sync
++M mode
+-M priv-validator-key-file
+-M priv-validator-laddr
+-M priv-validator-state-file
++S blocksync
++M blocksync.enable
++M blocksync.version
+-S fastsync
+-M fastsync.version
++M mempool.ttl-duration
++M mempool.ttl-num-blocks
++M mempool.version
+-M mempool.wal-dir
++M p2p.bootstrap-peers
++M p2p.max-connections
++M p2p.max-incoming-connection-attempts
++M p2p.queue-type
+-M p2p.seed-mode
++M p2p.use-legacy
++S priv-validator
++M priv-validator.key-file
++M priv-validator.state-file
++M priv-validator.laddr
++M priv-validator.client-certificate-file
++M priv-validator.client-key-file
++M priv-validator.root-ca-file
+-M statesync.chunk-fetchers
++M statesync.fetchers
++M statesync.use-p2p
++M tx-index.psql-conn

scripts/confix/testdata/diff-35-36.txt

@@ -0,0 +1,27 @@
+-S blocksync
+-M blocksync.enable
+-M blocksync.version
+-M consensus.skip-timeout-commit
+-M consensus.timeout-commit
+-M consensus.timeout-precommit
+-M consensus.timeout-precommit-delta
+-M consensus.timeout-prevote
+-M consensus.timeout-prevote-delta
+-M consensus.timeout-propose
+-M consensus.timeout-propose-delta
+-M mempool.version
+-M p2p.addr-book-file
+-M p2p.addr-book-strict
+-M p2p.max-num-inbound-peers
+-M p2p.max-num-outbound-peers
+-M p2p.persistent-peers-max-dial-period
+-M p2p.unconditional-peer-ids
+-M p2p.use-legacy
++M rpc.event-log-max-items
++M rpc.event-log-window-size
+-M rpc.experimental-close-on-slow-client
++M rpc.experimental-disable-websocket
+-M rpc.experimental-subscription-buffer-size
+-M rpc.experimental-websocket-write-buffer-size
+-M rpc.grpc-laddr
+-M rpc.grpc-max-open-connections