Extract a library from the confix command-line tool. (backport #9012) (#9025)

(cherry picked from commit 18b5a500da)

Pull out the library functionality from scripts/confix and move it to
internal/libs/confix. Replace scripts/confix with a simple stub that has the
same command-line API, but uses the library instead.

Related:

- Move and update unit tests.
- Move scripts/confix/condiff to scripts/condiff.
- Update test data for v34, v35, and v36.
- Update reference diffs.
- Update testdata README.

Co-authored-by: M. J. Fromberger <fromberger@interchain.io>
This commit is contained in:
mergify[bot]
2022-07-15 08:46:28 -07:00
committed by GitHub
parent 9f2522148b
commit 6b18dfcea1
29 changed files with 452 additions and 130 deletions

View File

@@ -0,0 +1,155 @@
// Package confix applies changes to a Tendermint TOML configuration file, to
// update configurations created with an older version of Tendermint to a
// compatible format for a newer version.
package confix
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"os"
"github.com/creachadair/atomicfile"
"github.com/creachadair/tomledit"
"github.com/creachadair/tomledit/transform"
"github.com/spf13/viper"
"github.com/tendermint/tendermint/config"
)
// Upgrade reads the configuration file at configPath and applies any
// transformations necessary to upgrade it to the current version. If this
// succeeds, the transformed output is written to outputPath. As a special
// case, if outputPath == "" the output is written to stdout.
//
// It is safe if outputPath == inputPath. If a regular file outputPath already
// exists, it is overwritten. In case of error, the output is not written.
//
// Upgrade is a convenience wrapper for calls to LoadConfig, ApplyFixes, and
// CheckValid. If the caller requires more control over the behavior of the
// upgrade, call those functions directly.
func Upgrade(ctx context.Context, configPath, outputPath string) error {
if configPath == "" {
return errors.New("empty input configuration path")
}
doc, err := LoadConfig(configPath)
if err != nil {
return fmt.Errorf("loading config: %v", err)
}
if err := ApplyFixes(ctx, doc); err != nil {
return fmt.Errorf("updating %q: %v", configPath, err)
}
var buf bytes.Buffer
if err := tomledit.Format(&buf, doc); err != nil {
return fmt.Errorf("formatting config: %v", err)
}
// Verify that Tendermint can parse the results after our edits.
if err := CheckValid(buf.Bytes()); err != nil {
return fmt.Errorf("updated config is invalid: %v", err)
}
if outputPath == "" {
_, err = os.Stdout.Write(buf.Bytes())
} else {
err = atomicfile.WriteData(outputPath, buf.Bytes(), 0600)
}
return err
}
// ApplyFixes transforms doc and reports whether it succeeded.
func ApplyFixes(ctx context.Context, doc *tomledit.Document) error {
// Check what version of Tendermint might have created this config file, as
// a safety check for the updates we are about to make.
tmVersion := GuessConfigVersion(doc)
if tmVersion == vUnknown {
return errors.New("cannot tell what Tendermint version created this config")
} else if tmVersion < v34 || tmVersion > v36 {
// TODO(creachadair): Add in rewrites for older versions. This will
// require some digging to discover what the changes were. The upgrade
// instructions do not give specifics.
return fmt.Errorf("unable to update version %s config", tmVersion)
}
return plan.Apply(ctx, doc)
}
// LoadConfig loads and parses the TOML document from path.
func LoadConfig(path string) (*tomledit.Document, error) {
f, err := os.Open(path)
if err != nil {
return nil, err
}
defer f.Close()
return tomledit.Parse(f)
}
const (
vUnknown = ""
v32 = "v0.32"
v33 = "v0.33"
v34 = "v0.34"
v35 = "v0.35"
v36 = "v0.36"
)
// GuessConfigVersion attempts to figure out which version of Tendermint
// created the specified config document. It returns "" if the creating version
// cannot be determined, otherwise a string of the form "vX.YY".
func GuessConfigVersion(doc *tomledit.Document) string {
hasDisableWS := doc.First("rpc", "experimental-disable-websocket") != nil
hasUseLegacy := doc.First("p2p", "use-legacy") != nil // v0.35 only
if hasDisableWS && !hasUseLegacy {
return v36
}
hasBlockSync := transform.FindTable(doc, "blocksync") != nil // add: v0.35
hasStateSync := transform.FindTable(doc, "statesync") != nil // add: v0.34
if hasBlockSync && hasStateSync {
return v35
} else if hasStateSync {
return v34
}
hasIndexKeys := doc.First("tx_index", "index_keys") != nil // add: v0.33
hasIndexTags := doc.First("tx_index", "index_tags") != nil // rem: v0.33
if hasIndexKeys && !hasIndexTags {
return v33
}
hasFastSync := transform.FindTable(doc, "fastsync") != nil // add: v0.32
if hasIndexTags && hasFastSync {
return v32
}
// Something older, probably.
return vUnknown
}
// CheckValid checks whether the specified config appears to be a valid
// Tendermint config file. This emulates how the node loads the config.
func CheckValid(data []byte) error {
v := viper.New()
v.SetConfigType("toml")
if err := v.ReadConfig(bytes.NewReader(data)); err != nil {
return fmt.Errorf("reading config: %w", err)
}
var cfg config.Config
if err := v.Unmarshal(&cfg); err != nil {
return fmt.Errorf("decoding config: %w", err)
}
return cfg.ValidateBasic()
}
// WithLogWriter returns a child of ctx with a logger attached that sends
// output to w. This is a convenience wrapper for transform.WithLogWriter.
func WithLogWriter(ctx context.Context, w io.Writer) context.Context {
return transform.WithLogWriter(ctx, w)
}

View File

@@ -1,4 +1,4 @@
package main_test
package confix_test
import (
"bytes"
@@ -9,7 +9,7 @@ import (
"github.com/creachadair/tomledit"
"github.com/google/go-cmp/cmp"
confix "github.com/tendermint/tendermint/scripts/confix"
"github.com/tendermint/tendermint/internal/libs/confix"
)
func mustParseConfig(t *testing.T, path string) *tomledit.Document {

View File

@@ -1,4 +1,4 @@
package main
package confix
import (
"context"

View File

@@ -41,12 +41,12 @@ The files named `diff-XX-YY.txt` were generated by using the `condiff` tool on
the config samples for versions v0.XX and v0.YY:
```shell
go run ./scripts/confix/condiff -desnake vXX-config vYY-config.toml > diff-XX-YY.txt
go run ./scripts/condiff -desnake vXX-config vYY-config.toml > diff-XX-YY.txt
```
The `baseline.txt` was computed in the same way, but using an empty starting
file so that we capture all the settings in the target:
```shell
go run ./scripts/confix/condiff -desnake /dev/null v26-config.toml > baseline.txt
go run ./scripts/condiff -desnake /dev/null v26-config.toml > baseline.txt
```

View File

@@ -0,0 +1,5 @@
+S fastsync
+M fastsync.version
+M mempool.max-tx-bytes
+M rpc.max-body-bytes
+M rpc.max-header-bytes

View File

@@ -0,0 +1,6 @@
+M p2p.persistent-peers-max-dial-period
+M p2p.unconditional-peer-ids
+M tx-index.index-all-keys
-M tx-index.index-all-tags
+M tx-index.index-keys
-M tx-index.index-tags

View File

@@ -0,0 +1,20 @@
-M prof-laddr
+M consensus.double-sign-check-height
+M mempool.keep-invalid-txs-in-cache
+M mempool.max-batch-bytes
+M rpc.experimental-close-on-slow-client
+M rpc.experimental-subscription-buffer-size
+M rpc.experimental-websocket-write-buffer-size
+M rpc.pprof-laddr
+S statesync
+M statesync.enable
+M statesync.rpc-servers
+M statesync.trust-height
+M statesync.trust-hash
+M statesync.trust-period
+M statesync.discovery-time
+M statesync.temp-dir
+M statesync.chunk-request-timeout
+M statesync.chunk-fetchers
-M tx-index.index-all-keys
-M tx-index.index-keys

View File

@@ -0,0 +1,28 @@
-M fast-sync
+M mode
-M priv-validator-key-file
-M priv-validator-laddr
-M priv-validator-state-file
+S blocksync
+M blocksync.enable
+M blocksync.version
-S fastsync
-M fastsync.version
-M mempool.wal-dir
+M p2p.bootstrap-peers
+M p2p.max-connections
+M p2p.max-incoming-connection-attempts
+M p2p.max-outgoing-connections
+M p2p.queue-type
-M p2p.seed-mode
+M p2p.use-legacy
+S priv-validator
+M priv-validator.key-file
+M priv-validator.state-file
+M priv-validator.laddr
+M priv-validator.client-certificate-file
+M priv-validator.client-key-file
+M priv-validator.root-ca-file
-M statesync.chunk-fetchers
+M statesync.fetchers
+M statesync.use-p2p

View File

@@ -0,0 +1,29 @@
-S blocksync
-M blocksync.enable
-M blocksync.version
-M consensus.skip-timeout-commit
-M consensus.timeout-commit
-M consensus.timeout-precommit
-M consensus.timeout-precommit-delta
-M consensus.timeout-prevote
-M consensus.timeout-prevote-delta
-M consensus.timeout-propose
-M consensus.timeout-propose-delta
-M mempool.recheck
-M mempool.version
-M p2p.addr-book-file
-M p2p.addr-book-strict
-M p2p.max-num-inbound-peers
-M p2p.max-num-outbound-peers
-M p2p.persistent-peers-max-dial-period
-M p2p.seeds
-M p2p.unconditional-peer-ids
-M p2p.use-legacy
+M rpc.event-log-max-items
+M rpc.event-log-window-size
-M rpc.experimental-close-on-slow-client
+M rpc.experimental-disable-websocket
-M rpc.experimental-subscription-buffer-size
-M rpc.experimental-websocket-write-buffer-size
-M rpc.grpc-laddr
-M rpc.grpc-max-open-connections

View File

@@ -272,6 +272,11 @@ dial_timeout = "3s"
#######################################################
[mempool]
# Mempool version to use:
# 1) "v0" - (default) FIFO mempool.
# 2) "v1" - prioritized mempool.
version = "v0"
recheck = true
broadcast = true
wal_dir = ""
@@ -301,6 +306,22 @@ max_tx_bytes = 1048576
# XXX: Unused due to https://github.com/tendermint/tendermint/issues/5796
max_batch_bytes = 0
# ttl-duration, if non-zero, defines the maximum amount of time a transaction
# can exist for in the mempool.
#
# Note, if ttl-num-blocks is also defined, a transaction will be removed if it
# has existed in the mempool at least ttl-num-blocks number of blocks or if it's
# insertion time into the mempool is beyond ttl-duration.
ttl-duration = "0s"
# ttl-num-blocks, if non-zero, defines the maximum number of blocks a transaction
# can exist for in the mempool.
#
# Note, if ttl-duration is also defined, a transaction will be removed if it
# has existed in the mempool at least ttl-num-blocks number of blocks or if
# it's insertion time into the mempool is beyond ttl-duration.
ttl-num-blocks = 0
#######################################################
### State Sync Configuration Options ###
#######################################################
@@ -403,8 +424,14 @@ peer_query_maj23_sleep_duration = "2s"
# 1) "null"
# 2) "kv" (default) - the simplest possible indexer, backed by key-value storage (defaults to levelDB; see DBBackend).
# - When "kv" is chosen "tx.height" and "tx.hash" will always be indexed.
# 3) "psql" - the indexer services backed by PostgreSQL.
# When "kv" or "psql" is chosen "tx.height" and "tx.hash" will always be indexed.
indexer = "kv"
# The PostgreSQL connection configuration, the connection format:
# postgresql://<user>:<password>@<host>:<port>/<db>?<opts>
psql-conn = ""
#######################################################
### Instrumentation Configuration Options ###
#######################################################

View File

@@ -227,7 +227,9 @@ pprof-laddr = ""
# Enable the legacy p2p layer.
use-legacy = false
# Select the p2p internal queue
# Select the p2p internal queue.
# Options are: "fifo", "simple-priority", "priority", and "wdrr"
# with the default being "priority".
queue-type = "priority"
# Address to listen for incoming connections
@@ -281,6 +283,10 @@ max-num-outbound-peers = 10
# Maximum number of connections (inbound and outbound).
max-connections = 64
# Maximum number of connections reserved for outgoing
# connections. Must be less than max-connections
max-outgoing-connections = 12
# Rate limits the number of incoming connection attempts per IP address.
max-incoming-connection-attempts = 100

View File

@@ -208,8 +208,10 @@ pprof-laddr = ""
#######################################################
[p2p]
# Select the p2p internal queue
queue-type = "priority"
# Select the p2p internal queue.
# Options are: "fifo", "simple-priority", and "priority",
# with the default being "priority".
queue-type = "simple-priority"
# Address to listen for incoming connections
laddr = "tcp://0.0.0.0:26656"
@@ -242,6 +244,10 @@ upnp = false
# Maximum number of connections (inbound and outbound).
max-connections = 64
# Maximum number of connections reserved for outgoing
# connections. Must be less than max-connections
max-outgoing-connections = 12
# Rate limits the number of incoming connection attempts per IP address.
max-incoming-connection-attempts = 100

152
scripts/condiff/condiff.go Normal file
View File

@@ -0,0 +1,152 @@
// Program condiff performs a keyspace diff on two TOML documents.
package main
import (
"context"
"flag"
"fmt"
"io"
"log"
"os"
"path/filepath"
"sort"
"strings"
"github.com/creachadair/tomledit"
"github.com/creachadair/tomledit/parser"
"github.com/creachadair/tomledit/transform"
)
var (
doDesnake = flag.Bool("desnake", false, "Convert snake_case to kebab-case before comparing")
)
func init() {
flag.Usage = func() {
fmt.Fprintf(os.Stderr, `Usage: %[1]s [options] f1 f2
Diff the keyspaces of the TOML documents in files f1 and f2.
The output prints one line per key that differs:
-S name -- section exists in f1 but not f2
+S name -- section exists in f2 but not f1
-M name -- mapping exists in f1 but not f2
+M name -- mapping exists in f2 but not f1
Comments, order, and values are ignored for comparison purposes.
Options:
`, filepath.Base(os.Args[0]))
flag.PrintDefaults()
}
}
func main() {
flag.Parse()
if flag.NArg() != 2 {
log.Fatalf("Usage: %[1]s <lhs> <rhs>", filepath.Base(os.Args[0]))
}
lhs := mustParse(flag.Arg(0))
rhs := mustParse(flag.Arg(1))
if *doDesnake {
log.Printf("Converting all names from snake_case to kebab-case")
fix := transform.SnakeToKebab()
_ = fix(context.Background(), lhs)
_ = fix(context.Background(), rhs)
}
diffDocs(os.Stdout, lhs, rhs)
}
func mustParse(path string) *tomledit.Document {
f, err := os.Open(path)
if err != nil {
log.Fatalf("Opening TOML input: %v", err)
}
defer f.Close()
doc, err := tomledit.Parse(f)
if err != nil {
log.Fatalf("Parsing %q: %v", path, err)
}
return doc
}
func allKeys(s *tomledit.Section) []string {
var keys []string
s.Scan(func(key parser.Key, _ *tomledit.Entry) bool {
keys = append(keys, key.String())
return true
})
return keys
}
const (
delSection = "-S"
delMapping = "-M"
addSection = "+S"
addMapping = "+M"
delMapSep = "\n" + delMapping + " "
addMapSep = "\n" + addMapping + " "
)
func diffDocs(w io.Writer, lhs, rhs *tomledit.Document) {
diffSections(w, lhs.Global, rhs.Global)
lsec, rsec := lhs.Sections, rhs.Sections
transform.SortSectionsByName(lsec)
transform.SortSectionsByName(rsec)
i, j := 0, 0
for i < len(lsec) && j < len(rsec) {
if lsec[i].Name.Before(rsec[j].Name) {
fmt.Fprintln(w, delSection, lsec[i].Name)
fmt.Fprintln(w, delMapping, strings.Join(allKeys(lsec[i]), delMapSep))
i++
} else if rsec[j].Name.Before(lsec[i].Name) {
fmt.Fprintln(w, addSection, rsec[j].Name)
fmt.Fprintln(w, addMapping, strings.Join(allKeys(rsec[j]), addMapSep))
j++
} else {
diffSections(w, lsec[i], rsec[j])
i++
j++
}
}
for ; i < len(lsec); i++ {
fmt.Fprintln(w, delSection, lsec[i].Name)
fmt.Fprintln(w, delMapping, strings.Join(allKeys(lsec[i]), delMapSep))
}
for ; j < len(rsec); j++ {
fmt.Fprintln(w, addSection, rsec[j].Name)
fmt.Fprintln(w, addMapping, strings.Join(allKeys(rsec[j]), addMapSep))
}
}
func diffSections(w io.Writer, lhs, rhs *tomledit.Section) {
diffKeys(w, allKeys(lhs), allKeys(rhs))
}
func diffKeys(w io.Writer, lhs, rhs []string) {
sort.Strings(lhs)
sort.Strings(rhs)
i, j := 0, 0
for i < len(lhs) && j < len(rhs) {
if lhs[i] < rhs[j] {
fmt.Fprintln(w, delMapping, lhs[i])
i++
} else if lhs[i] > rhs[j] {
fmt.Fprintln(w, addMapping, rhs[j])
j++
} else {
i++
j++
}
}
for ; i < len(lhs); i++ {
fmt.Fprintln(w, delMapping, lhs[i])
}
for ; j < len(rhs); j++ {
fmt.Fprintln(w, addMapping, rhs[j])
}
}

View File

@@ -1,23 +1,17 @@
// Program confix applies fixes to a Tendermint TOML configuration file to
// update a file created with an older version of Tendermint to a compatible
// format for a newer version.
// Program confix applies changes to a Tendermint TOML configuration file, to
// update configurations created with an older version of Tendermint to a
// compatible format for a newer version.
package main
import (
"bytes"
"context"
"errors"
"flag"
"fmt"
"log"
"os"
"path/filepath"
"github.com/creachadair/atomicfile"
"github.com/creachadair/tomledit"
"github.com/creachadair/tomledit/transform"
"github.com/spf13/viper"
"github.com/tendermint/tendermint/config"
"github.com/tendermint/tendermint/internal/libs/confix"
)
func init() {
@@ -41,6 +35,7 @@ Options:
var (
configPath = flag.String("config", "", "Config file path (required)")
outPath = flag.String("out", "", "Output file path (default stdout)")
doVerbose = flag.Bool("v", false, "Log changes to stderr")
)
func main() {
@@ -49,118 +44,11 @@ func main() {
log.Fatal("You must specify a non-empty -config path")
}
doc, err := LoadConfig(*configPath)
if err != nil {
log.Fatalf("Loading config: %v", err)
ctx := context.Background()
if *doVerbose {
ctx = confix.WithLogWriter(ctx, os.Stderr)
}
ctx := transform.WithLogWriter(context.Background(), os.Stderr)
if err := ApplyFixes(ctx, doc); err != nil {
log.Fatalf("Updating %q: %v", *configPath, err)
}
var buf bytes.Buffer
if err := tomledit.Format(&buf, doc); err != nil {
log.Fatalf("Formatting config: %v", err)
}
// Verify that Tendermint can parse the results after our edits.
if err := CheckValid(buf.Bytes()); err != nil {
log.Fatalf("Updated config is invalid: %v", err)
}
if *outPath == "" {
os.Stdout.Write(buf.Bytes())
} else if err := atomicfile.WriteData(*outPath, buf.Bytes(), 0600); err != nil {
log.Fatalf("Writing output: %v", err)
if err := confix.Upgrade(ctx, *configPath, *outPath); err != nil {
log.Fatalf("Upgrading config: %v", err)
}
}
// ApplyFixes transforms doc and reports whether it succeeded.
func ApplyFixes(ctx context.Context, doc *tomledit.Document) error {
// Check what version of Tendermint might have created this config file, as
// a safety check for the updates we are about to make.
tmVersion := GuessConfigVersion(doc)
if tmVersion == vUnknown {
return errors.New("cannot tell what Tendermint version created this config")
} else if tmVersion < v34 || tmVersion > v35 {
// TODO(creachadair): Add in rewrites for older versions. This will
// require some digging to discover what the changes were. The upgrade
// instructions do not give specifics.
return fmt.Errorf("unable to update version %s config", tmVersion)
}
return plan.Apply(ctx, doc)
}
// LoadConfig loads and parses the TOML document from path.
func LoadConfig(path string) (*tomledit.Document, error) {
f, err := os.Open(path)
if err != nil {
return nil, err
}
defer f.Close()
return tomledit.Parse(f)
}
const (
vUnknown = ""
v32 = "v0.32"
v33 = "v0.33"
v34 = "v0.34"
v35 = "v0.35"
v36 = "v0.36"
)
// GuessConfigVersion attempts to figure out which version of Tendermint
// created the specified config document. It returns "" if the creating version
// cannot be determined, otherwise a string of the form "vX.YY".
func GuessConfigVersion(doc *tomledit.Document) string {
hasDisableWS := doc.First("rpc", "experimental-disable-websocket") != nil
hasUseLegacy := doc.First("p2p", "use-legacy") != nil // v0.35 only
if hasDisableWS && !hasUseLegacy {
return v36
}
hasBlockSync := transform.FindTable(doc, "blocksync") != nil // add: v0.35
hasStateSync := transform.FindTable(doc, "statesync") != nil // add: v0.34
if hasBlockSync && hasStateSync {
return v35
} else if hasStateSync {
return v34
}
hasIndexKeys := doc.First("tx_index", "index_keys") != nil // add: v0.33
hasIndexTags := doc.First("tx_index", "index_tags") != nil // rem: v0.33
if hasIndexKeys && !hasIndexTags {
return v33
}
hasFastSync := transform.FindTable(doc, "fastsync") != nil // add: v0.32
if hasIndexTags && hasFastSync {
return v32
}
// Something older, probably.
return vUnknown
}
// CheckValid checks whether the specified config appears to be a valid
// Tendermint config file. This emulates how the node loads the config.
func CheckValid(data []byte) error {
v := viper.New()
v.SetConfigType("toml")
if err := v.ReadConfig(bytes.NewReader(data)); err != nil {
return fmt.Errorf("reading config: %w", err)
}
var cfg config.Config
if err := v.Unmarshal(&cfg); err != nil {
return fmt.Errorf("decoding config: %w", err)
}
// Stub in required value not stored in the config file, so that validation
// will not fail spuriously.
cfg.Mode = config.ModeValidator
return cfg.ValidateBasic()
}