From 30bfe51ebeba61e64abdc9d00fa78f7389fe5073 Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Thu, 2 Jun 2022 10:01:16 +0200 Subject: [PATCH] cmd: add tool for compaction of goleveldb (#8564) --- cmd/tendermint/commands/compact.go | 71 ++++++++++++++++++++++++++++++ cmd/tendermint/main.go | 1 + go.mod | 64 +++++++++++++-------------- go.sum | 9 ++-- 4 files changed, 108 insertions(+), 37 deletions(-) create mode 100644 cmd/tendermint/commands/compact.go diff --git a/cmd/tendermint/commands/compact.go b/cmd/tendermint/commands/compact.go new file mode 100644 index 000000000..eadd828ae --- /dev/null +++ b/cmd/tendermint/commands/compact.go @@ -0,0 +1,71 @@ +package commands + +import ( + "errors" + "path/filepath" + "sync" + + "github.com/spf13/cobra" + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/opt" + "github.com/syndtr/goleveldb/leveldb/util" + + "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/libs/log" +) + +func MakeCompactDBCommand(cfg *config.Config, logger log.Logger) *cobra.Command { + cmd := &cobra.Command{ + Use: "experimental-compact-goleveldb", + Short: "force compacts the tendermint storage engine (only GoLevelDB supported)", + Long: ` +This is a temporary utility command that performs a force compaction on the state +and blockstores to reduce disk space for a pruning node. This should only be run +once the node has stopped. This command will likely be omitted in the future after +the planned refactor to the storage engine. + +Currently, only GoLevelDB is supported. + `, + RunE: func(cmd *cobra.Command, args []string) error { + if cfg.DBBackend != "goleveldb" { + return errors.New("compaction is currently only supported with goleveldb") + } + + compactGoLevelDBs(cfg.RootDir, logger) + return nil + }, + } + + return cmd +} + +func compactGoLevelDBs(rootDir string, logger log.Logger) { + dbNames := []string{"state", "blockstore"} + o := &opt.Options{ + DisableSeeksCompaction: true, + } + wg := sync.WaitGroup{} + + for _, dbName := range dbNames { + dbName := dbName + wg.Add(1) + go func() { + defer wg.Done() + dbPath := filepath.Join(rootDir, "data", dbName+".db") + store, err := leveldb.OpenFile(dbPath, o) + if err != nil { + logger.Error("failed to initialize tendermint db", "path", dbPath, "err", err) + return + } + defer store.Close() + + logger.Info("starting compaction...", "db", dbPath) + + err = store.CompactRange(util.Range{Start: nil, Limit: nil}) + if err != nil { + logger.Error("failed to compact tendermint db", "path", dbPath, "err", err) + } + }() + } + wg.Wait() +} diff --git a/cmd/tendermint/main.go b/cmd/tendermint/main.go index 715e8e1d4..1be614aea 100644 --- a/cmd/tendermint/main.go +++ b/cmd/tendermint/main.go @@ -44,6 +44,7 @@ func main() { commands.MakeKeyMigrateCommand(conf, logger), debug.GetDebugCommand(logger), commands.NewCompletionCmd(rcmd, true), + commands.MakeCompactDBCommand(conf, logger), ) // NOTE: diff --git a/go.mod b/go.mod index fd2f15cb8..69284bead 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/tendermint/tm-db v0.6.6 golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2 - golang.org/x/sync v0.0.0-20210220032951-036812b2e83c + golang.org/x/sync v0.0.0-20220513210516-0976fa681c29 google.golang.org/grpc v1.47.0 pgregory.net/rapid v0.4.7 ) @@ -46,31 +46,6 @@ require ( gotest.tools v2.2.0+incompatible ) -require ( - github.com/GaijinEntertainment/go-exhaustruct/v2 v2.1.0 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect - github.com/firefart/nonamedreturns v1.0.1 // indirect - github.com/gofrs/uuid v4.2.0+incompatible // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/jdxcode/netrc v0.0.0-20210204082910-926c7f70242a // indirect - github.com/jhump/protocompile v0.0.0-20220216033700-d705409f108f // indirect - github.com/jhump/protoreflect v1.12.1-0.20220417024638-438db461d753 // indirect - github.com/klauspost/compress v1.15.1 // indirect - github.com/klauspost/pgzip v1.2.5 // indirect - github.com/lufeee/execinquery v1.0.0 // indirect - github.com/pelletier/go-toml/v2 v2.0.1 // indirect - github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect - github.com/pkg/profile v1.6.0 // indirect - github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 // indirect - github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/stbenjam/no-sprintf-host-port v0.1.1 // indirect - go.opencensus.io v0.23.0 // indirect - go.uber.org/atomic v1.9.0 // indirect - go.uber.org/multierr v1.8.0 // indirect - go.uber.org/zap v1.21.0 // indirect - golang.org/x/exp/typeparams v0.0.0-20220218215828-6cf2b201936e // indirect -) - require ( 4d63.com/gochecknoglobals v0.1.0 // indirect github.com/Antonboom/errname v0.1.6 // indirect @@ -78,6 +53,7 @@ require ( github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/DataDog/zstd v1.4.1 // indirect github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24 // indirect + github.com/GaijinEntertainment/go-exhaustruct/v2 v2.1.0 // indirect github.com/Masterminds/semver v1.5.0 // indirect github.com/Microsoft/go-winio v0.5.1 // indirect github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect @@ -98,7 +74,7 @@ require ( github.com/charithe/durationcheck v0.0.9 // indirect github.com/chavacava/garif v0.0.0-20220316182200-5cad0b5181d4 // indirect github.com/containerd/continuity v0.2.1 // indirect - github.com/creachadair/tomledit v0.0.22 + github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect github.com/daixiang0/gci v0.3.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/denis-tingaikin/go-header v0.4.3 // indirect @@ -110,11 +86,12 @@ require ( github.com/dustin/go-humanize v1.0.0 // indirect github.com/esimonov/ifshort v1.0.4 // indirect github.com/ettle/strcase v0.1.1 // indirect - github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 // indirect + github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c // indirect github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect - github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 // indirect + github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect github.com/fatih/color v1.13.0 // indirect github.com/fatih/structtag v1.2.0 // indirect + github.com/firefart/nonamedreturns v1.0.1 // indirect github.com/fsnotify/fsnotify v1.5.4 // indirect github.com/fzipp/gocyclo v0.5.1 // indirect github.com/go-critic/go-critic v0.6.3 // indirect @@ -128,6 +105,8 @@ require ( github.com/go-xmlfmt/xmlfmt v0.0.0-20191208150333-d5b6f63a941b // indirect github.com/gobwas/glob v0.2.3 // indirect github.com/gofrs/flock v0.8.1 // indirect + github.com/gofrs/uuid v4.2.0+incompatible // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/snappy v0.0.3 // indirect github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 // indirect github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a // indirect @@ -150,19 +129,25 @@ require ( github.com/hashicorp/hcl v1.0.0 // indirect github.com/hexops/gotextdiff v1.0.3 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/jdxcode/netrc v0.0.0-20210204082910-926c7f70242a // indirect github.com/jgautheron/goconst v1.5.1 // indirect + github.com/jhump/protocompile v0.0.0-20220216033700-d705409f108f // indirect + github.com/jhump/protoreflect v1.12.1-0.20220417024638-438db461d753 // indirect github.com/jingyugao/rowserrcheck v1.1.1 // indirect github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af // indirect github.com/jmhodges/levigo v1.0.0 // indirect github.com/julz/importas v0.1.0 // indirect github.com/kisielk/errcheck v1.6.0 // indirect github.com/kisielk/gotool v1.0.0 // indirect + github.com/klauspost/compress v1.15.1 // indirect + github.com/klauspost/pgzip v1.2.5 // indirect github.com/kulti/thelper v0.6.2 // indirect github.com/kunwardeep/paralleltest v1.0.3 // indirect github.com/kyoh86/exportloopref v0.1.8 // indirect github.com/ldez/gomoddirectives v0.2.3 // indirect github.com/ldez/tagliatelle v0.3.1 // indirect github.com/leonklingele/grouper v1.1.0 // indirect + github.com/lufeee/execinquery v1.0.0 // indirect github.com/magiconair/properties v1.8.6 // indirect github.com/maratori/testpackage v1.0.1 // indirect github.com/matoous/godox v0.0.0-20210227103229-6504466cf951 // indirect @@ -184,16 +169,19 @@ require ( github.com/opencontainers/image-spec v1.0.2 // indirect github.com/opencontainers/runc v1.0.3 // indirect github.com/pelletier/go-toml v1.9.5 // indirect + github.com/pelletier/go-toml/v2 v2.0.1 // indirect github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d // indirect + github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pkg/profile v1.6.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/polyfloyd/go-errorlint v0.0.0-20211125173453-6d6d39c5bb8b // indirect - github.com/prometheus/client_model v0.2.0 - github.com/prometheus/common v0.34.0 github.com/prometheus/procfs v0.7.3 // indirect github.com/quasilyte/go-ruleguard v0.3.16-0.20220213074421-6aa060fab41a // indirect github.com/quasilyte/gogrep v0.0.0-20220120141003-628d8b3623b5 // indirect github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95 // indirect + github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/ryancurrah/gomodguard v1.2.3 // indirect github.com/ryanrolds/sqlclosecheck v0.3.0 // indirect github.com/sanposhiho/wastedassign/v2 v2.0.6 // indirect @@ -209,10 +197,10 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/ssgreg/nlreturn/v2 v2.2.1 // indirect + github.com/stbenjam/no-sprintf-host-port v0.1.1 // indirect github.com/stretchr/objx v0.1.1 // indirect github.com/subosito/gotenv v1.3.0 // indirect github.com/sylvia7788/contextcheck v1.0.4 // indirect - github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca // indirect github.com/tdakkota/asciicheck v0.1.1 // indirect github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect github.com/tetafro/godot v1.4.11 // indirect @@ -226,6 +214,11 @@ require ( github.com/yeya24/promlinter v0.2.0 // indirect gitlab.com/bosi/decorder v0.2.1 // indirect go.etcd.io/bbolt v1.3.6 // indirect + go.opencensus.io v0.23.0 // indirect + go.uber.org/atomic v1.9.0 // indirect + go.uber.org/multierr v1.8.0 // indirect + go.uber.org/zap v1.21.0 // indirect + golang.org/x/exp/typeparams v0.0.0-20220218215828-6cf2b201936e // indirect golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect golang.org/x/term v0.0.0-20220411215600-e5f449aeb171 // indirect @@ -243,3 +236,10 @@ require ( mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b // indirect mvdan.cc/unparam v0.0.0-20211214103731-d0ef000c54e5 // indirect ) + +require ( + github.com/creachadair/tomledit v0.0.22 + github.com/prometheus/client_model v0.2.0 + github.com/prometheus/common v0.34.0 + github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca +) diff --git a/go.sum b/go.sum index 3d036cf0c..b107e95b5 100644 --- a/go.sum +++ b/go.sum @@ -295,12 +295,12 @@ github.com/esimonov/ifshort v1.0.4 h1:6SID4yGWfRae/M7hkVDVVyppy8q/v9OuxNdmjLQStB github.com/esimonov/ifshort v1.0.4/go.mod h1:Pe8zjlRrJ80+q2CxHLfEOfTwxCZ4O+MuhcHcfgNWTk0= github.com/ettle/strcase v0.1.1 h1:htFueZyVeE1XNnMEfbqp5r67qAN/4r6ya1ysq8Q+Zcw= github.com/ettle/strcase v0.1.1/go.mod h1:hzDLsPC7/lwKyBOywSHEP89nt2pDgdy+No1NBA9o9VY= -github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 h1:0JZ+dUmQeA8IIVUMzysrX4/AKuQwWhV2dYQuPZdvdSQ= -github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= +github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0= +github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A= github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= -github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 h1:E2s37DuLxFhQDg5gKsWoLBOB0n+ZW8s599zru8FJ2/Y= -github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= +github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk= +github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= @@ -1369,7 +1369,6 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220513210516-0976fa681c29 h1:w8s32wxx3sY+OjLlv9qltkLU5yvJzxjjgiHWLjdIcw4= golang.org/x/sync v0.0.0-20220513210516-0976fa681c29/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=