cmd: add tool for compaction of goleveldb (backport #8564) (#8674)

CHANGELOG_PENDING.md

@@ -20,6 +20,8 @@ Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermi
 
 ### FEATURES
 
+- [cli] [\#8674] Add command to force compact goleveldb databases (@cmwaters)
+
 ### IMPROVEMENTS
 
 ### BUG FIXES

cmd/tendermint/commands/compact.go

@@ -0,0 +1,66 @@
+package commands
+
+import (
+	"errors"
+	"path/filepath"
+	"sync"
+
+	"github.com/spf13/cobra"
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syndtr/goleveldb/leveldb/util"
+
+	"github.com/tendermint/tendermint/libs/log"
+)
+
+var CompactGoLevelDBCmd = &cobra.Command{
+	Use:   "experimental-compact-goleveldb",
+	Short: "force compacts the tendermint storage engine (only GoLevelDB supported)",
+	Long: `
+This is a temporary utility command that performs a force compaction on the state 
+and blockstores to reduce disk space for a pruning node. This should only be run 
+once the node has stopped. This command will likely be omitted in the future after
+the planned refactor to the storage engine.
+
+Currently, only GoLevelDB is supported.
+	`,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		if config.DBBackend != "goleveldb" {
+			return errors.New("compaction is currently only supported with goleveldb")
+		}
+
+		compactGoLevelDBs(config.RootDir, logger)
+		return nil
+	},
+}
+
+func compactGoLevelDBs(rootDir string, logger log.Logger) {
+	dbNames := []string{"state", "blockstore"}
+	o := &opt.Options{
+		DisableSeeksCompaction: true,
+	}
+	wg := sync.WaitGroup{}
+
+	for _, dbName := range dbNames {
+		dbName := dbName
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			dbPath := filepath.Join(rootDir, "data", dbName+".db")
+			store, err := leveldb.OpenFile(dbPath, o)
+			if err != nil {
+				logger.Error("failed to initialize tendermint db", "path", dbPath, "err", err)
+				return
+			}
+			defer store.Close()
+
+			logger.Info("starting compaction...", "db", dbPath)
+
+			err = store.CompactRange(util.Range{Start: nil, Limit: nil})
+			if err != nil {
+				logger.Error("failed to compact tendermint db", "path", dbPath, "err", err)
+			}
+		}()
+	}
+	wg.Wait()
+}

cmd/tendermint/main.go

@@ -29,6 +29,7 @@ func main() {
 		cmd.GenNodeKeyCmd,
 		cmd.VersionCmd,
 		cmd.RollbackStateCmd,
+		cmd.CompactGoLevelDBCmd,
 		debug.DebugCmd,
 		cli.NewCompletionCmd(rootCmd, true),
 	)

go.mod

@@ -42,3 +42,8 @@ require (
 	google.golang.org/grpc v1.47.0
 	gotest.tools v2.2.0+incompatible // indirect
 )
+
+require (
+	github.com/prometheus/common v0.34.0 // indirect
+	github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca
+)

go.sum

@@ -580,6 +580,7 @@ github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3O
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
 github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
+github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
 github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34=
 github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
@@ -592,8 +593,9 @@ github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8b
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
 github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
-github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4=
 github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
+github.com/prometheus/common v0.34.0 h1:RBmGO9d/FVjqHT0yUGQwBJhkwKV+wPCn7KGpvfab0uE=
+github.com/prometheus/common v0.34.0/go.mod h1:gB3sOl7P0TvJabZpLY5uQMpUqRCPPCyRLCZYc7JZTNE=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=