diff --git a/tools/goleveldb_perf/README.md b/tools/goleveldb_perf/README.md new file mode 100644 index 000000000..0f1db1514 --- /dev/null +++ b/tools/goleveldb_perf/README.md @@ -0,0 +1,31 @@ +# GoLevelDb performance tests + +Simple tool to manipulate (insert & delete) records from a goLevelDb database. +The aim is to investigate how the database size fluctuates in time and how +it correlates with the number of records in the database. + +Handles printing of the total size (in kb) of the database files and of +the number of records present in the db at various steps throughout a test. + +Sample execution: + + +```txt +% go run one.go + + ---- starting tests with GoLevelDB + + ---- inserted 10000 records + ---- deleted 9000 records + ---- inserted 10000 records + ---- deleted 9000 records + ---- deleted 2000 records + Steps: + name size (kb) records # + { initial 0 0} + { insert 1289 10000} + { delete 1562 1000} + { insert 2851 11000} + { delete 3123 2000} + { delete 3184 0} +``` diff --git a/tools/goleveldb_perf/one.go b/tools/goleveldb_perf/one.go new file mode 100644 index 000000000..83e4453b2 --- /dev/null +++ b/tools/goleveldb_perf/one.go @@ -0,0 +1,201 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/gookit/goutil/dump" + tmrand "github.com/tendermint/tendermint/libs/rand" + dbm "github.com/tendermint/tm-db" +) + +// This struct is used to keep track of the steps taken during the test. +// Describes the size of the db and the number of records in it at various +// points during a test +type Step struct { + // The name of the step + Name string + // The size of the db in kb + Size int64 + // The number of records in the db + Records int +} + +// This variable stores the path in the filesystem where the db will be created. +var dbPath string = "./db" + +// Number of records we'll insert in the db +var recordsCount int = 10_000 + +// Defines the size (bytes) of each key written in the DB +var keySize int = 10 + +// Defines the size (bytes) of each value written in the DB +var valueSize int = 100 + +func main() { + fmt.Println("\n\t\t\t---- starting tests with GoLevelDB") + + // Run one test + steps := runTest() + PrintSteps(steps) +} + +// Runs one test, including cleanup and stats printing. +// Returns the steps taken during the test. +func runTest() []Step { + dbPathRemove() + + // Keep track of steps taken during the test + var steps []Step + + // Instantiates a new db + db, err := dbm.NewDB("goleveldb_perf_one", dbm.GoLevelDBBackend, dbPath) + if err != nil { + panic(fmt.Errorf("error instantiating the db: %w", err)) + } + defer db.Close() + + steps = append(steps, Step{ + Name: "initial", + Size: dirSize(dbPath), + Records: dbCount(db), + }) + + // Insert records + steps = append(steps, step("insert", recordsCount, db)) + + // Delete records + steps = append(steps, step("delete", 9_000, db)) + + // Insert records + steps = append(steps, step("insert", recordsCount, db)) + + // Delete records + steps = append(steps, step("delete", 9_000, db)) + + // Delete records + steps = append(steps, step("delete", 2_000, db)) + + return steps +} + +// Performs one step as part of a test. +// +// The step is defined by `stepType`: `delete` or `insert`. +// The `count` defines the number of records to delete or insert. +// The `db` is the db to perform the step on. +// +// Returns the step that was performed. +func step(stepType string, count int, db dbm.DB) Step { + if stepType == "delete" { + // Handle the deletion of records + // Iterate over the db and delete the first `count` records + iter, err := db.Iterator(nil, nil) + if err != nil { + panic(fmt.Errorf("error calling Iterator(): %w", err)) + } + defer iter.Close() + + deleted := 0 + for ; iter.Valid(); iter.Next() { + if err := db.Delete(iter.Key()); err != nil { + panic(fmt.Errorf("error during Delete(): %w", err)) + } + deleted++ + if deleted == count { + fmt.Printf("\n\t\t\t---- deleted %v records", count) + break + } + } + } else if stepType == "insert" { + // Handle the insertion of records + // Write `count` records to the db + for i := 0; i < count; i++ { + if err := db.Set(tmrand.Bytes(keySize), tmrand.Bytes(valueSize)); err != nil { + panic(fmt.Errorf("error during Set(): %w", err)) + } + } + fmt.Printf("\n\t\t\t---- inserted %v records", recordsCount) + } else { + panic("invalid step type") + } + + // Print some stats about the db + // dumpDbStats(db, dbPath) + + return Step{ + Name: stepType, + Size: dirSize(dbPath), + Records: dbCount(db), + } +} + +// Handles the removal of all db files. +// Should be called at the beginning of any test. +func dbPathRemove() { + err := os.RemoveAll(dbPath) + if err != nil { + panic(fmt.Errorf("error removing the db directory: %w", err)) + } +} + +// Fetches and prints the stats of the db. +func dumpDbStats(db dbm.DB, path string) { + stats := db.Stats() + + // Add in the stats the directory size_bytes in the stats + stats["dir_size_bytes"] = fmt.Sprintf("%v", dirSize(path)) + + // Add in the stats the total # of bytes `Set` + stats["set_bytes"] = fmt.Sprintf("%v", recordsCount*(keySize+valueSize)) + + // Add to the stats the total # of records present in the db + stats["records_count"] = fmt.Sprintf("%v", dbCount(db)) + + dump.P(stats) +} + +// Counts the number of records in the `db`. +func dbCount(db dbm.DB) int { + iter, err := db.Iterator(nil, nil) + if err != nil { + panic(fmt.Errorf("error calling Iterator(): %w", err)) + } + defer iter.Close() + iterCount := 0 + for ; iter.Valid(); iter.Next() { + iterCount++ + } + return iterCount +} + +// Computes the size of the given `path` directory in bytes. +// Panics if any error occurs. +func dirSize(path string) int64 { + var size int64 + err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error { + if err != nil { + panic(fmt.Errorf("error opening files inside the db directory: %w", err)) + } + if !info.IsDir() { + size += info.Size() + } + return err + }) + if err != nil { + panic(fmt.Errorf("error walking the db directory: %w", err)) + } + + return size / 1024 +} + +// Prints the content of an array of `Step` structs. +func PrintSteps(steps []Step) { + fmt.Println("\n\tSteps:") + fmt.Printf("\t%15s%15s%15s\n", "name", "size (kb)", "records #") + for _, step := range steps { + fmt.Printf("\t%15v\n", step) + } +}