mirror of
https://github.com/tendermint/tendermint.git
synced 2026-09-20 06:54:41 +00:00
db: migration script for key format change (#6355)
This commit is contained in:
Sam Kleinman
@@ -0,0 +1,241 @@
|
||||
package keymigrate
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
"github.com/google/orderedcode"
|
||||
"github.com/stretchr/testify/require"
|
||||
dbm "github.com/tendermint/tm-db"
|
||||
)
|
||||
|
||||
func makeKey(t *testing.T, elems ...interface{}) []byte {
|
||||
t.Helper()
|
||||
out, err := orderedcode.Append([]byte{}, elems...)
|
||||
require.NoError(t, err)
|
||||
return out
|
||||
}
|
||||
|
||||
func getLegacyPrefixKeys(val int) map[string][]byte {
|
||||
return map[string][]byte{
|
||||
"Height": []byte(fmt.Sprintf("H:%d", val)),
|
||||
"BlockPart": []byte(fmt.Sprintf("P:%d:%d", val, val)),
|
||||
"BlockPartTwo": []byte(fmt.Sprintf("P:%d:%d", val+2, val+val)),
|
||||
"BlockCommit": []byte(fmt.Sprintf("C:%d", val)),
|
||||
"SeenCommit": []byte(fmt.Sprintf("SC:%d", val)),
|
||||
"BlockHeight": []byte(fmt.Sprintf("BH:%d", val)),
|
||||
"Validators": []byte(fmt.Sprintf("validatorsKey:%d", val)),
|
||||
"ConsensusParams": []byte(fmt.Sprintf("consensusParamsKey:%d", val)),
|
||||
"ABCIResponse": []byte(fmt.Sprintf("abciResponsesKey:%d", val)),
|
||||
"State": []byte("stateKey"),
|
||||
"CommittedEvidence": append([]byte{0x00}, []byte(fmt.Sprintf("%0.16X/%X", int64(val), []byte("committed")))...),
|
||||
"PendingEvidence": append([]byte{0x01}, []byte(fmt.Sprintf("%0.16X/%X", int64(val), []byte("pending")))...),
|
||||
"LightBLock": []byte(fmt.Sprintf("lb/foo/%020d", val)),
|
||||
"Size": []byte("size"),
|
||||
"UserKey0": []byte(fmt.Sprintf("foo/bar/%d/%d", val, val)),
|
||||
"UserKey1": []byte(fmt.Sprintf("foo/bar/baz/%d/%d", val, val)),
|
||||
"TxHeight": []byte(fmt.Sprintf("tx.height/%s/%d/%d", fmt.Sprint(val), val, val)),
|
||||
"TxHash": append(
|
||||
bytes.Repeat([]byte{fmt.Sprint(val)[0]}, 16),
|
||||
bytes.Repeat([]byte{fmt.Sprint(val)[len([]byte(fmt.Sprint(val)))-1]}, 16)...,
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
func getNewPrefixKeys(t *testing.T, val int) map[string][]byte {
|
||||
t.Helper()
|
||||
return map[string][]byte{
|
||||
"Height": makeKey(t, int64(0), int64(val)),
|
||||
"BlockPart": makeKey(t, int64(1), int64(val), int64(val)),
|
||||
"BlockPartTwo": makeKey(t, int64(1), int64(val+2), int64(val+val)),
|
||||
"BlockCommit": makeKey(t, int64(2), int64(val)),
|
||||
"SeenCommit": makeKey(t, int64(3), int64(val)),
|
||||
"BlockHeight": makeKey(t, int64(4), int64(val)),
|
||||
"Validators": makeKey(t, int64(5), int64(val)),
|
||||
"ConsensusParams": makeKey(t, int64(6), int64(val)),
|
||||
"ABCIResponse": makeKey(t, int64(7), int64(val)),
|
||||
"State": makeKey(t, int64(8)),
|
||||
"CommittedEvidence": makeKey(t, int64(9), int64(val)),
|
||||
"PendingEvidence": makeKey(t, int64(10), int64(val)),
|
||||
"LightBLock": makeKey(t, int64(11), int64(val)),
|
||||
"Size": makeKey(t, int64(12)),
|
||||
"UserKey0": makeKey(t, "foo", "bar", int64(val), int64(val)),
|
||||
"UserKey1": makeKey(t, "foo", "bar/baz", int64(val), int64(val)),
|
||||
"TxHeight": makeKey(t, "tx.height", fmt.Sprint(val), int64(val), int64(val+2), int64(val+val)),
|
||||
"TxHash": makeKey(t, "tx.hash", string(bytes.Repeat([]byte{[]byte(fmt.Sprint(val))[0]}, 32))),
|
||||
}
|
||||
}
|
||||
|
||||
func getLegacyDatabase(t *testing.T) (int, dbm.DB) {
|
||||
db := dbm.NewMemDB()
|
||||
batch := db.NewBatch()
|
||||
ct := 0
|
||||
|
||||
generated := []map[string][]byte{
|
||||
getLegacyPrefixKeys(8),
|
||||
getLegacyPrefixKeys(9001),
|
||||
getLegacyPrefixKeys(math.MaxInt32 << 1),
|
||||
getLegacyPrefixKeys(math.MaxInt64 - 8),
|
||||
}
|
||||
|
||||
// populate database
|
||||
for _, km := range generated {
|
||||
for _, key := range km {
|
||||
ct++
|
||||
require.NoError(t, batch.Set(key, []byte(fmt.Sprintf(`{"value": %d}`, ct))))
|
||||
}
|
||||
}
|
||||
require.NoError(t, batch.WriteSync())
|
||||
require.NoError(t, batch.Close())
|
||||
return ct - (2 * len(generated)) + 2, db
|
||||
}
|
||||
|
||||
func TestMigration(t *testing.T) {
|
||||
t.Run("Idempotency", func(t *testing.T) {
|
||||
// we want to make sure that the key space for new and
|
||||
// legacy keys are entirely non-overlapping.
|
||||
|
||||
legacyPrefixes := getLegacyPrefixKeys(42)
|
||||
|
||||
newPrefixes := getNewPrefixKeys(t, 42)
|
||||
|
||||
require.Equal(t, len(legacyPrefixes), len(newPrefixes))
|
||||
|
||||
t.Run("Legacy", func(t *testing.T) {
|
||||
for kind, le := range legacyPrefixes {
|
||||
require.True(t, keyIsLegacy(le), kind)
|
||||
}
|
||||
})
|
||||
t.Run("New", func(t *testing.T) {
|
||||
for kind, ne := range newPrefixes {
|
||||
require.False(t, keyIsLegacy(ne), kind)
|
||||
}
|
||||
})
|
||||
t.Run("Conversion", func(t *testing.T) {
|
||||
for kind, le := range legacyPrefixes {
|
||||
nk, err := migarateKey(le)
|
||||
require.NoError(t, err, kind)
|
||||
require.False(t, keyIsLegacy(nk), kind)
|
||||
}
|
||||
})
|
||||
t.Run("Hashes", func(t *testing.T) {
|
||||
t.Run("NewKeysAreNotHashes", func(t *testing.T) {
|
||||
for _, key := range getNewPrefixKeys(t, 9001) {
|
||||
require.True(t, len(key) != 32)
|
||||
}
|
||||
})
|
||||
t.Run("ContrivedLegacyKeyDetection", func(t *testing.T) {
|
||||
require.True(t, keyIsLegacy([]byte("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx")))
|
||||
require.False(t, keyIsLegacy([]byte("xxxxxxxxxxxxxxx/xxxxxxxxxxxxxxxx")))
|
||||
})
|
||||
})
|
||||
})
|
||||
t.Run("Migrations", func(t *testing.T) {
|
||||
t.Run("Errors", func(t *testing.T) {
|
||||
table := map[string][]byte{
|
||||
"Height": []byte(fmt.Sprintf("H:%f", 4.22222)),
|
||||
"BlockPart": []byte(fmt.Sprintf("P:%f", 4.22222)),
|
||||
"BlockPartTwo": []byte(fmt.Sprintf("P:%d", 42)),
|
||||
"BlockPartThree": []byte(fmt.Sprintf("P:%f:%f", 4.222, 8.444)),
|
||||
"BlockPartFour": []byte(fmt.Sprintf("P:%d:%f", 4222, 8.444)),
|
||||
"BlockCommit": []byte(fmt.Sprintf("C:%f", 4.22222)),
|
||||
"SeenCommit": []byte(fmt.Sprintf("SC:%f", 4.22222)),
|
||||
"BlockHeight": []byte(fmt.Sprintf("BH:%f", 4.22222)),
|
||||
"Validators": []byte(fmt.Sprintf("validatorsKey:%f", 4.22222)),
|
||||
"ConsensusParams": []byte(fmt.Sprintf("consensusParamsKey:%f", 4.22222)),
|
||||
"ABCIResponse": []byte(fmt.Sprintf("abciResponsesKey:%f", 4.22222)),
|
||||
"LightBlockShort": []byte(fmt.Sprintf("lb/foo/%010d", 42)),
|
||||
"LightBlockLong": []byte("lb/foo/12345678910.1234567890"),
|
||||
"Invalid": {0x03},
|
||||
"BadTXHeight0": []byte(fmt.Sprintf("tx.height/%s/%f/%f", "boop", 4.4, 4.5)),
|
||||
"BadTXHeight1": []byte(fmt.Sprintf("tx.height/%s/%f", "boop", 4.4)),
|
||||
"UserKey0": []byte("foo/bar/1.3/3.4"),
|
||||
"UserKey1": []byte("foo/bar/1/3.4"),
|
||||
"UserKey2": []byte("foo/bar/baz/1/3.4"),
|
||||
"UserKey3": []byte("foo/bar/baz/1.2/4"),
|
||||
}
|
||||
for kind, key := range table {
|
||||
out, err := migarateKey(key)
|
||||
require.Error(t, err, kind)
|
||||
require.Nil(t, out, kind)
|
||||
}
|
||||
})
|
||||
t.Run("Replacement", func(t *testing.T) {
|
||||
t.Run("MissingKey", func(t *testing.T) {
|
||||
db := dbm.NewMemDB()
|
||||
require.NoError(t, replaceKey(db, keyID("hi"), nil))
|
||||
})
|
||||
t.Run("ReplacementFails", func(t *testing.T) {
|
||||
db := dbm.NewMemDB()
|
||||
key := keyID("hi")
|
||||
require.NoError(t, db.Set(key, []byte("world")))
|
||||
require.Error(t, replaceKey(db, key, func(k keyID) (keyID, error) {
|
||||
return nil, errors.New("hi")
|
||||
}))
|
||||
})
|
||||
t.Run("KeyDisapears", func(t *testing.T) {
|
||||
db := dbm.NewMemDB()
|
||||
key := keyID("hi")
|
||||
require.NoError(t, db.Set(key, []byte("world")))
|
||||
require.Error(t, replaceKey(db, key, func(k keyID) (keyID, error) {
|
||||
require.NoError(t, db.Delete(key))
|
||||
return keyID("wat"), nil
|
||||
}))
|
||||
|
||||
exists, err := db.Has(key)
|
||||
require.NoError(t, err)
|
||||
require.False(t, exists)
|
||||
|
||||
exists, err = db.Has(keyID("wat"))
|
||||
require.NoError(t, err)
|
||||
require.False(t, exists)
|
||||
})
|
||||
})
|
||||
})
|
||||
t.Run("Integration", func(t *testing.T) {
|
||||
t.Run("KeyDiscovery", func(t *testing.T) {
|
||||
size, db := getLegacyDatabase(t)
|
||||
keys, err := getAllLegacyKeys(db)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, size, len(keys))
|
||||
legacyKeys := 0
|
||||
for _, k := range keys {
|
||||
if keyIsLegacy(k) {
|
||||
legacyKeys++
|
||||
}
|
||||
}
|
||||
require.Equal(t, size, legacyKeys)
|
||||
})
|
||||
t.Run("KeyIdempotency", func(t *testing.T) {
|
||||
for _, key := range getNewPrefixKeys(t, 84) {
|
||||
require.False(t, keyIsLegacy(key))
|
||||
}
|
||||
})
|
||||
t.Run("ChannelConversion", func(t *testing.T) {
|
||||
ch := makeKeyChan([]keyID{
|
||||
makeKey(t, "abc", int64(2), int64(42)),
|
||||
makeKey(t, int64(42)),
|
||||
})
|
||||
count := 0
|
||||
for range ch {
|
||||
count++
|
||||
}
|
||||
require.Equal(t, 2, count)
|
||||
})
|
||||
t.Run("Migrate", func(t *testing.T) {
|
||||
_, db := getLegacyDatabase(t)
|
||||
|
||||
ctx := context.Background()
|
||||
err := Migrate(ctx, db)
|
||||
require.NoError(t, err)
|
||||
keys, err := getAllLegacyKeys(db)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 0, len(keys))
|
||||
|
||||
})
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user