migration: scope key migration to stores (#9005) (#9028)

(cherry picked from commit cc07318866)

Co-authored-by: Sam Kleinman <garen@tychoish.com>

scripts/keymigrate/migrate_test.go

@@ -1,16 +1,12 @@
 package keymigrate
 
 import (
-	"context"
-	"errors"
 	"fmt"
-	"math"
 	"strings"
 	"testing"
 
 	"github.com/google/orderedcode"
 	"github.com/stretchr/testify/require"
-	dbm "github.com/tendermint/tm-db"
 )
 
 func makeKey(t *testing.T, elems ...interface{}) []byte {
@@ -78,30 +74,6 @@ func getNewPrefixKeys(t *testing.T, val int) map[string][]byte {
 	}
 }
 
-func getLegacyDatabase(t *testing.T) (int, dbm.DB) {
-	db := dbm.NewMemDB()
-	batch := db.NewBatch()
-	ct := 0
-
-	generated := []map[string][]byte{
-		getLegacyPrefixKeys(8),
-		getLegacyPrefixKeys(9001),
-		getLegacyPrefixKeys(math.MaxInt32 << 1),
-		getLegacyPrefixKeys(math.MaxInt64 - 8),
-	}
-
-	// populate database
-	for _, km := range generated {
-		for _, key := range km {
-			ct++
-			require.NoError(t, batch.Set(key, []byte(fmt.Sprintf(`{"value": %d}`, ct))))
-		}
-	}
-	require.NoError(t, batch.WriteSync())
-	require.NoError(t, batch.Close())
-	return ct - (2 * len(generated)) + 2, db
-}
-
 func TestMigration(t *testing.T) {
 	t.Run("Idempotency", func(t *testing.T) {
 		// we want to make sure that the key space for new and
@@ -113,37 +85,12 @@ func TestMigration(t *testing.T) {
 
 		require.Equal(t, len(legacyPrefixes), len(newPrefixes))
 
-		t.Run("Legacy", func(t *testing.T) {
-			for kind, le := range legacyPrefixes {
-				require.True(t, checkKeyType(le).isLegacy(), kind)
-			}
-		})
-		t.Run("New", func(t *testing.T) {
-			for kind, ne := range newPrefixes {
-				require.False(t, checkKeyType(ne).isLegacy(), kind)
-			}
-		})
-		t.Run("Conversion", func(t *testing.T) {
-			for kind, le := range legacyPrefixes {
-				nk, err := migrateKey(le)
-				require.NoError(t, err, kind)
-				require.False(t, checkKeyType(nk).isLegacy(), kind)
-			}
-		})
 		t.Run("Hashes", func(t *testing.T) {
 			t.Run("NewKeysAreNotHashes", func(t *testing.T) {
 				for _, key := range getNewPrefixKeys(t, 9001) {
 					require.True(t, len(key) != 32)
 				}
 			})
-			t.Run("ContrivedLegacyKeyDetection", func(t *testing.T) {
-				// length 32: should appear to be a hash
-				require.Equal(t, txHashKey, checkKeyType([]byte("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx")))
-
-				// length ≠ 32: should not appear to be a hash
-				require.Equal(t, nonLegacyKey, checkKeyType([]byte("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx--")))
-				require.Equal(t, nonLegacyKey, checkKeyType([]byte("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx")))
-			})
 		})
 	})
 	t.Run("Migrations", func(t *testing.T) {
@@ -171,72 +118,101 @@ func TestMigration(t *testing.T) {
 				"UserKey3":        []byte("foo/bar/baz/1.2/4"),
 			}
 			for kind, key := range table {
-				out, err := migrateKey(key)
+				out, err := migrateKey(key, "")
+				// TODO probably these error at the
+				// moment because of store missmatches
 				require.Error(t, err, kind)
 				require.Nil(t, out, kind)
 			}
 		})
-		t.Run("Replacement", func(t *testing.T) {
-			t.Run("MissingKey", func(t *testing.T) {
-				db := dbm.NewMemDB()
-				require.NoError(t, replaceKey(db, keyID("hi"), nil))
-			})
-			t.Run("ReplacementFails", func(t *testing.T) {
-				db := dbm.NewMemDB()
-				key := keyID("hi")
-				require.NoError(t, db.Set(key, []byte("world")))
-				require.Error(t, replaceKey(db, key, func(k keyID) (keyID, error) {
-					return nil, errors.New("hi")
-				}))
-			})
-			t.Run("KeyDisappears", func(t *testing.T) {
-				db := dbm.NewMemDB()
-				key := keyID("hi")
-				require.NoError(t, db.Set(key, []byte("world")))
-				require.Error(t, replaceKey(db, key, func(k keyID) (keyID, error) {
-					require.NoError(t, db.Delete(key))
-					return keyID("wat"), nil
-				}))
-
-				exists, err := db.Has(key)
-				require.NoError(t, err)
-				require.False(t, exists)
-
-				exists, err = db.Has(keyID("wat"))
-				require.NoError(t, err)
-				require.False(t, exists)
-			})
-		})
-	})
-	t.Run("Integration", func(t *testing.T) {
-		t.Run("KeyDiscovery", func(t *testing.T) {
-			size, db := getLegacyDatabase(t)
-			keys, err := getAllLegacyKeys(db)
-			require.NoError(t, err)
-			require.Equal(t, size, len(keys))
-			legacyKeys := 0
-			for _, k := range keys {
-				if checkKeyType(k).isLegacy() {
-					legacyKeys++
-				}
-			}
-			require.Equal(t, size, legacyKeys)
-		})
-		t.Run("KeyIdempotency", func(t *testing.T) {
-			for _, key := range getNewPrefixKeys(t, 84) {
-				require.False(t, checkKeyType(key).isLegacy())
-			}
-		})
-		t.Run("Migrate", func(t *testing.T) {
-			_, db := getLegacyDatabase(t)
-
-			ctx := context.Background()
-			err := Migrate(ctx, db)
-			require.NoError(t, err)
-			keys, err := getAllLegacyKeys(db)
-			require.NoError(t, err)
-			require.Equal(t, 0, len(keys))
-
-		})
 	})
 }
+
+func TestGlobalDataStructuresForRefactor(t *testing.T) {
+	defer func() {
+		if t.Failed() {
+			t.Log("number of migrations:", len(migrations))
+		}
+	}()
+
+	const unPrefixedLegacyKeys = 3
+
+	t.Run("MigrationsAreDefined", func(t *testing.T) {
+		if len(prefixes)+unPrefixedLegacyKeys != len(migrations) {
+			t.Fatal("migrationse are not correctly defined",
+				"prefixes", len(prefixes),
+				"migrations", len(migrations))
+		}
+	})
+	t.Run("AllMigrationsHavePrefixDefined", func(t *testing.T) {
+		for _, m := range migrations {
+			if m.prefix == nil && m.storeName != "tx_index" {
+				t.Errorf("migration named %q for store %q does not have a prefix defined", m.name, m.storeName)
+			}
+		}
+	})
+	t.Run("Deduplication", func(t *testing.T) {
+		t.Run("Prefixes", func(t *testing.T) {
+			set := map[string]struct{}{}
+			for _, prefix := range prefixes {
+				set[string(prefix.prefix)] = struct{}{}
+			}
+			if len(set) != len(prefixes) {
+				t.Fatal("duplicate prefix definition",
+					"set", len(set),
+					"values", set)
+			}
+		})
+		t.Run("MigrationName", func(t *testing.T) {
+			set := map[string]struct{}{}
+			for _, migration := range migrations {
+				set[migration.name] = struct{}{}
+			}
+			if len(set) != len(migrations) {
+				t.Fatal("duplicate migration name defined",
+					"set", len(set),
+					"values", set)
+			}
+		})
+		t.Run("MigrationPrefix", func(t *testing.T) {
+			set := map[string]struct{}{}
+			for _, migration := range migrations {
+				set[string(migration.prefix)] = struct{}{}
+			}
+			// three keys don't have prefixes in the
+			// legacy system; this is fine but it means
+			// the set will have 1  less than expected
+			// (well 2 less, but the empty key takes one
+			// of the slots):
+			expectedDupl := unPrefixedLegacyKeys - 1
+
+			if len(set) != len(migrations)-expectedDupl {
+				t.Fatal("duplicate migration prefix defined",
+					"set", len(set),
+					"expected", len(migrations)-expectedDupl,
+					"values", set)
+			}
+		})
+		t.Run("MigrationStoreName", func(t *testing.T) {
+			set := map[string]struct{}{}
+			for _, migration := range migrations {
+				set[migration.storeName] = struct{}{}
+			}
+			if len(set) != 5 {
+				t.Fatal("duplicate migration store name defined",
+					"set", len(set),
+					"values", set)
+			}
+			if _, ok := set[""]; ok {
+				t.Fatal("empty store name defined")
+			}
+		})
+	})
+	t.Run("NilPrefix", func(t *testing.T) {
+		_, err := getMigrationFunc("tx_index", []byte("fooo"))
+		if err != nil {
+			t.Fatal("should find an index for tx", err)
+		}
+	})
+
+}