tendermint/scripts/scmigrate/migrate.go

// Package scmigrate implements a migration for SeenCommit data
// between 0.34 and 0.35
//
// The Migrate implementation is idempotent and finds all seen commit
// records and deletes all *except* the record corresponding to the
// highest height.
package scmigrate

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"sort"

	"github.com/gogo/protobuf/proto"
	"github.com/google/orderedcode"
	dbm "github.com/tendermint/tm-db"

	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
	"github.com/tendermint/tendermint/types"
)

type toMigrate struct {
	key    []byte
	commit *types.Commit
}

const prefixSeenCommit = int64(3)

func makeKeyFromPrefix(ids ...int64) []byte {
	vals := make([]interface{}, len(ids))
	for idx := range ids {
		vals[idx] = ids[idx]
	}

	key, err := orderedcode.Append(nil, vals...)
	if err != nil {
		panic(err)
	}
	return key
}

func makeToMigrate(val []byte) (*types.Commit, error) {
	if len(val) == 0 {
		return nil, errors.New("empty value")
	}

	var pbc = new(tmproto.Commit)

	if err := proto.Unmarshal(val, pbc); err != nil {
		return nil, fmt.Errorf("error reading block seen commit: %w", err)
	}

	commit, err := types.CommitFromProto(pbc)
	if commit == nil {
		// theoretically we should error for all errors, but
		// there's no reason to keep junk data in the
		// database, and it makes testing easier.
		if err != nil {
			return nil, fmt.Errorf("error from proto commit: %w", err)
		}
		return nil, fmt.Errorf("missing commit")
	}

	return commit, nil
}

func sortMigrations(scData []toMigrate) {
	// put this in it's own function just to make it testable
	sort.SliceStable(scData, func(i, j int) bool {
		return scData[i].commit.Height > scData[j].commit.Height
	})
}

func getAllSeenCommits(ctx context.Context, db dbm.DB) ([]toMigrate, error) {
	scKeyPrefix := makeKeyFromPrefix(prefixSeenCommit)
	iter, err := db.Iterator(
		scKeyPrefix,
		makeKeyFromPrefix(prefixSeenCommit+1),
	)
	if err != nil {
		return nil, err
	}

	scData := []toMigrate{}
	for ; iter.Valid(); iter.Next() {
		if err := ctx.Err(); err != nil {
			return nil, err
		}

		k := iter.Key()
		nk := make([]byte, len(k))
		copy(nk, k)

		if !bytes.HasPrefix(nk, scKeyPrefix) {
			break
		}
		commit, err := makeToMigrate(iter.Value())
		if err != nil {
			return nil, err
		}

		scData = append(scData, toMigrate{
			key:    nk,
			commit: commit,
		})
	}
	if err := iter.Error(); err != nil {
		return nil, err
	}
	if err := iter.Close(); err != nil {
		return nil, err
	}
	return scData, nil
}

func renameRecord(ctx context.Context, db dbm.DB, keep toMigrate) error {
	wantKey := makeKeyFromPrefix(prefixSeenCommit)
	if bytes.Equal(keep.key, wantKey) {
		return nil // we already did this conversion
	}

	// This record's key has already been converted to the "new" format, we just
	// now need to trim off the tail.
	val, err := db.Get(keep.key)
	if err != nil {
		return err
	}

	batch := db.NewBatch()
	if err := batch.Delete(keep.key); err != nil {
		return err
	}
	if err := batch.Set(wantKey, val); err != nil {
		return err
	}
	werr := batch.Write()
	cerr := batch.Close()
	if werr != nil {
		return werr
	}
	return cerr
}

func deleteRecords(ctx context.Context, db dbm.DB, scData []toMigrate) error {
	// delete all the remaining stale values in a single batch
	batch := db.NewBatch()

	for _, mg := range scData {
		if err := batch.Delete(mg.key); err != nil {
			return err
		}
	}

	if err := batch.WriteSync(); err != nil {
		return err
	}

	if err := batch.Close(); err != nil {
		return err
	}
	return nil
}

func Migrate(ctx context.Context, db dbm.DB) error {
	scData, err := getAllSeenCommits(ctx, db)
	if err != nil {
		return fmt.Errorf("sourcing tasks to migrate: %w", err)
	} else if len(scData) == 0 {
		return nil // nothing to do
	}

	// Sort commits in decreasing order of height.
	sortMigrations(scData)

	// Keep and rename the newest seen commit, delete the rest.
	// In TM < v0.35 we kept a last-seen commit for each height; in v0.35 we
	// retain only the latest.
	keep, remove := scData[0], scData[1:]

	if err := renameRecord(ctx, db, keep); err != nil {
		return fmt.Errorf("renaming seen commit record: %w", err)
	}

	if len(remove) == 0 {
		return nil
	}

	// Remove any older seen commits. Prior to v0.35, we kept these records for
	// all heights, but v0.35 keeps only the latest.
	if err := deleteRecords(ctx, db, remove); err != nil {
		return fmt.Errorf("writing data: %w", err)
	}

	return nil
}