From 1949095c519ebefa8c4df6a6c7abd02125e071ac Mon Sep 17 00:00:00 2001 From: Simon Kirillov Date: Tue, 29 Mar 2022 19:06:32 +0300 Subject: [PATCH 1/2] Fix empty tendermint version in Docker (#8161) * Fix Dockerfile and scripts * Fix docker scripts * Remove unused scripts * Retrigger checks Co-authored-by: Simon Kirillov Co-authored-by: William Banfield <4561443+williambanfield@users.noreply.github.com> --- DOCKER/.gitignore | 1 - DOCKER/Dockerfile | 3 +-- DOCKER/Dockerfile.build_c-amazonlinux | 28 --------------------------- DOCKER/Dockerfile.testing | 16 --------------- DOCKER/Makefile | 13 ------------- DOCKER/build.sh | 20 ------------------- DOCKER/push.sh | 22 --------------------- Makefile | 5 +---- 8 files changed, 2 insertions(+), 106 deletions(-) delete mode 100644 DOCKER/.gitignore delete mode 100644 DOCKER/Dockerfile.build_c-amazonlinux delete mode 100644 DOCKER/Dockerfile.testing delete mode 100644 DOCKER/Makefile delete mode 100755 DOCKER/build.sh delete mode 100755 DOCKER/push.sh diff --git a/DOCKER/.gitignore b/DOCKER/.gitignore deleted file mode 100644 index 9059c6848..000000000 --- a/DOCKER/.gitignore +++ /dev/null @@ -1 +0,0 @@ -tendermint diff --git a/DOCKER/Dockerfile b/DOCKER/Dockerfile index 2785d7e24..a4792981d 100644 --- a/DOCKER/Dockerfile +++ b/DOCKER/Dockerfile @@ -2,7 +2,7 @@ FROM golang:1.17-alpine as builder RUN apk update && \ apk upgrade && \ - apk --no-cache add make + apk --no-cache add make git COPY / /tendermint WORKDIR /tendermint RUN make build-linux @@ -53,4 +53,3 @@ CMD ["start"] # Expose the data directory as a volume since there's mutable state in there VOLUME [ "$TMHOME" ] - diff --git a/DOCKER/Dockerfile.build_c-amazonlinux b/DOCKER/Dockerfile.build_c-amazonlinux deleted file mode 100644 index df492d188..000000000 --- a/DOCKER/Dockerfile.build_c-amazonlinux +++ /dev/null @@ -1,28 +0,0 @@ -FROM amazonlinux:2 - -RUN yum -y update && \ - yum -y install wget - -RUN wget http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm && \ - rpm -ivh epel-release-latest-7.noarch.rpm - -RUN yum -y groupinstall "Development Tools" -RUN yum -y install leveldb-devel which - -ENV GOVERSION=1.16.5 - -RUN cd /tmp && \ - wget https://dl.google.com/go/go${GOVERSION}.linux-amd64.tar.gz && \ - tar -C /usr/local -xf go${GOVERSION}.linux-amd64.tar.gz && \ - mkdir -p /go/src && \ - mkdir -p /go/bin - -ENV PATH=$PATH:/usr/local/go/bin:/go/bin -ENV GOBIN=/go/bin -ENV GOPATH=/go/src - -RUN mkdir -p /tendermint -WORKDIR /tendermint - -CMD ["/usr/bin/make", "build", "TENDERMINT_BUILD_OPTIONS=cleveldb"] - diff --git a/DOCKER/Dockerfile.testing b/DOCKER/Dockerfile.testing deleted file mode 100644 index 7f86ee180..000000000 --- a/DOCKER/Dockerfile.testing +++ /dev/null @@ -1,16 +0,0 @@ -FROM golang:latest - -# Grab deps (jq, hexdump, xxd, killall) -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - jq bsdmainutils vim-common psmisc netcat - -# Add testing deps for curl -RUN echo 'deb http://httpredir.debian.org/debian testing main non-free contrib' >> /etc/apt/sources.list && \ - apt-get update && \ - apt-get install -y --no-install-recommends curl - -VOLUME /go - -EXPOSE 26656 -EXPOSE 26657 diff --git a/DOCKER/Makefile b/DOCKER/Makefile deleted file mode 100644 index 8f6dff372..000000000 --- a/DOCKER/Makefile +++ /dev/null @@ -1,13 +0,0 @@ -build: - @sh -c "'$(CURDIR)/build.sh'" - -push: - @sh -c "'$(CURDIR)/push.sh'" - -build_testing: - docker build --tag tendermint/testing -f ./Dockerfile.testing . - -build_amazonlinux_buildimage: - docker build -t "tendermint/tendermint:build_c-amazonlinux" -f Dockerfile.build_c-amazonlinux . - -.PHONY: build push build_testing build_amazonlinux_buildimage diff --git a/DOCKER/build.sh b/DOCKER/build.sh deleted file mode 100755 index 2aa42a6cd..000000000 --- a/DOCKER/build.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -set -e - -# Get the tag from the version, or try to figure it out. -if [ -z "$TAG" ]; then - TAG=$(awk -F\" '/TMCoreSemVer =/ { print $2; exit }' < ../version/version.go) -fi -if [ -z "$TAG" ]; then - echo "Please specify a tag." - exit 1 -fi - -TAG_NO_PATCH=${TAG%.*} - -read -p "==> Build 3 docker images with the following tags (latest, $TAG, $TAG_NO_PATCH)? y/n" -n 1 -r -echo -if [[ $REPLY =~ ^[Yy]$ ]] -then - docker build -t "tendermint/tendermint" -t "tendermint/tendermint:$TAG" -t "tendermint/tendermint:$TAG_NO_PATCH" . -fi diff --git a/DOCKER/push.sh b/DOCKER/push.sh deleted file mode 100755 index f228406d4..000000000 --- a/DOCKER/push.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash -set -e - -# Get the tag from the version, or try to figure it out. -if [ -z "$TAG" ]; then - TAG=$(awk -F\" '/TMCoreSemVer =/ { print $2; exit }' < ../version/version.go) -fi -if [ -z "$TAG" ]; then - echo "Please specify a tag." - exit 1 -fi - -TAG_NO_PATCH=${TAG%.*} - -read -p "==> Push 3 docker images with the following tags (latest, $TAG, $TAG_NO_PATCH)? y/n" -n 1 -r -echo -if [[ $REPLY =~ ^[Yy]$ ]] -then - docker push "tendermint/tendermint:latest" - docker push "tendermint/tendermint:$TAG" - docker push "tendermint/tendermint:$TAG_NO_PATCH" -fi diff --git a/Makefile b/Makefile index 13d6ada56..85cd3202d 100644 --- a/Makefile +++ b/Makefile @@ -243,10 +243,8 @@ build-docs: ### Docker image ### ############################################################################### -build-docker: build-linux - cp $(BUILDDIR)/tendermint DOCKER/tendermint +build-docker: docker build --label=tendermint --tag="tendermint/tendermint" -f DOCKER/Dockerfile . - rm -rf DOCKER/tendermint .PHONY: build-docker @@ -343,4 +341,3 @@ split-test-packages:$(BUILDDIR)/packages.txt split -d -n l/$(NUM_SPLIT) $< $<. test-group-%:split-test-packages cat $(BUILDDIR)/packages.txt.$* | xargs go test -mod=readonly -timeout=5m -race -coverprofile=$(BUILDDIR)/$*.profile.out - From 9e643f3628446617fe00eeb408f4173449f6ebb2 Mon Sep 17 00:00:00 2001 From: Sam Kleinman Date: Tue, 29 Mar 2022 12:31:37 -0400 Subject: [PATCH 2/2] migration: remove stale seen commits (#8205) --- cmd/tendermint/commands/key_migrate.go | 8 ++ go.mod | 4 +- scripts/scmigrate/migrate.go | 162 +++++++++++++++++++++++ scripts/scmigrate/migrate_test.go | 176 +++++++++++++++++++++++++ 4 files changed, 348 insertions(+), 2 deletions(-) create mode 100644 scripts/scmigrate/migrate.go create mode 100644 scripts/scmigrate/migrate_test.go diff --git a/cmd/tendermint/commands/key_migrate.go b/cmd/tendermint/commands/key_migrate.go index 928821586..5866be341 100644 --- a/cmd/tendermint/commands/key_migrate.go +++ b/cmd/tendermint/commands/key_migrate.go @@ -9,6 +9,7 @@ import ( cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/libs/log" "github.com/tendermint/tendermint/scripts/keymigrate" + "github.com/tendermint/tendermint/scripts/scmigrate" ) func MakeKeyMigrateCommand(conf *cfg.Config, logger log.Logger) *cobra.Command { @@ -51,6 +52,13 @@ func MakeKeyMigrateCommand(conf *cfg.Config, logger log.Logger) *cobra.Command { return fmt.Errorf("running migration for context %q: %w", dbctx, err) } + + if dbctx == "blockstore" { + if err := scmigrate.Migrate(ctx, db); err != nil { + return fmt.Errorf("running seen commit migration: %w", err) + + } + } } logger.Info("completed database migration successfully") diff --git a/go.mod b/go.mod index c4c4529d4..ed88618e0 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,6 @@ require ( github.com/go-kit/kit v0.12.0 github.com/gogo/protobuf v1.3.2 github.com/golang/protobuf v1.5.2 - github.com/golangci/golangci-lint v1.45.2 github.com/google/orderedcode v0.0.1 github.com/google/uuid v1.3.0 github.com/gorilla/websocket v1.5.0 @@ -30,7 +29,6 @@ require ( github.com/spf13/viper v1.10.1 github.com/stretchr/testify v1.7.1 github.com/tendermint/tm-db v0.6.6 - github.com/vektra/mockery/v2 v2.10.0 golang.org/x/crypto v0.0.0-20220214200702-86341886e292 golang.org/x/net v0.0.0-20211208012354-db4efeb81f4b golang.org/x/sync v0.0.0-20210220032951-036812b2e83c @@ -41,7 +39,9 @@ require ( require ( github.com/creachadair/atomicfile v0.2.4 + github.com/golangci/golangci-lint v1.45.2 github.com/google/go-cmp v0.5.7 + github.com/vektra/mockery/v2 v2.10.0 gotest.tools v2.2.0+incompatible ) diff --git a/scripts/scmigrate/migrate.go b/scripts/scmigrate/migrate.go new file mode 100644 index 000000000..e6eee3d95 --- /dev/null +++ b/scripts/scmigrate/migrate.go @@ -0,0 +1,162 @@ +// Package scmigrate implements a migration for SeenCommit data +// between 0.34 and 0.35 +// +// The Migrate implementation is idempotent and finds all seen commit +// records and deletes all *except* the record corresponding to the +// highest height. +package scmigrate + +import ( + "bytes" + "context" + "errors" + "fmt" + "sort" + + "github.com/gogo/protobuf/proto" + "github.com/google/orderedcode" + dbm "github.com/tendermint/tm-db" + + tmproto "github.com/tendermint/tendermint/proto/tendermint/types" + "github.com/tendermint/tendermint/types" +) + +type toMigrate struct { + key []byte + commit *types.Commit +} + +const prefixSeenCommit = int64(3) + +func makeKeyFromPrefix(ids ...int64) []byte { + vals := make([]interface{}, len(ids)) + for idx := range ids { + vals[idx] = ids[idx] + } + + key, err := orderedcode.Append(nil, vals...) + if err != nil { + panic(err) + } + return key +} + +func makeToMigrate(val []byte) (*types.Commit, error) { + if len(val) == 0 { + return nil, errors.New("empty value") + } + + var pbc = new(tmproto.Commit) + + if err := proto.Unmarshal(val, pbc); err != nil { + return nil, fmt.Errorf("error reading block seen commit: %w", err) + } + + commit, err := types.CommitFromProto(pbc) + if commit == nil { + // theoretically we should error for all errors, but + // there's no reason to keep junk data in the + // database, and it makes testing easier. + if err != nil { + return nil, fmt.Errorf("error from proto commit: %w", err) + } + return nil, fmt.Errorf("missing commit") + } + + return commit, nil +} + +func sortMigrations(scData []toMigrate) { + // put this in it's own function just to make it testable + sort.SliceStable(scData, func(i, j int) bool { + return scData[i].commit.Height > scData[j].commit.Height + }) +} + +func getMigrationsToDelete(in []toMigrate) []toMigrate { return in[1:] } + +func getAllSeenCommits(ctx context.Context, db dbm.DB) ([]toMigrate, error) { + scKeyPrefix := makeKeyFromPrefix(prefixSeenCommit) + iter, err := db.Iterator( + scKeyPrefix, + makeKeyFromPrefix(prefixSeenCommit+1), + ) + if err != nil { + return nil, err + } + + scData := []toMigrate{} + for ; iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return nil, err + } + + k := iter.Key() + nk := make([]byte, len(k)) + copy(nk, k) + + if !bytes.HasPrefix(nk, scKeyPrefix) { + break + } + commit, err := makeToMigrate(iter.Value()) + if err != nil { + return nil, err + } + + scData = append(scData, toMigrate{ + key: nk, + commit: commit, + }) + } + if err := iter.Error(); err != nil { + return nil, err + } + if err := iter.Close(); err != nil { + return nil, err + } + return scData, nil +} + +func deleteRecords(ctx context.Context, db dbm.DB, scData []toMigrate) error { + // delete all the remaining stale values in a single batch + batch := db.NewBatch() + + for _, mg := range scData { + if err := batch.Delete(mg.key); err != nil { + return err + } + } + + if err := batch.WriteSync(); err != nil { + return err + } + + if err := batch.Close(); err != nil { + return err + } + return nil +} + +func Migrate(ctx context.Context, db dbm.DB) error { + scData, err := getAllSeenCommits(ctx, db) + if err != nil { + return fmt.Errorf("sourcing tasks to migrate: %w", err) + } + + // sort earliest->latest commits. + sortMigrations(scData) + + // trim the one we want to save: + scData = getMigrationsToDelete(scData) + + if len(scData) <= 1 { + return nil + } + + // write the migration (remove ) + if err := deleteRecords(ctx, db, scData); err != nil { + return fmt.Errorf("writing data: %w", err) + } + + return nil +} diff --git a/scripts/scmigrate/migrate_test.go b/scripts/scmigrate/migrate_test.go new file mode 100644 index 000000000..abe12584d --- /dev/null +++ b/scripts/scmigrate/migrate_test.go @@ -0,0 +1,176 @@ +package scmigrate + +import ( + "context" + "math/rand" + "testing" + + "github.com/gogo/protobuf/proto" + dbm "github.com/tendermint/tm-db" + + "github.com/tendermint/tendermint/types" +) + +func appendRandomMigrations(in []toMigrate, num int) []toMigrate { + if in == nil { + in = []toMigrate{} + } + + for i := 0; i < num; i++ { + height := rand.Int63() + if height <= 0 { + continue + } + in = append(in, toMigrate{commit: &types.Commit{Height: height}}) + } + return in +} + +func assertWellOrderedMigrations(t *testing.T, testData []toMigrate) { + t.Run("ValuesDescend", func(t *testing.T) { + for idx := range testData { + height := testData[idx].commit.Height + if idx == 0 { + continue + } + prev := testData[idx-1].commit.Height + if prev < height { + t.Fatal("height decreased in sort order") + } + } + }) + t.Run("EarliestIsZero", func(t *testing.T) { + earliestHeight := testData[len(testData)-1].commit.Height + if earliestHeight != 0 { + t.Fatalf("the earliest height is not 0: %d", earliestHeight) + } + }) +} + +func getLatestHeight(data []toMigrate) int64 { + var out int64 + + for _, d := range data { + if d.commit.Height >= out { + out = d.commit.Height + } + } + + return out +} + +func insertTestData(t *testing.T, db dbm.DB, data []toMigrate) { + t.Helper() + + batch := db.NewBatch() + + for idx, val := range data { + payload, err := proto.Marshal(val.commit.ToProto()) + if err != nil { + t.Fatal(err) + } + + if err := batch.Set(makeKeyFromPrefix(prefixSeenCommit, int64(idx)), payload); err != nil { + t.Fatal(err) + } + } + if err := batch.WriteSync(); err != nil { + t.Fatal(err) + } + if err := batch.Close(); err != nil { + t.Fatal(err) + } +} + +func TestMigrations(t *testing.T) { + t.Run("Sort", func(t *testing.T) { + t.Run("HandCraftedData", func(t *testing.T) { + testData := []toMigrate{ + {commit: &types.Commit{Height: 100}}, + {commit: &types.Commit{Height: 0}}, + {commit: &types.Commit{Height: 8}}, + {commit: &types.Commit{Height: 1}}, + } + + sortMigrations(testData) + assertWellOrderedMigrations(t, testData) + }) + t.Run("RandomGeneratedData", func(t *testing.T) { + testData := []toMigrate{{commit: &types.Commit{Height: 0}}} + + testData = appendRandomMigrations(testData, 10000) + + sortMigrations(testData) + assertWellOrderedMigrations(t, testData) + }) + }) + t.Run("GetMigrationsToDelete", func(t *testing.T) { + for i := 1; i < 100; i++ { + data := appendRandomMigrations([]toMigrate{}, i) + toMigrate := getMigrationsToDelete(data) + if len(data) != len(toMigrate)+1 { + t.Fatalf("migration prep did not save one document [original=%d migrations=%d]", len(data), len(toMigrate)) + } + } + }) + t.Run("InvalidMigrations", func(t *testing.T) { + if _, err := makeToMigrate(nil); err == nil { + t.Fatal("should error for nil migrations") + } + if _, err := makeToMigrate([]byte{}); err == nil { + t.Fatal("should error for empty migrations") + } + if _, err := makeToMigrate([]byte("invalid")); err == nil { + t.Fatal("should error for empty migrations") + } + }) + + t.Run("GetSeenCommits", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + db := dbm.NewMemDB() + data := appendRandomMigrations([]toMigrate{}, 100) + insertTestData(t, db, data) + commits, err := getAllSeenCommits(ctx, db) + if err != nil { + t.Fatal(err) + } + if len(commits) != len(data) { + t.Log("inputs", len(data)) + t.Log("commits", len(commits)) + t.Fatal("migrations not found in database") + } + }) + t.Run("Integration", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + db := dbm.NewMemDB() + data := appendRandomMigrations([]toMigrate{}, 1000) + insertTestData(t, db, data) + + latestHeight := getLatestHeight(data) + for _, test := range []string{"Migration", "Idempotency"} { + // run the test twice to make sure that it's + // safe to rerun + t.Run(test, func(t *testing.T) { + if err := Migrate(ctx, db); err != nil { + t.Fatal(err) + } + + post, err := getAllSeenCommits(ctx, db) + if err != nil { + t.Fatal(err) + } + if len(post) != 1 { + t.Fatal("migration was not successful") + } + if post[0].commit.Height != latestHeight { + t.Fatal("migration did not save correct document") + } + }) + } + }) + +}