mirrors/tendermint
1
0
Fork 0
mirror of https://github.com/tendermint/tendermint.git synced 2026-02-08 04:50:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into thane/7655-vote-extensions

Browse Source
This commit is contained in:
Thane Thomson
2022-03-29 14:12:22 -04:00
committed by GitHub
parent a03504ddae 9e643f3628
commit 730b245e3b
12 changed files with 350 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
DOCKER/.gitignore vendored
View File

@@ -1 +0,0 @@
tendermint

3
DOCKER/Dockerfile
View File

@@ -2,7 +2,7 @@
FROM golang:1.17-alpine as builder
RUN apk update && \
apk upgrade && \
apk --no-cache add make
apk --no-cache add make git
COPY / /tendermint
WORKDIR /tendermint
RUN make build-linux
@@ -53,4 +53,3 @@ CMD ["start"]
# Expose the data directory as a volume since there's mutable state in there
VOLUME [ "$TMHOME" ]

28
DOCKER/Dockerfile.build_c-amazonlinux
View File

@@ -1,28 +0,0 @@
FROM amazonlinux:2
RUN yum -y update && \
yum -y install wget
RUN wget http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm && \
rpm -ivh epel-release-latest-7.noarch.rpm
RUN yum -y groupinstall "Development Tools"
RUN yum -y install leveldb-devel which
ENV GOVERSION=1.16.5
RUN cd /tmp && \
wget https://dl.google.com/go/go${GOVERSION}.linux-amd64.tar.gz && \
tar -C /usr/local -xf go${GOVERSION}.linux-amd64.tar.gz && \
mkdir -p /go/src && \
mkdir -p /go/bin
ENV PATH=$PATH:/usr/local/go/bin:/go/bin
ENV GOBIN=/go/bin
ENV GOPATH=/go/src
RUN mkdir -p /tendermint
WORKDIR /tendermint
CMD ["/usr/bin/make", "build", "TENDERMINT_BUILD_OPTIONS=cleveldb"]

16
DOCKER/Dockerfile.testing
View File

@@ -1,16 +0,0 @@
FROM golang:latest
# Grab deps (jq, hexdump, xxd, killall)
RUN apt-get update && \
apt-get install -y --no-install-recommends \
jq bsdmainutils vim-common psmisc netcat
# Add testing deps for curl
RUN echo 'deb http://httpredir.debian.org/debian testing main non-free contrib' >> /etc/apt/sources.list && \
apt-get update && \
apt-get install -y --no-install-recommends curl
VOLUME /go
EXPOSE 26656
EXPOSE 26657

13
DOCKER/Makefile
View File

@@ -1,13 +0,0 @@
build:
@sh -c "'$(CURDIR)/build.sh'"
push:
@sh -c "'$(CURDIR)/push.sh'"
build_testing:
docker build --tag tendermint/testing -f ./Dockerfile.testing .
build_amazonlinux_buildimage:
docker build -t "tendermint/tendermint:build_c-amazonlinux" -f Dockerfile.build_c-amazonlinux .
.PHONY: build push build_testing build_amazonlinux_buildimage

20
DOCKER/build.sh
View File

@@ -1,20 +0,0 @@
#!/usr/bin/env bash
set -e
# Get the tag from the version, or try to figure it out.
if [ -z "$TAG" ]; then
TAG=$(awk -F\" '/TMCoreSemVer =/ { print $2; exit }' < ../version/version.go)
fi
if [ -z "$TAG" ]; then
echo "Please specify a tag."
exit 1
fi
TAG_NO_PATCH=${TAG%.*}
read -p "==> Build 3 docker images with the following tags (latest, $TAG, $TAG_NO_PATCH)? y/n" -n 1 -r
echo
if [[ $REPLY =~ ^[Yy]$ ]]
then
docker build -t "tendermint/tendermint" -t "tendermint/tendermint:$TAG" -t "tendermint/tendermint:$TAG_NO_PATCH" .
fi

22
DOCKER/push.sh
View File

@@ -1,22 +0,0 @@
#!/usr/bin/env bash
set -e
# Get the tag from the version, or try to figure it out.
if [ -z "$TAG" ]; then
TAG=$(awk -F\" '/TMCoreSemVer =/ { print $2; exit }' < ../version/version.go)
fi
if [ -z "$TAG" ]; then
echo "Please specify a tag."
exit 1
fi
TAG_NO_PATCH=${TAG%.*}
read -p "==> Push 3 docker images with the following tags (latest, $TAG, $TAG_NO_PATCH)? y/n" -n 1 -r
echo
if [[ $REPLY =~ ^[Yy]$ ]]
then
docker push "tendermint/tendermint:latest"
docker push "tendermint/tendermint:$TAG"
docker push "tendermint/tendermint:$TAG_NO_PATCH"
fi

5
Makefile
View File

@@ -243,10 +243,8 @@ build-docs:
### Docker image ###
###############################################################################
build-docker: build-linux
cp $(BUILDDIR)/tendermint DOCKER/tendermint
build-docker:
docker build --label=tendermint --tag="tendermint/tendermint" -f DOCKER/Dockerfile .
rm -rf DOCKER/tendermint
.PHONY: build-docker
@@ -343,4 +341,3 @@ split-test-packages:$(BUILDDIR)/packages.txt
split -d -n l/$(NUM_SPLIT) $< $<.
test-group-%:split-test-packages
cat $(BUILDDIR)/packages.txt.$* | xargs go test -mod=readonly -timeout=5m -race -coverprofile=$(BUILDDIR)/$*.profile.out

8
cmd/tendermint/commands/key_migrate.go
View File

@@ -9,6 +9,7 @@ import (
cfg "github.com/tendermint/tendermint/config"
"github.com/tendermint/tendermint/libs/log"
"github.com/tendermint/tendermint/scripts/keymigrate"
"github.com/tendermint/tendermint/scripts/scmigrate"
)
func MakeKeyMigrateCommand(conf *cfg.Config, logger log.Logger) *cobra.Command {
@@ -51,6 +52,13 @@ func MakeKeyMigrateCommand(conf *cfg.Config, logger log.Logger) *cobra.Command {
return fmt.Errorf("running migration for context %q: %w",
dbctx, err)
}
if dbctx == "blockstore" {
if err := scmigrate.Migrate(ctx, db); err != nil {
return fmt.Errorf("running seen commit migration: %w", err)
}
}
}
logger.Info("completed database migration successfully")

4
go.mod
View File

@@ -11,7 +11,6 @@ require (
github.com/go-kit/kit v0.12.0
github.com/gogo/protobuf v1.3.2
github.com/golang/protobuf v1.5.2
github.com/golangci/golangci-lint v1.45.2
github.com/google/orderedcode v0.0.1
github.com/google/uuid v1.3.0
github.com/gorilla/websocket v1.5.0
@@ -30,7 +29,6 @@ require (
github.com/spf13/viper v1.10.1
github.com/stretchr/testify v1.7.1
github.com/tendermint/tm-db v0.6.6
github.com/vektra/mockery/v2 v2.10.0
golang.org/x/crypto v0.0.0-20220214200702-86341886e292
golang.org/x/net v0.0.0-20211208012354-db4efeb81f4b
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
@@ -41,7 +39,9 @@ require (
require (
github.com/creachadair/atomicfile v0.2.4
github.com/golangci/golangci-lint v1.45.2
github.com/google/go-cmp v0.5.7
github.com/vektra/mockery/v2 v2.10.0
gotest.tools v2.2.0+incompatible
)

162
scripts/scmigrate/migrate.go Normal file
View File

@@ -0,0 +1,162 @@
// Package scmigrate implements a migration for SeenCommit data
// between 0.34 and 0.35
//
// The Migrate implementation is idempotent and finds all seen commit
// records and deletes all *except* the record corresponding to the
// highest height.
package scmigrate
import (
"bytes"
"context"
"errors"
"fmt"
"sort"
"github.com/gogo/protobuf/proto"
"github.com/google/orderedcode"
dbm "github.com/tendermint/tm-db"
tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
"github.com/tendermint/tendermint/types"
)
type toMigrate struct {
key []byte
commit *types.Commit
}
const prefixSeenCommit = int64(3)
func makeKeyFromPrefix(ids ...int64) []byte {
vals := make([]interface{}, len(ids))
for idx := range ids {
vals[idx] = ids[idx]
}
key, err := orderedcode.Append(nil, vals...)
if err != nil {
panic(err)
}
return key
}
func makeToMigrate(val []byte) (*types.Commit, error) {
if len(val) == 0 {
return nil, errors.New("empty value")
}
var pbc = new(tmproto.Commit)
if err := proto.Unmarshal(val, pbc); err != nil {
return nil, fmt.Errorf("error reading block seen commit: %w", err)
}
commit, err := types.CommitFromProto(pbc)
if commit == nil {
// theoretically we should error for all errors, but
// there's no reason to keep junk data in the
// database, and it makes testing easier.
if err != nil {
return nil, fmt.Errorf("error from proto commit: %w", err)
}
return nil, fmt.Errorf("missing commit")
}
return commit, nil
}
func sortMigrations(scData []toMigrate) {
// put this in it's own function just to make it testable
sort.SliceStable(scData, func(i, j int) bool {
return scData[i].commit.Height > scData[j].commit.Height
})
}
func getMigrationsToDelete(in []toMigrate) []toMigrate { return in[1:] }
func getAllSeenCommits(ctx context.Context, db dbm.DB) ([]toMigrate, error) {
scKeyPrefix := makeKeyFromPrefix(prefixSeenCommit)
iter, err := db.Iterator(
scKeyPrefix,
makeKeyFromPrefix(prefixSeenCommit+1),
)
if err != nil {
return nil, err
}
scData := []toMigrate{}
for ; iter.Valid(); iter.Next() {
if err := ctx.Err(); err != nil {
return nil, err
}
k := iter.Key()
nk := make([]byte, len(k))
copy(nk, k)
if !bytes.HasPrefix(nk, scKeyPrefix) {
break
}
commit, err := makeToMigrate(iter.Value())
if err != nil {
return nil, err
}
scData = append(scData, toMigrate{
key: nk,
commit: commit,
})
}
if err := iter.Error(); err != nil {
return nil, err
}
if err := iter.Close(); err != nil {
return nil, err
}
return scData, nil
}
func deleteRecords(ctx context.Context, db dbm.DB, scData []toMigrate) error {
// delete all the remaining stale values in a single batch
batch := db.NewBatch()
for _, mg := range scData {
if err := batch.Delete(mg.key); err != nil {
return err
}
}
if err := batch.WriteSync(); err != nil {
return err
}
if err := batch.Close(); err != nil {
return err
}
return nil
}
func Migrate(ctx context.Context, db dbm.DB) error {
scData, err := getAllSeenCommits(ctx, db)
if err != nil {
return fmt.Errorf("sourcing tasks to migrate: %w", err)
}
// sort earliest->latest commits.
sortMigrations(scData)
// trim the one we want to save:
scData = getMigrationsToDelete(scData)
if len(scData) <= 1 {
return nil
}
// write the migration (remove )
if err := deleteRecords(ctx, db, scData); err != nil {
return fmt.Errorf("writing data: %w", err)
}
return nil
}

176
scripts/scmigrate/migrate_test.go Normal file
View File

@@ -0,0 +1,176 @@
package scmigrate
import (
"context"
"math/rand"
"testing"
"github.com/gogo/protobuf/proto"
dbm "github.com/tendermint/tm-db"
"github.com/tendermint/tendermint/types"
)
func appendRandomMigrations(in []toMigrate, num int) []toMigrate {
if in == nil {
in = []toMigrate{}
}
for i := 0; i < num; i++ {
height := rand.Int63()
if height <= 0 {
continue
}
in = append(in, toMigrate{commit: &types.Commit{Height: height}})
}
return in
}
func assertWellOrderedMigrations(t *testing.T, testData []toMigrate) {
t.Run("ValuesDescend", func(t *testing.T) {
for idx := range testData {
height := testData[idx].commit.Height
if idx == 0 {
continue
}
prev := testData[idx-1].commit.Height
if prev < height {
t.Fatal("height decreased in sort order")
}
}
})
t.Run("EarliestIsZero", func(t *testing.T) {
earliestHeight := testData[len(testData)-1].commit.Height
if earliestHeight != 0 {
t.Fatalf("the earliest height is not 0: %d", earliestHeight)
}
})
}
func getLatestHeight(data []toMigrate) int64 {
var out int64
for _, d := range data {
if d.commit.Height >= out {
out = d.commit.Height
}
}
return out
}
func insertTestData(t *testing.T, db dbm.DB, data []toMigrate) {
t.Helper()
batch := db.NewBatch()
for idx, val := range data {
payload, err := proto.Marshal(val.commit.ToProto())
if err != nil {
t.Fatal(err)
}
if err := batch.Set(makeKeyFromPrefix(prefixSeenCommit, int64(idx)), payload); err != nil {
t.Fatal(err)
}
}
if err := batch.WriteSync(); err != nil {
t.Fatal(err)
}
if err := batch.Close(); err != nil {
t.Fatal(err)
}
}
func TestMigrations(t *testing.T) {
t.Run("Sort", func(t *testing.T) {
t.Run("HandCraftedData", func(t *testing.T) {
testData := []toMigrate{
{commit: &types.Commit{Height: 100}},
{commit: &types.Commit{Height: 0}},
{commit: &types.Commit{Height: 8}},
{commit: &types.Commit{Height: 1}},
}
sortMigrations(testData)
assertWellOrderedMigrations(t, testData)
})
t.Run("RandomGeneratedData", func(t *testing.T) {
testData := []toMigrate{{commit: &types.Commit{Height: 0}}}
testData = appendRandomMigrations(testData, 10000)
sortMigrations(testData)
assertWellOrderedMigrations(t, testData)
})
})
t.Run("GetMigrationsToDelete", func(t *testing.T) {
for i := 1; i < 100; i++ {
data := appendRandomMigrations([]toMigrate{}, i)
toMigrate := getMigrationsToDelete(data)
if len(data) != len(toMigrate)+1 {
t.Fatalf("migration prep did not save one document [original=%d migrations=%d]", len(data), len(toMigrate))
}
}
})
t.Run("InvalidMigrations", func(t *testing.T) {
if _, err := makeToMigrate(nil); err == nil {
t.Fatal("should error for nil migrations")
}
if _, err := makeToMigrate([]byte{}); err == nil {
t.Fatal("should error for empty migrations")
}
if _, err := makeToMigrate([]byte("invalid")); err == nil {
t.Fatal("should error for empty migrations")
}
})
t.Run("GetSeenCommits", func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
db := dbm.NewMemDB()
data := appendRandomMigrations([]toMigrate{}, 100)
insertTestData(t, db, data)
commits, err := getAllSeenCommits(ctx, db)
if err != nil {
t.Fatal(err)
}
if len(commits) != len(data) {
t.Log("inputs", len(data))
t.Log("commits", len(commits))
t.Fatal("migrations not found in database")
}
})
t.Run("Integration", func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
db := dbm.NewMemDB()
data := appendRandomMigrations([]toMigrate{}, 1000)
insertTestData(t, db, data)
latestHeight := getLatestHeight(data)
for _, test := range []string{"Migration", "Idempotency"} {
// run the test twice to make sure that it's
// safe to rerun
t.Run(test, func(t *testing.T) {
if err := Migrate(ctx, db); err != nil {
t.Fatal(err)
}
post, err := getAllSeenCommits(ctx, db)
if err != nil {
t.Fatal(err)
}
if len(post) != 1 {
t.Fatal("migration was not successful")
}
if post[0].commit.Height != latestHeight {
t.Fatal("migration did not save correct document")
}
})
}
})
}