Merge branch 'master' into callum/consolidate-store-into-pool

CHANGELOG_PENDING.md

@@ -29,5 +29,11 @@ Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermi
 - [types] [\#4638](https://github.com/tendermint/tendermint/pull/4638) Implement `Header#ValidateBasic` (@alexanderbez)
 - [txindex] [\#4466](https://github.com/tendermint/tendermint/pull/4466) Allow to index an event at runtime (@favadi)
 - [evidence] [\#4722](https://github.com/tendermint/tendermint/pull/4722) Improved evidence db (@cmwaters)
+- [buildsystem] [\#4378](https://github.com/tendermint/tendermint/pull/4738) Replace build_c and install_c with TENDERMINT_BUILD_OPTIONS parsing. The following options are available:
+  - nostrip: don't strip debugging symbols nor DWARF tables.
+  - cleveldb: use cleveldb as db backend instead of goleveldb.
+  - race: pass -race to go build and enable data race detection.
 
 ### BUG FIXES:
+
+- [light] [\#4741](https://github.com/tendermint/tendermint/pull/4741) Correctly return  `ErrSignedHeaderNotFound` and `ErrValidatorSetNotFound` on corresponding RPC errors (@erikgrinaker)

DOCKER/Dockerfile.build_c-amazonlinux

@@ -24,5 +24,5 @@ ENV GOPATH=/go/src
 RUN mkdir -p /tendermint
 WORKDIR /tendermint
 
-CMD ["/usr/bin/make", "build_c"]
+CMD ["/usr/bin/make", "build", "TENDERMINT_BUILD_OPTIONS=cleveldb"]
 

Makefile

@@ -1,11 +1,33 @@
 PACKAGES=$(shell go list ./...)
 OUTPUT?=build/tendermint
 
-BUILD_TAGS?='tendermint'
-LD_FLAGS = -X github.com/tendermint/tendermint/version.GitCommit=`git rev-parse --short=8 HEAD` -s -w
+BUILD_TAGS?=tendermint
+LD_FLAGS = -X github.com/tendermint/tendermint/version.GitCommit=`git rev-parse --short=8 HEAD`
 BUILD_FLAGS = -mod=readonly -ldflags "$(LD_FLAGS)"
 HTTPS_GIT := https://github.com/tendermint/tendermint.git
 DOCKER_BUF := docker run -v $(shell pwd):/workspace --workdir /workspace bufbuild/buf
+CGO_ENABLED ?= 0
+
+# handle nostrip
+ifeq (,$(findstring nostrip,$(TENDERMINT_BUILD_OPTIONS)))
+  BUILD_FLAGS += -trimpath
+  LD_FLAGS += -s -w
+endif
+
+# handle race
+ifeq (race,$(findstring race,$(TENDERMINT_BUILD_OPTIONS)))
+  CGO_ENABLED=1
+  BUILD_FLAGS += -race
+endif
+
+# handle cleveldb
+ifeq (cleveldb,$(findstring cleveldb,$(TENDERMINT_BUILD_OPTIONS)))
+  CGO_ENABLED=1
+  BUILD_TAGS += cleveldb
+endif
+
+# allow users to pass additional flags via the conventional LDFLAGS variable
+LD_FLAGS += $(LDFLAGS)
 
 all: check build test install
 .PHONY: all
@@ -19,25 +41,13 @@ include tests.mk
 ###############################################################################
 
 build:
-	CGO_ENABLED=0 go build $(BUILD_FLAGS) -tags $(BUILD_TAGS) -o $(OUTPUT) ./cmd/tendermint/
+	CGO_ENABLED=$(CGO_ENABLED) go build $(BUILD_FLAGS) -tags '$(BUILD_TAGS)' -o $(OUTPUT) ./cmd/tendermint/
 .PHONY: build
 
-build_c:
-	CGO_ENABLED=1 go build $(BUILD_FLAGS) -tags "$(BUILD_TAGS) cleveldb" -o $(OUTPUT) ./cmd/tendermint/
-.PHONY: build_c
-
-build_race:
-	CGO_ENABLED=1 go build -race $(BUILD_FLAGS) -tags $(BUILD_TAGS) -o $(OUTPUT) ./cmd/tendermint
-.PHONY: build_race
-
 install:
-	CGO_ENABLED=0 go install $(BUILD_FLAGS) -tags $(BUILD_TAGS) ./cmd/tendermint
+	CGO_ENABLED=$(CGO_ENABLED) go install $(BUILD_FLAGS) -tags $(BUILD_TAGS) ./cmd/tendermint
 .PHONY: install
 
-install_c:
-	CGO_ENABLED=1 go install $(BUILD_FLAGS) -tags "$(BUILD_TAGS) cleveldb" ./cmd/tendermint
-.PHONY: install_c
-
 ###############################################################################
 ###                                Protobuf                                 ###
 ###############################################################################
@@ -197,9 +207,9 @@ build-docker-localnode:
 	@cd networks/local && make
 .PHONY: build-docker-localnode
 
-# Runs `make build_c` from within an Amazon Linux (v2)-based Docker build
-# container in order to build an Amazon Linux-compatible binary. Produces a
-# compatible binary at ./build/tendermint
+# Runs `make build TENDERMINT_BUILD_OPTIONS=cleveldb` from within an Amazon
+# Linux (v2)-based Docker build container in order to build an Amazon
+# Linux-compatible binary. Produces a compatible binary at ./build/tendermint
 build_c-amazonlinux:
 	$(MAKE) -C ./DOCKER build_amazonlinux_buildimage
 	docker run --rm -it -v `pwd`:/tendermint tendermint/tendermint:build_c-amazonlinux

docs/introduction/install.md

@@ -119,13 +119,13 @@ db_backend = "cleveldb"
 To install Tendermint, run:
 
 ```
-CGO_LDFLAGS="-lsnappy" make install_c
+CGO_LDFLAGS="-lsnappy" make install TENDERMINT_BUILD_OPTIONS=cleveldb
 ```
 
 or run:
 
 ```
-CGO_LDFLAGS="-lsnappy" make build_c
+CGO_LDFLAGS="-lsnappy" make build TENDERMINT_BUILD_OPTIONS=cleveldb
 ```
 
 which puts the binary in `./build`.

docs/tendermint-core/running-in-production.md

@@ -10,7 +10,7 @@ By default, Tendermint uses the `syndtr/goleveldb` package for its in-process
 key-value database. Unfortunately, this implementation of LevelDB seems to suffer under heavy load (see
 [#226](https://github.com/syndtr/goleveldb/issues/226)). It may be best to
 install the real C-implementation of LevelDB and compile Tendermint to use
-that using `make build_c`. See the [install instructions](../introduction/install.md) for details.
+that using `make build TENDERMINT_BUILD_OPTIONS=cleveldb`. See the [install instructions](../introduction/install.md) for details.
 
 Tendermint keeps multiple distinct databases in the `$TMROOT/data`:
 

lite2/provider/http/http.go

@@ -3,6 +3,7 @@ package http
 import (
 	"errors"
 	"fmt"
+	"regexp"
 	"strings"
 
 	"github.com/tendermint/tendermint/lite2/provider"
@@ -11,6 +12,9 @@ import (
 	"github.com/tendermint/tendermint/types"
 )
 
+// This is very brittle, see: https://github.com/tendermint/tendermint/issues/4740
+var regexpMissingHeight = regexp.MustCompile(`height \d+ (must be less than or equal to|is not available)`)
+
 // http provider uses an RPC client to obtain the necessary information.
 type http struct {
 	chainID string
@@ -62,7 +66,7 @@ func (p *http) SignedHeader(height int64) (*types.SignedHeader, error) {
 	commit, err := p.client.Commit(h)
 	if err != nil {
 		// TODO: standartise errors on the RPC side
-		if strings.Contains(err.Error(), "height must be less than or equal") {
+		if regexpMissingHeight.MatchString(err.Error()) {
 			return nil, provider.ErrSignedHeaderNotFound
 		}
 		return nil, err
@@ -92,7 +96,7 @@ func (p *http) ValidatorSet(height int64) (*types.ValidatorSet, error) {
 	res, err := p.client.Validators(h, 0, maxPerPage)
 	if err != nil {
 		// TODO: standartise errors on the RPC side
-		if strings.Contains(err.Error(), "height must be less than or equal") {
+		if regexpMissingHeight.MatchString(err.Error()) {
 			return nil, provider.ErrValidatorSetNotFound
 		}
 		return nil, err

lite2/provider/http/http_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/tendermint/tendermint/abci/example/kvstore"
+	"github.com/tendermint/tendermint/lite2/provider"
 	"github.com/tendermint/tendermint/lite2/provider/http"
 	litehttp "github.com/tendermint/tendermint/lite2/provider/http"
 	rpcclient "github.com/tendermint/tendermint/rpc/client"
@@ -33,6 +34,7 @@ func TestNewProvider(t *testing.T) {
 
 func TestMain(m *testing.M) {
 	app := kvstore.NewApplication()
+	app.RetainBlocks = 5
 	node := rpctest.StartTendermint(app)
 
 	code := m.Run()
@@ -73,8 +75,25 @@ func TestProvider(t *testing.T) {
 	assert.Nil(t, sh.ValidateBasic(chainID))
 
 	// historical queries now work :)
-	lower := sh.Height - 5
+	lower := sh.Height - 3
 	sh, err = p.SignedHeader(lower)
 	assert.Nil(t, err, "%+v", err)
 	assert.Equal(t, lower, sh.Height)
+
+	// fetching missing heights (both future and pruned) should return appropriate errors
+	_, err = p.SignedHeader(1000)
+	require.Error(t, err)
+	assert.Equal(t, provider.ErrSignedHeaderNotFound, err)
+
+	_, err = p.ValidatorSet(1000)
+	require.Error(t, err)
+	assert.Equal(t, provider.ErrValidatorSetNotFound, err)
+
+	_, err = p.SignedHeader(1)
+	require.Error(t, err)
+	assert.Equal(t, provider.ErrSignedHeaderNotFound, err)
+
+	_, err = p.ValidatorSet(1)
+	require.Error(t, err)
+	assert.Equal(t, provider.ErrValidatorSetNotFound, err)
 }