diff --git a/.circleci/config.yml b/.circleci/config.yml index 5a50a61c5..aa6a1089f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -31,8 +31,7 @@ jobs: name: binaries command: | export PATH="$GOBIN:$PATH" - make install - cd abci && make install + make install install_abci - persist_to_workspace: root: /tmp/workspace paths: diff --git a/.gitignore b/.gitignore index be52e6447..c013b0e86 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,6 @@ test/p2p/data/ test/logs coverage.txt docs/_build -docs/tools *.log abci-cli abci/types/types.pb.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 74cb95c93..0f15507c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,22 @@ # Changelog +## 0.22.2 + +*July 10th, 2018* + +IMPROVEMENTS +- More cleanup post repo merge! +- [docs] Include `ecosystem.json` and `tendermint-bft.md` from deprecated `aib-data` repository. +- [config] Add `instrumentation.max_open_connections`, which limits the number + of requests in flight to Prometheus server (if enabled). Default: 3. + + +BUG FIXES +- [rpc] Allow unquoted integers in requests + - NOTE: this is only for URI requests. JSONRPC requests and all responses + will use quoted integers (the proto3 JSON standard). +- [consensus] Fix halt on shutdown + ## 0.22.1 *July 5th, 2018* @@ -20,7 +37,14 @@ BUG FIXES *July 2nd, 2018* BREAKING CHANGES: -- [config] Rename `skip_upnp` to `upnp`, and turn it off by default. +- [config] + * Remove `max_block_size_txs` and `max_block_size_bytes` in favor of + consensus params from the genesis file. + * Rename `skip_upnp` to `upnp`, and turn it off by default. + * Change `max_packet_msg_size` back to `max_packet_msg_payload_size` +- [rpc] + * All integers are encoded as strings (part of the update for Amino v0.10.1) + * `syncing` is now called `catching_up` - [types] Update Amino to v0.10.1 * Amino is now fully proto3 compatible for the basic types * JSON-encoded types now use the type name instead of the prefix bytes @@ -31,9 +55,6 @@ BREAKING CHANGES: * `tmlibs/merkle` -> `crypto/merkle`. Uses SHA256 instead of RIPEMD160 - [tmlibs] Update to v0.9.0 and merge into `libs` * remove `merkle` package (moved to `crypto/merkle`) -- [rpc] - * All integers are encoded as strings (part of the update for Amino v0.10.1) - * `syncing` is now called `catching_up` FEATURES - [cmd] Added metrics (served under `/metrics` using a Prometheus client; diff --git a/abci/Dockerfile.develop b/DOCKER/Dockerfile.abci similarity index 100% rename from abci/Dockerfile.develop rename to DOCKER/Dockerfile.abci diff --git a/Gopkg.lock b/Gopkg.lock index 17b74d74b..b1beaa208 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -180,13 +180,13 @@ version = "v1.0.0" [[projects]] + branch = "master" name = "github.com/prometheus/client_golang" packages = [ "prometheus", "prometheus/promhttp" ] - revision = "c5b7fccd204277076155f10851dad72b76a49317" - version = "v0.8.0" + revision = "d6a9817c4afc94d51115e4a30d449056a3fbf547" [[projects]] branch = "master" @@ -414,6 +414,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "71753a9d4ece4252d23941f116f5ff66c0d5da730a099e5a9867491d223ed93b" + inputs-digest = "6e854634d6c203278ce83bef7725cecbcf90023b0d0e440fb3374acedacbd5ad" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 28394b8b3..ecce0e417 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -88,7 +88,7 @@ [[constraint]] name = "github.com/prometheus/client_golang" - version = "0.8.0" + branch = "master" [[constraint]] branch = "master" diff --git a/Makefile b/Makefile index 079c58f90..b929dbe5c 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,12 @@ GOTOOLS = \ + github.com/mitchellh/gox \ github.com/golang/dep/cmd/dep \ - gopkg.in/alecthomas/gometalinter.v2 + gopkg.in/alecthomas/gometalinter.v2 \ + github.com/gogo/protobuf/protoc-gen-gogo \ + github.com/gogo/protobuf/gogoproto \ + github.com/square/certstrap PACKAGES=$(shell go list ./... | grep -v '/vendor/') +INCLUDE = -I=. -I=${GOPATH}/src -I=${GOPATH}/src/github.com/gogo/protobuf/protobuf BUILD_TAGS?=tendermint BUILD_FLAGS = -ldflags "-X github.com/tendermint/tendermint/version.GitCommit=`git rev-parse --short=8 HEAD`" @@ -11,7 +16,7 @@ check: check_tools ensure_deps ######################################## -### Build +### Build Tendermint build: CGO_ENABLED=0 go build $(BUILD_FLAGS) -tags '$(BUILD_TAGS)' -o build/tendermint ./cmd/tendermint/ @@ -22,10 +27,29 @@ build_race: install: CGO_ENABLED=0 go install $(BUILD_FLAGS) -tags '$(BUILD_TAGS)' ./cmd/tendermint +######################################## +### Build ABCI + +protoc_abci: + ## If you get the following error, + ## "error while loading shared libraries: libprotobuf.so.14: cannot open shared object file: No such file or directory" + ## See https://stackoverflow.com/a/25518702 + protoc $(INCLUDE) --gogo_out=plugins=grpc:. abci/types/*.proto + @echo "--> adding nolint declarations to protobuf generated files" + @awk '/package abci/types/ { print "//nolint: gas"; print; next }1' abci/types/types.pb.go > abci/types/types.pb.go.new + @mv abci/types/types.pb.go.new abci/types/types.pb.go + +build_abci: + @go build -i ./abci/cmd/... + +install_abci: + @go install ./abci/cmd/... + ######################################## ### Distribution # dist builds binaries for all platforms and packages them for distribution +# TODO add abci to these scripts dist: @BUILD_TAGS='$(BUILD_TAGS)' sh -c "'$(CURDIR)/scripts/dist.sh'" @@ -59,6 +83,17 @@ ensure_deps: @echo "--> Running dep" @dep ensure +#For ABCI and libs +get_protoc: + @# https://github.com/google/protobuf/releases + curl -L https://github.com/google/protobuf/releases/download/v3.4.1/protobuf-cpp-3.4.1.tar.gz | tar xvz && \ + cd protobuf-3.4.1 && \ + DIST_LANG=cpp ./configure && \ + make && \ + make install && \ + cd .. && \ + rm -rf protobuf-3.4.1 + draw_deps: @# requires brew install graphviz or apt-get install graphviz go get github.com/RobotsAndPencils/goviz @@ -70,6 +105,37 @@ get_deps_bin_size: @find $(WORK) -type f -name "*.a" | xargs -I{} du -hxs "{}" | sort -rh | sed -e s:${WORK}/::g > deps_bin_size.log @echo "Results can be found here: $(CURDIR)/deps_bin_size.log" +######################################## +### Libs + +protoc_libs: + ## If you get the following error, + ## "error while loading shared libraries: libprotobuf.so.14: cannot open shared object file: No such file or directory" + ## See https://stackoverflow.com/a/25518702 + protoc $(INCLUDE) --go_out=plugins=grpc:. libs/common/*.proto + @echo "--> adding nolint declarations to protobuf generated files" + @awk '/package libs/common/ { print "//nolint: gas"; print; next }1' libs/common/types.pb.go > libs/common/types.pb.go.new + @mv libs/common/types.pb.go.new libs/common/types.pb.go + +gen_certs: clean_certs + ## Generating certificates for TLS testing... + certstrap init --common-name "tendermint.com" --passphrase "" + certstrap request-cert -ip "::" --passphrase "" + certstrap sign "::" --CA "tendermint.com" --passphrase "" + mv out/::.crt out/::.key db/remotedb + +clean_certs: + ## Cleaning TLS testing certificates... + rm -rf out + rm -f db/remotedb/::.crt db/remotedb/::.key + +test_libs: gen_certs + GOCACHE=off go test -tags gcc $(shell go list ./... | grep -v vendor) + make clean_certs + +grpc_dbserver: + protoc -I db/remotedb/proto/ db/remotedb/proto/defs.proto --go_out=plugins=grpc:db/remotedb/proto + ######################################## ### Testing @@ -87,6 +153,15 @@ test_apps: # requires `abci-cli` and `tendermint` binaries installed bash test/app/test.sh +test_abci_apps: + bash abci/tests/test_app/test.sh + +test_abci_cli: + # test the cli against the examples in the tutorial at: + # ./docs/abci-cli.md + # if test fails, update the docs ^ + @ bash abci/tests/test_cli/test.sh + test_persistence: # run the persistence tests using bash # requires `abci-cli` installed @@ -105,17 +180,16 @@ test_p2p: # requires 'tester' the image from above bash test/p2p/test.sh tester -need_abci: - bash scripts/install_abci_apps.sh - test_integrations: make build_docker_test_image make get_tools make get_vendor_deps make install - make need_abci make test_cover make test_apps + make test_abci_apps + make test_abci_cli + make test_libs make test_persistence make test_p2p @@ -233,4 +307,4 @@ build-slate: # To avoid unintended conflicts with file names, always add to .PHONY # unless there is a reason not to. # https://www.gnu.org/software/make/manual/html_node/Phony-Targets.html -.PHONY: check build build_race dist install check_tools get_tools update_tools get_vendor_deps draw_deps test_cover test_apps test_persistence test_p2p test test_race test_integrations test_release test100 vagrant_test fmt build-linux localnet-start localnet-stop build-docker build-docker-localnode sentry-start sentry-config sentry-stop build-slate +.PHONY: check build build_race build_abci dist install install_abci check_tools get_tools update_tools get_vendor_deps draw_deps get_protoc protoc_abci protoc_libs gen_certs clean_certs grpc_dbserver test_cover test_apps test_persistence test_p2p test test_race test_integrations test_release test100 vagrant_test fmt build-linux localnet-start localnet-stop build-docker build-docker-localnode sentry-start sentry-config sentry-stop build-slate diff --git a/abci/Makefile b/abci/Makefile deleted file mode 100644 index 7d1c4b2e5..000000000 --- a/abci/Makefile +++ /dev/null @@ -1,174 +0,0 @@ -GOTOOLS = \ - github.com/mitchellh/gox \ - github.com/golang/dep/cmd/dep \ - gopkg.in/alecthomas/gometalinter.v2 \ - github.com/gogo/protobuf/protoc-gen-gogo \ - github.com/gogo/protobuf/gogoproto -GOTOOLS_CHECK = gox dep gometalinter.v2 protoc protoc-gen-gogo -PACKAGES=$(shell go list ./... | grep -v '/vendor/') -INCLUDE = -I=. -I=${GOPATH}/src -I=${GOPATH}/src/github.com/gogo/protobuf/protobuf - -all: check get_vendor_deps protoc build test install metalinter - -check: check_tools - - -######################################## -### Build - -protoc: - ## If you get the following error, - ## "error while loading shared libraries: libprotobuf.so.14: cannot open shared object file: No such file or directory" - ## See https://stackoverflow.com/a/25518702 - protoc $(INCLUDE) --gogo_out=plugins=grpc:. types/*.proto - @echo "--> adding nolint declarations to protobuf generated files" - @awk '/package types/ { print "//nolint: gas"; print; next }1' types/types.pb.go > types/types.pb.go.new - @mv types/types.pb.go.new types/types.pb.go - -build: - @go build -i ./cmd/... - -dist: - @bash scripts/dist.sh - @bash scripts/publish.sh - -install: - @go install ./cmd/... - - -######################################## -### Tools & dependencies - -check_tools: - @# https://stackoverflow.com/a/25668869 - @echo "Found tools: $(foreach tool,$(GOTOOLS_CHECK),\ - $(if $(shell which $(tool)),$(tool),$(error "No $(tool) in PATH")))" - -get_tools: - @echo "--> Installing tools" - go get -u -v $(GOTOOLS) - @gometalinter.v2 --install - -get_protoc: - @# https://github.com/google/protobuf/releases - curl -L https://github.com/google/protobuf/releases/download/v3.4.1/protobuf-cpp-3.4.1.tar.gz | tar xvz && \ - cd protobuf-3.4.1 && \ - DIST_LANG=cpp ./configure && \ - make && \ - make install && \ - cd .. && \ - rm -rf protobuf-3.4.1 - -update_tools: - @echo "--> Updating tools" - @go get -u $(GOTOOLS) - -get_vendor_deps: - @rm -rf vendor/ - @echo "--> Running dep ensure" - @dep ensure - - -######################################## -### Testing - -test: - @find . -path ./vendor -prune -o -name "*.sock" -exec rm {} \; - @echo "==> Running go test" - @go test $(PACKAGES) - -test_race: - @find . -path ./vendor -prune -o -name "*.sock" -exec rm {} \; - @echo "==> Running go test --race" - @go test -v -race $(PACKAGES) - -### three tests tested by Jenkins -test_cover: - @ bash tests/test_cover.sh - -test_apps: - # test the counter using a go test script - @ bash tests/test_app/test.sh - -test_cli: - # test the cli against the examples in the tutorial at: - # http://tendermint.readthedocs.io/projects/tools/en/master/abci-cli.html - # - # XXX: if this test fails, fix it and update the docs at: - # https://github.com/tendermint/tendermint/blob/develop/docs/abci-cli.rst - @ bash tests/test_cli/test.sh - -######################################## -### Formatting, linting, and vetting - -fmt: - @go fmt ./... - -metalinter: - @echo "==> Running linter" - gometalinter.v2 --vendor --deadline=600s --disable-all \ - --enable=maligned \ - --enable=deadcode \ - --enable=goconst \ - --enable=goimports \ - --enable=gosimple \ - --enable=ineffassign \ - --enable=megacheck \ - --enable=misspell \ - --enable=staticcheck \ - --enable=safesql \ - --enable=structcheck \ - --enable=unconvert \ - --enable=unused \ - --enable=varcheck \ - --enable=vetshadow \ - ./... - #--enable=gas \ - #--enable=dupl \ - #--enable=errcheck \ - #--enable=gocyclo \ - #--enable=golint \ <== comments on anything exported - #--enable=gotype \ - #--enable=interfacer \ - #--enable=unparam \ - #--enable=vet \ - -metalinter_all: - protoc $(INCLUDE) --lint_out=. types/*.proto - gometalinter.v2 --vendor --deadline=600s --enable-all --disable=lll ./... - - -######################################## -### Docker - -DEVDOC_SAVE = docker commit `docker ps -a -n 1 -q` devdoc:local - -docker_build: - docker build -t "tendermint/abci-dev" -f Dockerfile.develop . - -docker_run: - docker run -it -v "$(CURDIR):/go/src/github.com/tendermint/abci" -w "/go/src/github.com/tendermint/abci" "tendermint/abci-dev" /bin/bash - -docker_run_rm: - docker run -it --rm -v "$(CURDIR):/go/src/github.com/tendermint/abci" -w "/go/src/github.com/tendermint/abci" "tendermint/abci-dev" /bin/bash - -devdoc_init: - docker run -it -v "$(CURDIR):/go/src/github.com/tendermint/abci" -w "/go/src/github.com/tendermint/abci" tendermint/devdoc echo - # TODO make this safer - $(call DEVDOC_SAVE) - -devdoc: - docker run -it -v "$(CURDIR):/go/src/github.com/tendermint/abci" -w "/go/src/github.com/tendermint/abci" devdoc:local bash - -devdoc_save: - # TODO make this safer - $(call DEVDOC_SAVE) - -devdoc_clean: - docker rmi $$(docker images -f "dangling=true" -q) - - -# To avoid unintended conflicts with file names, always add to .PHONY -# unless there is a reason not to. -# https://www.gnu.org/software/make/manual/html_node/Phony-Targets.html -.PHONY: check protoc build dist install check_tools get_tools get_protoc update_tools get_vendor_deps test test_race fmt metalinter metalinter_all docker_build docker_run docker_run_rm devdoc_init devdoc devdoc_save devdoc_clean diff --git a/abci/scripts/abci-builder/Dockerfile b/abci/scripts/abci-builder/Dockerfile deleted file mode 100644 index 1182085b4..000000000 --- a/abci/scripts/abci-builder/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM golang:1.9.2 - -RUN apt-get update && apt-get install -y --no-install-recommends \ - zip \ - && rm -rf /var/lib/apt/lists/* - -# We want to ensure that release builds never have any cgo dependencies so we -# switch that off at the highest level. -ENV CGO_ENABLED 0 - -RUN mkdir -p $GOPATH/src/github.com/tendermint/abci -WORKDIR $GOPATH/src/github.com/tendermint/abci diff --git a/abci/scripts/dist.sh b/abci/scripts/dist.sh deleted file mode 100755 index d94ce20f7..000000000 --- a/abci/scripts/dist.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash -set -e - -REPO_NAME="abci" - -# Get the version from the environment, or try to figure it out. -if [ -z $VERSION ]; then - VERSION=$(awk -F\" '/Version =/ { print $2; exit }' < version/version.go) -fi -if [ -z "$VERSION" ]; then - echo "Please specify a version." - exit 1 -fi -echo "==> Building version $VERSION..." - -# Get the parent directory of where this script is. -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done -DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )" - -# Change into that dir because we expect that. -cd "$DIR" - -# Delete the old dir -echo "==> Removing old directory..." -rm -rf build/pkg -mkdir -p build/pkg - - -# Do a hermetic build inside a Docker container. -docker build -t tendermint/${REPO_NAME}-builder scripts/${REPO_NAME}-builder/ -docker run --rm -e "BUILD_TAGS=$BUILD_TAGS" -v "$(pwd)":/go/src/github.com/tendermint/${REPO_NAME} tendermint/${REPO_NAME}-builder ./scripts/dist_build.sh - -# Add $REPO_NAME and $VERSION prefix to package name. -rm -rf ./build/dist -mkdir -p ./build/dist -for FILENAME in $(find ./build/pkg -mindepth 1 -maxdepth 1 -type f); do - FILENAME=$(basename "$FILENAME") - cp "./build/pkg/${FILENAME}" "./build/dist/${REPO_NAME}_${VERSION}_${FILENAME}" -done - -# Make the checksums. -pushd ./build/dist -shasum -a256 ./* > "./${REPO_NAME}_${VERSION}_SHA256SUMS" -popd - -# Done -echo -echo "==> Results:" -ls -hl ./build/dist - -exit 0 diff --git a/abci/scripts/dist_build.sh b/abci/scripts/dist_build.sh deleted file mode 100755 index c45c752ef..000000000 --- a/abci/scripts/dist_build.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash -set -e - -# Get the parent directory of where this script is. -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done -DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )" - -# Change into that dir because we expect that. -cd "$DIR" - -# Get the git commit -GIT_COMMIT="$(git rev-parse --short HEAD)" -GIT_DESCRIBE="$(git describe --tags --always)" -GIT_IMPORT="github.com/tendermint/abci/version" - -# Determine the arch/os combos we're building for -XC_ARCH=${XC_ARCH:-"386 amd64 arm"} -XC_OS=${XC_OS:-"solaris darwin freebsd linux windows"} - -# Make sure build tools are available. -make get_tools - -# Get VENDORED dependencies -make get_vendor_deps - -BINARY="abci-cli" - -# Build! -echo "==> Building..." -"$(which gox)" \ - -os="${XC_OS}" \ - -arch="${XC_ARCH}" \ - -osarch="!darwin/arm !solaris/amd64 !freebsd/amd64" \ - -ldflags "-X ${GIT_IMPORT}.GitCommit='${GIT_COMMIT}' -X ${GIT_IMPORT}.GitDescribe='${GIT_DESCRIBE}'" \ - -output "build/pkg/{{.OS}}_{{.Arch}}/$BINARY" \ - -tags="${BUILD_TAGS}" \ - github.com/tendermint/abci/cmd/$BINARY - -# Zip all the files. -echo "==> Packaging..." -for PLATFORM in $(find ./build/pkg -mindepth 1 -maxdepth 1 -type d); do - OSARCH=$(basename "${PLATFORM}") - echo "--> ${OSARCH}" - - pushd "$PLATFORM" >/dev/null 2>&1 - zip "../${OSARCH}.zip" ./* - popd >/dev/null 2>&1 -done - - - -exit 0 diff --git a/abci/scripts/publish.sh b/abci/scripts/publish.sh deleted file mode 100644 index 715f6c11b..000000000 --- a/abci/scripts/publish.sh +++ /dev/null @@ -1,7 +0,0 @@ -#! /bin/bash - -# Get the version from the environment, or try to figure it out. -if [ -z $VERSION ]; then - VERSION=$(awk -F\" '/Version =/ { print $2; exit }' < version/version.go) -fi -aws s3 cp --recursive build/dist s3://tendermint/binaries/abci/v${VERSION} --acl public-read diff --git a/abci/tests/test_cover.sh b/abci/tests/test_cover.sh deleted file mode 100755 index abbbbe563..000000000 --- a/abci/tests/test_cover.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -set -e -echo "" > coverage.txt - -echo "==> Running unit tests" -for d in $(go list ./... | grep -v vendor); do - go test -race -coverprofile=profile.out -covermode=atomic "$d" - if [ -f profile.out ]; then - cat profile.out >> coverage.txt - rm profile.out - fi -done diff --git a/blockchain/reactor.go b/blockchain/reactor.go index 70a599bab..449a42ff0 100644 --- a/blockchain/reactor.go +++ b/blockchain/reactor.go @@ -5,12 +5,13 @@ import ( "reflect" "time" - "github.com/tendermint/go-amino" + amino "github.com/tendermint/go-amino" + + cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/libs/log" "github.com/tendermint/tendermint/p2p" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" - cmn "github.com/tendermint/tendermint/libs/common" - "github.com/tendermint/tendermint/libs/log" ) const ( @@ -174,7 +175,7 @@ func (bcR *BlockchainReactor) respondToPeer(msg *bcBlockRequestMessage, // Receive implements Reactor by handling 4 types of messages (look below). func (bcR *BlockchainReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) { - msg, err := DecodeMessage(msgBytes) + msg, err := decodeMsg(msgBytes) if err != nil { bcR.Logger.Error("Error decoding message", "src", src, "chId", chID, "msg", msg, "err", err, "bytes", msgBytes) bcR.Switch.StopPeerForError(src, err) @@ -342,17 +343,11 @@ func RegisterBlockchainMessages(cdc *amino.Codec) { cdc.RegisterConcrete(&bcStatusRequestMessage{}, "tendermint/mempool/StatusRequest", nil) } -// DecodeMessage decodes BlockchainMessage. -// TODO: ensure that bz is completely read. -func DecodeMessage(bz []byte) (msg BlockchainMessage, err error) { +func decodeMsg(bz []byte) (msg BlockchainMessage, err error) { if len(bz) > maxMsgSize { - return msg, fmt.Errorf("Msg exceeds max size (%d > %d)", - len(bz), maxMsgSize) + return msg, fmt.Errorf("Msg exceeds max size (%d > %d)", len(bz), maxMsgSize) } err = cdc.UnmarshalBinaryBare(bz, &msg) - if err != nil { - err = cmn.ErrorWrap(err, "DecodeMessage() had bytes left over") - } return } diff --git a/config/config.go b/config/config.go index 22cecf989..2df8eb8e8 100644 --- a/config/config.go +++ b/config/config.go @@ -606,6 +606,12 @@ type InstrumentationConfig struct { // Address to listen for Prometheus collector(s) connections. PrometheusListenAddr string `mapstructure:"prometheus_listen_addr"` + + // Maximum number of simultaneous connections. + // If you want to accept more significant number than the default, make sure + // you increase your OS limits. + // 0 - unlimited. + MaxOpenConnections int `mapstructure:"max_open_connections"` } // DefaultInstrumentationConfig returns a default configuration for metrics @@ -614,6 +620,7 @@ func DefaultInstrumentationConfig() *InstrumentationConfig { return &InstrumentationConfig{ Prometheus: false, PrometheusListenAddr: ":26660", + MaxOpenConnections: 3, } } diff --git a/config/toml.go b/config/toml.go index 084325baa..858d9b31d 100644 --- a/config/toml.go +++ b/config/toml.go @@ -262,6 +262,12 @@ prometheus = {{ .Instrumentation.Prometheus }} # Address to listen for Prometheus collector(s) connections prometheus_listen_addr = "{{ .Instrumentation.PrometheusListenAddr }}" + +# Maximum number of simultaneous connections. +# If you want to accept more significant number than the default, make sure +# you increase your OS limits. +# 0 - unlimited. +max_open_connections = {{ .Instrumentation.MaxOpenConnections }} ` /****** these are for test settings ***********/ diff --git a/consensus/reactor.go b/consensus/reactor.go index 1cac32b80..48ebcad23 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -9,11 +9,11 @@ import ( "github.com/pkg/errors" amino "github.com/tendermint/go-amino" - cmn "github.com/tendermint/tendermint/libs/common" - "github.com/tendermint/tendermint/libs/log" cstypes "github.com/tendermint/tendermint/consensus/types" + cmn "github.com/tendermint/tendermint/libs/common" tmevents "github.com/tendermint/tendermint/libs/events" + "github.com/tendermint/tendermint/libs/log" "github.com/tendermint/tendermint/p2p" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" @@ -80,7 +80,6 @@ func (conR *ConsensusReactor) OnStop() { conR.BaseReactor.OnStop() conR.unsubscribeFromBroadcastEvents() conR.conS.Stop() - conR.conS.Wait() } // SwitchToConsensus switches from fast_sync mode to consensus mode. @@ -184,7 +183,7 @@ func (conR *ConsensusReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) return } - msg, err := DecodeMessage(msgBytes) + msg, err := decodeMsg(msgBytes) if err != nil { conR.Logger.Error("Error decoding message", "src", src, "chId", chID, "msg", msg, "err", err, "bytes", msgBytes) conR.Switch.StopPeerForError(src, err) @@ -1307,11 +1306,9 @@ func RegisterConsensusMessages(cdc *amino.Codec) { cdc.RegisterConcrete(&ProposalHeartbeatMessage{}, "tendermint/ProposalHeartbeat", nil) } -// DecodeMessage decodes the given bytes into a ConsensusMessage. -func DecodeMessage(bz []byte) (msg ConsensusMessage, err error) { +func decodeMsg(bz []byte) (msg ConsensusMessage, err error) { if len(bz) > maxMsgSize { - return msg, fmt.Errorf("Msg exceeds max size (%d > %d)", - len(bz), maxMsgSize) + return msg, fmt.Errorf("Msg exceeds max size (%d > %d)", len(bz), maxMsgSize) } err = cdc.UnmarshalBinaryBare(bz, &msg) return diff --git a/crypto/Makefile b/crypto/Makefile deleted file mode 100644 index a4fd3c37f..000000000 --- a/crypto/Makefile +++ /dev/null @@ -1,99 +0,0 @@ -GOTOOLS = \ - github.com/golang/dep/cmd/dep \ - # gopkg.in/alecthomas/gometalinter.v2 \ - -GOTOOLS_CHECK = dep #gometalinter.v2 - -all: check get_vendor_deps build test install - -check: check_tools - - -######################################## -### Build - -# Command to generate the workd list (kept here for documentation purposes only): -wordlist: - # To re-generate wordlist.go run: - # go-bindata -ignore ".*\.go" -o keys/words/bip39/wordlist.go -pkg "wordlist" keys/bip39/wordlist/... - -build: wordlist - # Nothing else to build! - -install: - # Nothing to install! - - -######################################## -### Tools & dependencies - -check_tools: - @# https://stackoverflow.com/a/25668869 - @echo "Found tools: $(foreach tool,$(GOTOOLS_CHECK),\ - $(if $(shell which $(tool)),$(tool),$(error "No $(tool) in PATH")))" - -get_tools: - @echo "--> Installing tools" - go get -u -v $(GOTOOLS) - #@gometalinter.v2 --install - -update_tools: - @echo "--> Updating tools" - @go get -u $(GOTOOLS) - -get_vendor_deps: - @rm -rf vendor/ - @echo "--> Running dep ensure" - @dep ensure - - -######################################## -### Testing - -test: - CGO_ENABLED=0 go test -p 1 $(shell go list ./... | grep -v vendor) - -######################################## -### Formatting, linting, and vetting - -fmt: - @go fmt ./... - -metalinter: - @echo "==> Running linter" - gometalinter.v2 --vendor --deadline=600s --disable-all \ - --enable=maligned \ - --enable=deadcode \ - --enable=goconst \ - --enable=goimports \ - --enable=gosimple \ - --enable=ineffassign \ - --enable=megacheck \ - --enable=misspell \ - --enable=staticcheck \ - --enable=safesql \ - --enable=structcheck \ - --enable=unconvert \ - --enable=unused \ - --enable=varcheck \ - --enable=vetshadow \ - ./... - #--enable=gas \ - #--enable=dupl \ - #--enable=errcheck \ - #--enable=gocyclo \ - #--enable=golint \ <== comments on anything exported - #--enable=gotype \ - #--enable=interfacer \ - #--enable=unparam \ - #--enable=vet \ - -metalinter_all: - protoc $(INCLUDE) --lint_out=. types/*.proto - gometalinter.v2 --vendor --deadline=600s --enable-all --disable=lll ./... - - -# To avoid unintended conflicts with file names, always add to .PHONY -# unless there is a reason not to. -# https://www.gnu.org/software/make/manual/html_node/Phony-Targets.html -.PHONEY: check build install check_tools get_tools update_tools get_vendor_deps test fmt metalinter metalinter_all diff --git a/docs/README.md b/docs/README.md index 0ed3aaec9..16ea708ad 100644 --- a/docs/README.md +++ b/docs/README.md @@ -11,18 +11,17 @@ replicates it on many machines. In other words, a blockchain. Tendermint requires an application running over the Application Blockchain Interface (ABCI) - and comes packaged with an example application to do so. -Follow the [installation instructions](./install) to get up and running -quickly. For more details on [using tendermint](./using-tendermint) see that +Follow the [installation instructions](./introduction/install) to get up and running +quickly. For more details on [using tendermint](./tendermint-core/using-tendermint) see that and the following sections. ## Networks Testnets can be setup manually on one or more machines, or automatically on one or more machine, using a variety of methods described in the [deploy testnets -section](./deploy-testnets). For more information (and to join) about the -Cosmos Network testnets, see [here](/getting-started/full-node.md). +section](./networks/deploy-testnets). ## Application Development The first step to building application on Tendermint is to [install -ABCI-CLI](./getting-started) and play with the example applications. +ABCI-CLI](./app-dev/getting-started) and play with the example applications. diff --git a/docs/abci-cli.md b/docs/app-dev/abci-cli.md similarity index 100% rename from docs/abci-cli.md rename to docs/app-dev/abci-cli.md diff --git a/docs/abci-spec.md b/docs/app-dev/abci-spec.md similarity index 100% rename from docs/abci-spec.md rename to docs/app-dev/abci-spec.md diff --git a/docs/app-architecture.md b/docs/app-dev/app-architecture.md similarity index 100% rename from docs/app-architecture.md rename to docs/app-dev/app-architecture.md diff --git a/docs/app-development.md b/docs/app-dev/app-development.md similarity index 100% rename from docs/app-development.md rename to docs/app-dev/app-development.md diff --git a/docs/app-dev/ecosystem.json b/docs/app-dev/ecosystem.json new file mode 100644 index 000000000..363f18902 --- /dev/null +++ b/docs/app-dev/ecosystem.json @@ -0,0 +1,213 @@ +{ + "abciApps": [ + { + "name": "Cosmos SDK", + "url": "https://github.com/cosmos/cosmos-sdk", + "language": "Go", + "author": "Cosmos", + "description": + "A prototypical account based crypto currency state machine supporting plugins" + }, + { + "name": "cb-ledger", + "url": "https://github.com/block-finance/cpp-abci", + "language": "C++", + "author": "Block Finance", + "description": + "Custodian Bank Ledger, integrating central banking with the blockchains of tomorrow" + }, + { + "name": "Clearchain", + "url": "https://github.com/tendermint/clearchain", + "language": "Go", + "author": "FXCLR", + "description": + "Application to manage a distributed ledger for money transfers that support multi-currency accounts" + }, + { + "name": "Ethermint", + "url": "https://github.com/tendermint/ethermint", + "language": "Go", + "author": "Tendermint", + "description": "The go-ethereum state machine run as an ABCI app" + }, + { + "name": "Merkle AVL Tree", + "url": "https://github.com/tendermint/merkleeyes", + "language": "Go", + "author": "Tendermint", + "description": "Tendermint IAVL tree implemented as an ABCI app" + }, + { + "name": "Burrow", + "url": "https://github.com/hyperledger/burrow", + "language": "Go", + "author": "Monax Industries", + "description": + "Ethereum Virtual Machine augmented with native permissioning scheme and global key-value store" + }, + { + "name": "Merkle AVL Tree", + "url": "https://github.com/jTMSP/MerkleTree", + "language": "Java", + "author": "jTMSP", + "description": "Tendermint IAVL tree implemented as an ABCI app" + }, + { + "name": "TMChat", + "url": "https://github.com/wolfposd/TMChat", + "language": "Java", + "author": "jTMSP", + "description": "P2P chat using Tendermint" + }, + { + "name": "Comit", + "url": "https://github.com/zballs/comit", + "language": "Go", + "author": "Zach Balder", + "description": "Public service reporting and tracking" + }, + { + "name": "Passchain", + "url": "https://github.com/trusch/passchain", + "language": "Go", + "author": "trusch", + "description": + "Tool to securely store and share passwords, tokens and other short secrets" + }, + { + "name": "Passwerk", + "url": "https://github.com/rigelrozanski/passwerk", + "language": "Go", + "author": "Rigel Rozanski", + "description": "Encrypted storage web-utility backed by Tendermint" + }, + { + "name": "py-tendermint", + "url": "https://github.com/davebryson/py-tendermint", + "language": "Python", + "author": "Dave Bryson", + "description": + "A Python microframework for building blockchain applications with Tendermint" + }, + { + "name": "Stratumn SDK", + "url": "https://github.com/stratumn/sdk", + "language": "Go", + "author": "Stratumn", + "description": "SDK for Proof-of-Process networks" + }, + { + "name": "Lotion", + "url": "https://github.com/keppel/lotion", + "language": "Javascript", + "author": "Judd Keppel", + "description": + "A Javascript microframework for building blockchain applications with Tendermint" + }, + { + "name": "Tendermint Blockchain Chat App", + "url": "https://github.com/SaifRehman/tendermint-chat-app/", + "language": "Javascript", + "author": "Saif Rehman", + "description": + "This is a minimal chat application based on Tendermint using Lotion.js in 30 lines of code!. It also includes web/mobile application built using Ionic 3." + }, + { + "name": "BigchainDB", + "url": "https://github.com/bigchaindb/bigchaindb", + "language": "Python", + "author": "BigchainDB GmbH and the BigchainDB community", + "description": "Blockchain database" + }, + { + "name": "Mint", + "url": "https://github.com/Hashnode/mint", + "language": "Go", + "author": "Hashnode", + "description": "Build blockchain-powered social apps" + } + ], + "abciServers": [ + { + "name": "abci", + "url": "https://github.com/tendermint/abci", + "language": "Go", + "author": "Tendermint" + }, + { + "name": "js-abci", + "url": "https://github.com/tendermint/js-abci", + "language": "Javascript", + "author": "Tendermint" + }, + { + "name": "cpp-tmsp", + "url": "https://github.com/mdyring/cpp-tmsp", + "language": "C++", + "author": "Martin Dyring" + }, + { + "name": "jabci", + "url": "https://github.com/jTendermint/jabci", + "language": "Java", + "author": "jTendermint" + }, + { + "name": "ocaml-tmsp", + "url": "https://github.com/zballs/ocaml-tmsp", + "language": "Ocaml", + "author": "Zach Balder" + }, + { + "name": "abci_server", + "url": "https://github.com/KrzysiekJ/abci_server", + "language": "Erlang", + "author": "Krzysztof Jurewicz" + }, + { + "name": "py-abci", + "url": "https://github.com/davebryson/py-abci", + "language": "Python", + "author": "Dave Bryson" + }, + { + "name": "Spearmint", + "url": "https://github.com/dennismckinnon/spearmint", + "language": "Javascript", + "author": "Dennis McKinnon" + } + ], + "deploymentTools": [ + { + "name": "mintnet-kubernetes", + "url": "https://github.com/tendermint/tools", + "technology": "Docker and Kubernetes", + "author": "Tendermint", + "description": + "Deploy a Tendermint test network using Google's kubernetes" + }, + { + "name": "terraforce", + "url": "https://github.com/tendermint/tools", + "technology": "Terraform", + "author": "Tendermint", + "description": + "Terraform + our custom terraforce tool; deploy a production Tendermint network with load balancing over multiple AWS availability zones" + }, + { + "name": "ansible-tendermint", + "url": "https://github.com/tendermint/tools", + "technology": "Ansible", + "author": "Tendermint", + "description": "Ansible playbooks + Tendermint" + }, + { + "name": "brooklyn-tendermint", + "url": "https://github.com/cloudsoft/brooklyn-tendermint", + "technology": "Clocker for Apache Brooklyn ", + "author": "Cloudsoft", + "description": "Deploy a tendermint test network in docker containers " + } + ] +} diff --git a/docs/ecosystem.md b/docs/app-dev/ecosystem.md similarity index 88% rename from docs/ecosystem.md rename to docs/app-dev/ecosystem.md index 0f1951a08..7960e6c0d 100644 --- a/docs/ecosystem.md +++ b/docs/app-dev/ecosystem.md @@ -7,7 +7,7 @@ Tendermint stack can be found at: We thank the community for their contributions thus far and welcome the addition of new projects. A pull request can be submitted to [this -file](https://github.com/tendermint/aib-data/blob/master/json/ecosystem.json) +file](https://github.com/tendermint/tendermint/blob/master/docs/app-dev/ecosystem.json) to include your project. ## Other Tools diff --git a/docs/getting-started.md b/docs/app-dev/getting-started.md similarity index 100% rename from docs/getting-started.md rename to docs/app-dev/getting-started.md diff --git a/docs/indexing-transactions.md b/docs/app-dev/indexing-transactions.md similarity index 100% rename from docs/indexing-transactions.md rename to docs/app-dev/indexing-transactions.md diff --git a/docs/subscribing-to-events-via-websocket.md b/docs/app-dev/subscribing-to-events-via-websocket.md similarity index 100% rename from docs/subscribing-to-events-via-websocket.md rename to docs/app-dev/subscribing-to-events-via-websocket.md diff --git a/docs/images/tmint-logo-blue.png b/docs/images/tmint-logo-blue.png deleted file mode 100644 index cc4c8fb82..000000000 Binary files a/docs/images/tmint-logo-blue.png and /dev/null differ diff --git a/docs/interviews/tendermint-bft.md b/docs/interviews/tendermint-bft.md new file mode 100644 index 000000000..8b3ad5743 --- /dev/null +++ b/docs/interviews/tendermint-bft.md @@ -0,0 +1,250 @@ +# Interview Transcript with Tendermint core researcher, Zarko Milosevic, by Chjango + +**ZM**: Regarding leader election, it's round robin, but a weighted one. You +take into account the amount of bonded tokens. Depending on how much weight +they have of voting power, they would be elected more frequently. So we do +rotate, but just the guys who are having more voting power would be elected +more frequently. We are having 4 validators, and 1 of them have 2 times more +voting power, they have 2 times more elected as a leader. + +**CC**: 2x more absolute voting power or probabilistic voting power? + +**ZM**: It's actually very deterministic. It's not probabilistic at all. See +[Tendermint proposal election specification][1]. In Tendermint, there is no +pseudorandom leader election. It's a deterministic protocol. So leader election +is a built-in function in the code, so you know exactly—depending on the voting +power in the validator set, you'd know who exactly would be the leader in round +x, x + 1, and so on. There is nothing random there; we are not trying to hide +who would be the leader. It's really well known. It's just that there is a +function, it's a mathematical function, and it's just basically—it's kind of an +implementation detail—it starts from the voting power, and when you are +elected, you get decreased some number, and in each round you keep increasing +depending on your voting power, so that you are elected after k rounds again. +But knowing the validator set and the voting power, it's very simple function, +you can calculate yourself to know exactly who would be next. For each round, +this function will return you the leader for that round. In every round, we do +this computation. It's all part of the same flow. It enforces the properties +which are: proportional to your voting power, you will be elected, and we keep +changing the leaders. So it can't happen to have one guy being more elected +than other guys, if they have the same voting power. So one time it will be guy +B, and next time it will be guy B1. So it's not random. + +**CC**: Assuming the validator set remains unchanged for a month, then if you +run this function, are you able to know exactly who is going to go for that +entire month? + +**ZM**: Yes. + +**CC**: What're the attack scenarios for this? + +**ZM**: This is something which is easily attacked by people who argue that +Tendermint is not decentralized enough. They say that by knowing the leader, +you can DDoS the leader. And by DDoSing the leader, you are able to stop the +progress. Because it's true. If you would be able to DDoS the leader, the +leader would not be able to propose and then effectively will not be making +progress. How we are addressing this thing is Sentry Architecture. So the +validator—or at least a proper validator—will never be available. You don't +know the ip address of the validator. You are never able to open the connection +to the validator. So validator is spawning sentry nodes and this is the single +administration domain and there is only connection from validator in the sense +of sentry nodes. And ip address of validator is not shared in the p2p network. +It’s completely private. This is our answer to DDoS attack. By playing clever +at this sentry node architecture and spawning additional sentry nodes in case, +for ex your sentry nodes are being DDoS’d, bc your sentry nodes are public, +then you will be able to connect to sentry nodes. this is where we will expect +the validator to be clever enough that so that in case they are DDoS’d at the +sentry level, they will spawn a different sentry node and then you communicate +through them. We are in a sense pushing the responsibility on the validator. + +**CC**: So if I understand this correctly, the public identity of the validator +doesn’t even matter because that entity can obfuscate where their real full +nodes reside via a proxy through this sentry architecture. + +**ZM**: Exactly. So you do know what is the address or identity of the validator +but you don’t know the network address of it; you’re not able to attack it +because you don’t know where they are. They are completely obfuscated by the +sentry nodes. There is now, if you really want to figure out….There is the +Tendermint protocol, the structure of the protocol is not fully decentralized +in the sense that the flow of information is going from the round proposer, or +the round coordinator, to other nodes, and then after they receive this it’s +basically like [inaudible: “O to 1”]. So by tracking where this information is +coming from, you might be able to identify who are the sentry nodes behind it. +So if you are doing some network analysis, you might be able to deduce +something. If the thing would be completely stuck, where the validator would +never change their sentry nodes or ip addresses of sentry nodes, it could be +possible to deduce something. This is where economic game comes into play. We +are doing an economics game there. We say that it’s a validator business. If +they are not able to hide themselves well enough, they’ll be DDoS’d and they +will be kicked out of the active validator set. So it’s in their interest. + +[Proposer Selection Procedure in Tendermint][1]. This is how it should work no +matter what implementation. + +**CC**: Going back to the proposer, lets say the validator does get DDoS’d, then +the proposer goes down. What happens? + +**ZM**: How the proposal mechanism works—there’s nothing special there—it goes +through a sequence of rounds. Normal execution of Tendermint is that for each +height, we are going through a sequence of rounds, starting from round 0, and +then we are incrementing through the rounds. The nodes are moving through the +rounds as part of normal procedure until they decide to commit. In case you +have one proposer—the proposer of a single round—being DDoS’d, we will probably +not decide in that round, because he will not be able to send his proposal. So +we will go to the next round, and hopefully the next proposer will be able to +communicate with the validators and then we’ll decide in the next round. + +**CC**: Are there timeouts between one round to another, if a round gets +skipped? + +**ZM**: There are timeouts. It’s a bit more complex. I think we have 5 timeouts. +We may be able to simplify this a bit. What is important to understand is: The +only condition which needs to be satisfied so we can go to the next round is +that your validator is able to communicate with more than 2/3rds of voting +power. To be able to move to the next round, you need to receive more than +2/3rd of voting power equivalent of pre-commit messages. + +We have two kinds of messages: 1) Proposal: Where the current round proposer is +suggesting how the next block should look like. This is first one. Every round +starts with proposer sending a proposal. And then there are two more rounds of +voting, where the validator is trying to agree whether they will commit the +proposal or not. And the first of such vote messages is called `pre-vote` and +the second one is `pre-commit`. Now, to be able to move between steps, between +a `pre-vote` and `pre-commit` step, you need to receive enough number of +messages where if message is sent by validator A, then also this message has a +weight, or voting power which is equal to the voting power of the validator who +sent this message. Before you receive more than 2/3 of voting power messages, you are not +able to move to the higher round. Only when you receive more than 2/3 of +messages, you actually start the timeout. The timeout is happening only after +you receive enough messages. And it happens because of the asynchrony of the +message communication so you give more time to guys with this timeout to +receive some messages which are maybe delayed. + +**CC**: In this way that you just described via the whole network gossiping +before we commit a block, that is what makes Tendermint BFT deterministic in a +partially synchronous setting vs Bitcoin which has synchrony assumptions +whereby blocks are first mined and then gossiped to the network. + +**ZM**: It's true that in Bitcoin, this is where the synchrony assumption comes +to play because if they're not able to communicate timely, they are not able to +converge to a single longest chain. Why are they not able to decrease timeout +in Bitcoin? Because if they would decrease, there would be so many forks that +they won't be able to converge to a single chain. By increasing this +complexity and the block time, they're able to have not so many forks. This is +effectively the timing assumption—the block duration in a sense because it's +enough time so that the decided block is propagated through the network before +someone else start deciding on the same block and creating forks. It's very +different from the consensus algorithms in a distributed computing setup where +Tendermint fits. In Tendermint, where we talk about the timing dependency, they +are really part of this 3-communication step protocol I just explained. We have +the following assumption: If the good guys are not able to communicate timely +and reliably without having message loss within a round, the Tendermint will +not make progress—it will not be making blocks. So if you are in a completely +asynchronous network where messages get lost or delayed unpredictably, +Tendermint will not make progress, it will not create forks, but it will not +decide, it will not tell you what is the next block. For termination, it's a +liveness property of consensus. It's a guarantee to decide. We do need timing +assumptions. Within a round, correct validators are able to communicate to each +other the consensus messages, not the transactions, but consensus messages. +They need to communicate in a timely and reliable fashion. But this doesn't +need to hold forever. It's just that what we are assuming when we say it's a +partially synchronous system, we assume that the system will be going through a +period of asynchrony, where we don't have this guarantee; the messages will be +delayed or some will be lost and then will not make progress for some period of +time, or we're not guaranteed to make progress. And the period of synchrony +where these guarantees hold. And if we think about internet, internet is best +described using such a model. Sometimes when we send a message to SF to +Belgrade, it takes 100 ms, sometimes it takes 300 ms, sometimes it takes 1 s. +But in most cases, it takes 100 ms or less than this. + +There is one thing which would be really nice if you understand it. In a global +wide area network, we can't make assumption on the communication unless we are +very conservative about this. If you want to be very fast, then we can't make +assumption and say we'll be for sure communicating with 1 ms communication +delay. Because of the complexity and various congestion issues on the network, +it might happen that during a short period of time, this doesn't hold. If this +doesn't hold and you depend on this for correctness of your protocol, you will +have a fork. So the partially synchronous protocol, most of them like +Tendermint, they don't depend on the timing assumption from the internet for +correctness. This is where we state: safety always. So we never make a fork no +matter how bad our estimates about the internet communication delays are. We'll +never make a fork, but we do make some assumptions, and these assumptions are +built-in our timeouts in our protocol which are actually adaptive. So we are +adapting to the current condition and this is where we're saying...We do assume +some properties, or some communication delays, to eventually hold on the +network. During this period, we guarantee that we will be deciding and +committing blocks. And we will be doing this very fast. We will be basically on +the speed of the current network. + +**CC**: We make liveness assumptions based on the integrity of the validator +businesses, assuming they're up and running fine. + +**ZM**: This is where we are saying, the protocol will be live if we have at +most 1/3, or a bit less than 1/3, of faulty validators. Which means that all +other guys should be online and available. This is also for liveness. This is +related to the condition that we are not able to make progress in rounds if we +don't receive enough messages. If half of our voting power, or half of our +validators are down, we don't have enough messages, so the protocol is +completely blocked. It doesn't make progress in a round, which means it's not +able to be signed. So it's completely critical for Tendermint that we make +progress in rounds. It's like breathing. Tendermint is breathing. If there is +no progress, it's dead; it's blocked, we're not able to breathe, that's why +we're not able to make progress. + +**CC**: How does Tendermint compare to other consensus algos? + +**ZM**: Tendermint is a very interesting protocol. From an academic point of +view, I'm convinced that there is value there. Hopefully, we prove it by +publishing it on some good conference. What is novel is, if we compare first +Tendermint to this existing BFT problem, it's a continuation of academic +research on BFT consensus. What is novel in Tendermint is that it somehow +merges consensus protocol with gossip. This is completely novel idea. +Originally, in BFT, people were assuming the single administration domain, +small number of nodes, local area network, 4-7 nodes max. If you look at the +research paper, 99% of them have this kind of setup. Wide area was studied but +there is significantly less work in wide area networks. No one studied how to +scale those protocols to hundreds or thousands of nodes before blockchain. It +was always a single administration domain. So in Tendermint now, you are able +to reach consensus among different administration domains which are potentially +hundreds of them in wide area network. The system model is potentially harder +because we have more nodes and wide area network. The second thing is that: +normally, in bft protocols, the protocol itself are normally designed in a way +that has two phases, or two parts. The one which is called normal case, which +is normally quite simple, in this normal case. In spite of some failures, which +are part of the normal execution of the protocol, like for example leader +crashes or leader being DDoS'd, they need to go through a quite complex +protocol, which is like being called view change or leader election or +whatever. These two parts of the same protocol are having quite different +complexity. And most of the people only understand this normal case. In +Tendermint, there is no this difference. We have only one protocol, there are +not two protocols. It's always the same steps and they are much closer to the +normal case than this complex view change protocol. + +_This is a bit too technical but this is on a high level things to remember, +that: The system it addresses it's harder than the others and the algorithm +complexity in Tendermint is simpler._ The initial goal of Jae and Bucky which +is inspired by Raft, is that it's simpler so normal engineers could understand. + +**CC**: Can you expand on the termination requirement? + +_Important point about Liveness in Tendermint_ + +**ZM**: In Tendermint, we are saying, for termination, we are making assumption +that the system is partially synchronous. And in a partially synchronous system +model, we are able to mathematically prove that the protocol will make +decisions; it will decide. + +**CC**: What is a persistent peer? + +**ZM**: It's a list of peer identities, which you will try to establish +connection to them, in case connection is broken, Tendermint will automatically +try to reestablish connection. These are important peers, you will really try +persistently to establish connection to them. For other peers, you just drop it +and try from your address book to connect to someone else. The address book is a +list of peers which you discover that they exist, because we are talking about a +very dynamic network—so the nodes are coming and going away—and the gossiping +protocol is discovering new nodes and gossiping them around. So every node will +keep the list of new nodes it discovers, and when you need to establish +connection to a peer, you'll look to address book and get some addresses from +there. There's categorization/ranking of nodes there. + +[1]: https://github.com/tendermint/tendermint/blob/master/docs/spec/reactors/consensus/proposer-selection.md diff --git a/docs/install.md b/docs/introduction/install.md similarity index 100% rename from docs/install.md rename to docs/introduction/install.md diff --git a/docs/introduction.md b/docs/introduction/introduction.md similarity index 100% rename from docs/introduction.md rename to docs/introduction/introduction.md diff --git a/docs/examples/getting-started.md b/docs/introduction/quick-start.md similarity index 77% rename from docs/examples/getting-started.md rename to docs/introduction/quick-start.md index 7c4663116..8e4908784 100644 --- a/docs/examples/getting-started.md +++ b/docs/introduction/quick-start.md @@ -3,8 +3,7 @@ ## Overview This is a quick start guide. If you have a vague idea about how Tendermint -works and want to get started right away, continue. Otherwise, [review the -documentation](http://tendermint.readthedocs.io/en/master/). +works and want to get started right away, continue. ## Install @@ -135,10 +134,10 @@ This will install `go` and other dependencies, get the Tendermint source code, t Next, use the `tendermint testnet` command to create four directories of config files (found in `./mytestnet`) and copy each directory to the relevant machine in the cloud, so that each machine has `$HOME/mytestnet/node[0-3]` directory. Then from each machine, run: ``` -tendermint node --home ./mytestnet/node0 --proxy_app=kvstore --p2p.persistent_peers="167b80242c300bf0ccfb3ced3dec60dc2a81776e@IP1:26656,3c7a5920811550c04bf7a0b2f1e02ab52317b5e6@IP2:26656,303a1a4312c30525c99ba66522dd81cca56a361a@IP3:26656,b686c2a7f4b1b46dca96af3a0f31a6a7beae0be4@IP4:26656" -tendermint node --home ./mytestnet/node1 --proxy_app=kvstore --p2p.persistent_peers="167b80242c300bf0ccfb3ced3dec60dc2a81776e@IP1:26656,3c7a5920811550c04bf7a0b2f1e02ab52317b5e6@IP2:26656,303a1a4312c30525c99ba66522dd81cca56a361a@IP3:26656,b686c2a7f4b1b46dca96af3a0f31a6a7beae0be4@IP4:26656" -tendermint node --home ./mytestnet/node2 --proxy_app=kvstore --p2p.persistent_peers="167b80242c300bf0ccfb3ced3dec60dc2a81776e@IP1:26656,3c7a5920811550c04bf7a0b2f1e02ab52317b5e6@IP2:26656,303a1a4312c30525c99ba66522dd81cca56a361a@IP3:26656,b686c2a7f4b1b46dca96af3a0f31a6a7beae0be4@IP4:26656" -tendermint node --home ./mytestnet/node3 --proxy_app=kvstore --p2p.persistent_peers="167b80242c300bf0ccfb3ced3dec60dc2a81776e@IP1:26656,3c7a5920811550c04bf7a0b2f1e02ab52317b5e6@IP2:26656,303a1a4312c30525c99ba66522dd81cca56a361a@IP3:26656,b686c2a7f4b1b46dca96af3a0f31a6a7beae0be4@IP4:26656" +tendermint node --home ./mytestnet/node0 --proxy_app=kvstore --p2p.persistent_peers="ID1@IP1:26656,ID2@IP2:26656,ID3@IP3:26656,ID4@IP4:26656" +tendermint node --home ./mytestnet/node1 --proxy_app=kvstore --p2p.persistent_peers="ID1@IP1:26656,ID2@IP2:26656,ID3@IP3:26656,ID4@IP4:26656" +tendermint node --home ./mytestnet/node2 --proxy_app=kvstore --p2p.persistent_peers="ID1@IP1:26656,ID2@IP2:26656,ID3@IP3:26656,ID4@IP4:26656" +tendermint node --home ./mytestnet/node3 --proxy_app=kvstore --p2p.persistent_peers="ID1@IP1:26656,ID2@IP2:26656,ID3@IP3:26656,ID4@IP4:26656" ``` Note that after the third node is started, blocks will start to stream in diff --git a/docs/deploy-testnets.md b/docs/networks/deploy-testnets.md similarity index 100% rename from docs/deploy-testnets.md rename to docs/networks/deploy-testnets.md diff --git a/docs/terraform-and-ansible.md b/docs/networks/terraform-and-ansible.md similarity index 100% rename from docs/terraform-and-ansible.md rename to docs/networks/terraform-and-ansible.md diff --git a/docs/determinism.md b/docs/research/determinism.md similarity index 100% rename from docs/determinism.md rename to docs/research/determinism.md diff --git a/docs/transactional-semantics.md b/docs/research/transactional-semantics.md similarity index 100% rename from docs/transactional-semantics.md rename to docs/research/transactional-semantics.md diff --git a/docs/specification/configuration.md b/docs/tendermint-core/configuration.md similarity index 96% rename from docs/specification/configuration.md rename to docs/tendermint-core/configuration.md index e298dfd8f..0453bdad7 100644 --- a/docs/specification/configuration.md +++ b/docs/tendermint-core/configuration.md @@ -209,4 +209,10 @@ prometheus = false # Address to listen for Prometheus collector(s) connections prometheus_listen_addr = ":26660" + +# Maximum number of simultaneous connections. +# If you want to accept a more significant number than the default, make sure +# you increase your OS limits. +# 0 - unlimited. +max_open_connections = 3 ``` diff --git a/docs/how-to-read-logs.md b/docs/tendermint-core/how-to-read-logs.md similarity index 100% rename from docs/how-to-read-logs.md rename to docs/tendermint-core/how-to-read-logs.md diff --git a/docs/metrics.md b/docs/tendermint-core/metrics.md similarity index 100% rename from docs/metrics.md rename to docs/tendermint-core/metrics.md diff --git a/docs/specification/rpc.md b/docs/tendermint-core/rpc.md similarity index 100% rename from docs/specification/rpc.md rename to docs/tendermint-core/rpc.md diff --git a/docs/running-in-production.md b/docs/tendermint-core/running-in-production.md similarity index 97% rename from docs/running-in-production.md rename to docs/tendermint-core/running-in-production.md index 1f87f9114..181d09428 100644 --- a/docs/running-in-production.md +++ b/docs/tendermint-core/running-in-production.md @@ -16,7 +16,7 @@ logging level, you can do so by running tendermint with Validators are supposed to setup [Sentry Node Architecture](https://blog.cosmos.network/tendermint-explained-bringing-bft-based-pos-to-the-public-blockchain-domain-f22e274a0fdb) to prevent Denial-of-service attacks. You can read more about it -[here](https://github.com/tendermint/aib-data/blob/develop/medium/TendermintBFT.md). +[here](../interviews/tendermint-bft.md). ### P2P @@ -49,7 +49,7 @@ second TODO is to query the /status RPC endpoint. It provides the necessary info: whenever the node is syncing or not, what height it is on, etc. -``` +``` curl http(s)://{ip}:{rpcPort}/status ``` @@ -198,11 +198,6 @@ theres something to send its peers. You can also try lowering `timeout_commit` (time we sleep before proposing the next block). -- `consensus.max_block_size_txs` - -By default, the maximum number of transactions per a block is 10_000. -Feel free to change it to suit your needs. - - `p2p.addr_book_strict` By default, Tendermint checks whenever a peer's address is routable before diff --git a/docs/using-tendermint.md b/docs/tendermint-core/using-tendermint.md similarity index 100% rename from docs/using-tendermint.md rename to docs/tendermint-core/using-tendermint.md diff --git a/docs/tools/benchmarking.md b/docs/tools/benchmarking.md new file mode 100644 index 000000000..20c368e29 --- /dev/null +++ b/docs/tools/benchmarking.md @@ -0,0 +1,80 @@ +# tm-bench + +Tendermint blockchain benchmarking tool: + +- https://github.com/tendermint/tools/tree/master/tm-bench + +For example, the following: + +``` +tm-bench -T 10 -r 1000 localhost:26657 +``` + +will output: + +``` +Stats Avg StdDev Max Total +Txs/sec 818 532 1549 9000 +Blocks/sec 0.818 0.386 1 9 +``` + +## Quick Start + +[Install Tendermint](../introduction/install) +This currently is setup to work on tendermint's develop branch. Please ensure +you are on that. (If not, update `tendermint` and `tmlibs` in gopkg.toml to use + the master branch.) + +then run: + +``` +tendermint init +tendermint node --proxy_app=kvstore +``` + +``` +tm-bench localhost:26657 +``` + +with the last command being in a seperate window. + +## Usage + +``` +tm-bench [-c 1] [-T 10] [-r 1000] [-s 250] [endpoints] + +Examples: + tm-bench localhost:26657 +Flags: + -T int + Exit after the specified amount of time in seconds (default 10) + -c int + Connections to keep open per endpoint (default 1) + -r int + Txs per second to send in a connection (default 1000) + -s int + Size per tx in bytes + -v Verbose output +``` + +## How stats are collected + +These stats are derived by having each connection send transactions at the +specified rate (or as close as it can get) for the specified time. After the +specified time, it iterates over all of the blocks that were created in that +time. The average and stddev per second are computed based off of that, by +grouping the data by second. + +To send transactions at the specified rate in each connection, we loop +through the number of transactions. If its too slow, the loop stops at one second. +If its too fast, we wait until the one second mark ends. The transactions per +second stat is computed based off of what ends up in the block. + +Each of the connections is handled via two separate goroutines. + +## Development + +``` +make get_vendor_deps +make test +``` diff --git a/docs/tools/monitoring.md b/docs/tools/monitoring.md new file mode 100644 index 000000000..5cc2ad3b1 --- /dev/null +++ b/docs/tools/monitoring.md @@ -0,0 +1,92 @@ +# tm-monitor + +Tendermint blockchain monitoring tool; watches over one or more nodes, +collecting and providing various statistics to the user: + +- https://github.com/tendermint/tools/tree/master/tm-monitor + +## Quick Start + +### Docker + +Assuming your application is running in another container with the name +`app`: + +``` +docker run -it --rm -v "/tmp:/tendermint" tendermint/tendermint init +docker run -it --rm -v "/tmp:/tendermint" -p "26657:26657" --name=tm --link=app tendermint/tendermint node --proxy_app=tcp://app:26658 + +docker run -it --rm -p "26670:26670" --link=tm tendermint/monitor tm:26657 +``` + +If you don't have an application yet, but still want to try monitor out, +use `kvstore`: + +``` +docker run -it --rm -v "/tmp:/tendermint" tendermint/tendermint init +docker run -it --rm -v "/tmp:/tendermint" -p "26657:26657" --name=tm tendermint/tendermint node --proxy_app=kvstore +``` +``` +docker run -it --rm -p "26670:26670" --link=tm tendermint/monitor tm:26657 +``` + +### Using Binaries + +[Install Tendermint](https://github.com/tendermint/tendermint#install) + +then run: + +``` +tendermint init +tendermint node --proxy_app=kvstore +``` + +``` +tm-monitor localhost:26657 +``` + +with the last command being in a seperate window. + +## Usage + +``` +tm-monitor [-v] [-no-ton] [-listen-addr="tcp://0.0.0.0:26670"] [endpoints] + +Examples: + # monitor single instance + tm-monitor localhost:26657 + + # monitor a few instances by providing comma-separated list of RPC endpoints + tm-monitor host1:26657,host2:26657 +Flags: + -listen-addr string + HTTP and Websocket server listen address (default "tcp://0.0.0.0:26670") + -no-ton + Do not show ton (table of nodes) + -v verbose logging +``` + +### RPC UI + +Run `tm-monitor` and visit http://localhost:26670 You should see the +list of the available RPC endpoints: + +``` +http://localhost:26670/status +http://localhost:26670/status/network +http://localhost:26670/monitor?endpoint=_ +http://localhost:26670/status/node?name=_ +http://localhost:26670/unmonitor?endpoint=_ +``` + +The API is available as GET requests with URI encoded parameters, or as +JSONRPC POST requests. The JSONRPC methods are also exposed over +websocket. + +## Development + +``` +make get_tools +make get_vendor_deps +make test +``` diff --git a/evidence/reactor.go b/evidence/reactor.go index 7b22b8dba..bf11ac105 100644 --- a/evidence/reactor.go +++ b/evidence/reactor.go @@ -5,10 +5,10 @@ import ( "reflect" "time" - "github.com/tendermint/go-amino" + amino "github.com/tendermint/go-amino" + clist "github.com/tendermint/tendermint/libs/clist" "github.com/tendermint/tendermint/libs/log" - "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/types" ) @@ -73,7 +73,7 @@ func (evR *EvidenceReactor) RemovePeer(peer p2p.Peer, reason interface{}) { // Receive implements Reactor. // It adds any received evidence to the evpool. func (evR *EvidenceReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) { - msg, err := DecodeMessage(msgBytes) + msg, err := decodeMsg(msgBytes) if err != nil { evR.Logger.Error("Error decoding message", "src", src, "chId", chID, "msg", msg, "err", err, "bytes", msgBytes) evR.Switch.StopPeerForError(src, err) @@ -204,11 +204,9 @@ func RegisterEvidenceMessages(cdc *amino.Codec) { "tendermint/evidence/EvidenceListMessage", nil) } -// DecodeMessage decodes a byte-array into a EvidenceMessage. -func DecodeMessage(bz []byte) (msg EvidenceMessage, err error) { +func decodeMsg(bz []byte) (msg EvidenceMessage, err error) { if len(bz) > maxMsgSize { - return msg, fmt.Errorf("Msg exceeds max size (%d > %d)", - len(bz), maxMsgSize) + return msg, fmt.Errorf("Msg exceeds max size (%d > %d)", len(bz), maxMsgSize) } err = cdc.UnmarshalBinaryBare(bz, &msg) return diff --git a/libs/Makefile b/libs/Makefile deleted file mode 100644 index a55bc139c..000000000 --- a/libs/Makefile +++ /dev/null @@ -1,137 +0,0 @@ -GOTOOLS = \ - github.com/golang/dep/cmd/dep \ - github.com/golang/protobuf/protoc-gen-go \ - github.com/square/certstrap - # github.com/alecthomas/gometalinter.v2 \ - -GOTOOLS_CHECK = dep gometalinter.v2 protoc protoc-gen-go -INCLUDE = -I=. -I=${GOPATH}/src - -all: check get_vendor_deps protoc grpc_dbserver build test install metalinter - -check: check_tools - -######################################## -### Build - -protoc: - ## If you get the following error, - ## "error while loading shared libraries: libprotobuf.so.14: cannot open shared object file: No such file or directory" - ## See https://stackoverflow.com/a/25518702 - protoc $(INCLUDE) --go_out=plugins=grpc:. common/*.proto - @echo "--> adding nolint declarations to protobuf generated files" - @awk '/package common/ { print "//nolint: gas"; print; next }1' common/types.pb.go > common/types.pb.go.new - @mv common/types.pb.go.new common/types.pb.go - -build: - # Nothing to build! - -install: - # Nothing to install! - - -######################################## -### Tools & dependencies - -check_tools: - @# https://stackoverflow.com/a/25668869 - @echo "Found tools: $(foreach tool,$(GOTOOLS_CHECK),\ - $(if $(shell which $(tool)),$(tool),$(error "No $(tool) in PATH")))" - -get_tools: - @echo "--> Installing tools" - go get -u -v $(GOTOOLS) - # @gometalinter.v2 --install - -get_protoc: - @# https://github.com/google/protobuf/releases - curl -L https://github.com/google/protobuf/releases/download/v3.4.1/protobuf-cpp-3.4.1.tar.gz | tar xvz && \ - cd protobuf-3.4.1 && \ - DIST_LANG=cpp ./configure && \ - make && \ - make install && \ - cd .. && \ - rm -rf protobuf-3.4.1 - -update_tools: - @echo "--> Updating tools" - @go get -u $(GOTOOLS) - -get_vendor_deps: - @rm -rf vendor/ - @echo "--> Running dep ensure" - @dep ensure - - -######################################## -### Testing - -gen_certs: clean_certs - ## Generating certificates for TLS testing... - certstrap init --common-name "tendermint.com" --passphrase "" - certstrap request-cert -ip "::" --passphrase "" - certstrap sign "::" --CA "tendermint.com" --passphrase "" - mv out/::.crt out/::.key db/remotedb - -clean_certs: - ## Cleaning TLS testing certificates... - rm -rf out - rm -f db/remotedb/::.crt db/remotedb/::.key - -test: gen_certs - GOCACHE=off go test -tags gcc $(shell go list ./... | grep -v vendor) - make clean_certs - -test100: - @for i in {1..100}; do make test; done - - -######################################## -### Formatting, linting, and vetting - -fmt: - @go fmt ./... - -metalinter: - @echo "==> Running linter" - gometalinter.v2 --vendor --deadline=600s --disable-all \ - --enable=deadcode \ - --enable=goconst \ - --enable=goimports \ - --enable=gosimple \ - --enable=ineffassign \ - --enable=megacheck \ - --enable=misspell \ - --enable=staticcheck \ - --enable=safesql \ - --enable=structcheck \ - --enable=unconvert \ - --enable=unused \ - --enable=varcheck \ - --enable=vetshadow \ - ./... - - #--enable=maligned \ - #--enable=gas \ - #--enable=aligncheck \ - #--enable=dupl \ - #--enable=errcheck \ - #--enable=gocyclo \ - #--enable=golint \ <== comments on anything exported - #--enable=gotype \ - #--enable=interfacer \ - #--enable=unparam \ - #--enable=vet \ - -metalinter_all: - protoc $(INCLUDE) --lint_out=. types/*.proto - gometalinter.v2 --vendor --deadline=600s --enable-all --disable=lll ./... - - -# To avoid unintended conflicts with file names, always add to .PHONY -# unless there is a reason not to. -# https://www.gnu.org/software/make/manual/html_node/Phony-Targets.html -.PHONY: check protoc build check_tools get_tools get_protoc update_tools get_vendor_deps test fmt metalinter metalinter_all gen_certs clean_certs - -grpc_dbserver: - protoc -I db/remotedb/proto/ db/remotedb/proto/defs.proto --go_out=plugins=grpc:db/remotedb/proto diff --git a/mempool/reactor.go b/mempool/reactor.go index e63ff58e8..96988be78 100644 --- a/mempool/reactor.go +++ b/mempool/reactor.go @@ -78,7 +78,7 @@ func (memR *MempoolReactor) RemovePeer(peer p2p.Peer, reason interface{}) { // Receive implements Reactor. // It adds any received transactions to the mempool. func (memR *MempoolReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) { - msg, err := DecodeMessage(msgBytes) + msg, err := decodeMsg(msgBytes) if err != nil { memR.Logger.Error("Error decoding message", "src", src, "chId", chID, "msg", msg, "err", err, "bytes", msgBytes) memR.Switch.StopPeerForError(src, err) @@ -174,11 +174,9 @@ func RegisterMempoolMessages(cdc *amino.Codec) { cdc.RegisterConcrete(&TxMessage{}, "tendermint/mempool/TxMessage", nil) } -// DecodeMessage decodes a byte-array into a MempoolMessage. -func DecodeMessage(bz []byte) (msg MempoolMessage, err error) { +func decodeMsg(bz []byte) (msg MempoolMessage, err error) { if len(bz) > maxMsgSize { - return msg, fmt.Errorf("Msg exceeds max size (%d > %d)", - len(bz), maxMsgSize) + return msg, fmt.Errorf("Msg exceeds max size (%d > %d)", len(bz), maxMsgSize) } err = cdc.UnmarshalBinaryBare(bz, &msg) return diff --git a/node/node.go b/node/node.go index 0780891ef..9f6428ec1 100644 --- a/node/node.go +++ b/node/node.go @@ -8,6 +8,7 @@ import ( "net" "net/http" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" amino "github.com/tendermint/go-amino" @@ -599,8 +600,13 @@ func (n *Node) startRPC() ([]net.Listener, error) { // collectors on addr. func (n *Node) startPrometheusServer(addr string) *http.Server { srv := &http.Server{ - Addr: addr, - Handler: promhttp.Handler(), + Addr: addr, + Handler: promhttp.InstrumentMetricHandler( + prometheus.DefaultRegisterer, promhttp.HandlerFor( + prometheus.DefaultGatherer, + promhttp.HandlerOpts{MaxRequestsInFlight: n.config.Instrumentation.MaxOpenConnections}, + ), + ), } go func() { if err := srv.ListenAndServe(); err != http.ErrServerClosed { diff --git a/p2p/pex/pex_reactor.go b/p2p/pex/pex_reactor.go index 2d93783df..e90665a37 100644 --- a/p2p/pex/pex_reactor.go +++ b/p2p/pex/pex_reactor.go @@ -206,7 +206,7 @@ func (r *PEXReactor) RemovePeer(p Peer, reason interface{}) { // Receive implements Reactor by handling incoming PEX messages. func (r *PEXReactor) Receive(chID byte, src Peer, msgBytes []byte) { - msg, err := DecodeMessage(msgBytes) + msg, err := decodeMsg(msgBytes) if err != nil { r.Logger.Error("Error decoding message", "src", src, "chId", chID, "msg", msg, "err", err, "bytes", msgBytes) r.Switch.StopPeerForError(src, err) @@ -287,7 +287,7 @@ func (r *PEXReactor) RequestAddrs(p Peer) { return } r.requestsSent.Set(id, struct{}{}) - p.Send(PexChannel, cdc.MustMarshalBinary(&pexRequestMessage{})) + p.Send(PexChannel, cdc.MustMarshalBinaryBare(&pexRequestMessage{})) } // ReceiveAddrs adds the given addrs to the addrbook if theres an open @@ -324,7 +324,7 @@ func (r *PEXReactor) ReceiveAddrs(addrs []*p2p.NetAddress, src Peer) error { // SendAddrs sends addrs to the peer. func (r *PEXReactor) SendAddrs(p Peer, netAddrs []*p2p.NetAddress) { - p.Send(PexChannel, cdc.MustMarshalBinary(&pexAddrsMessage{Addrs: netAddrs})) + p.Send(PexChannel, cdc.MustMarshalBinaryBare(&pexAddrsMessage{Addrs: netAddrs})) } // SetEnsurePeersPeriod sets period to ensure peers connected. @@ -670,13 +670,11 @@ func RegisterPexMessage(cdc *amino.Codec) { cdc.RegisterConcrete(&pexAddrsMessage{}, "tendermint/p2p/PexAddrsMessage", nil) } -// DecodeMessage implements interface registered above. -func DecodeMessage(bz []byte) (msg PexMessage, err error) { +func decodeMsg(bz []byte) (msg PexMessage, err error) { if len(bz) > maxMsgSize { - return msg, fmt.Errorf("Msg exceeds max size (%d > %d)", - len(bz), maxMsgSize) + return msg, fmt.Errorf("Msg exceeds max size (%d > %d)", len(bz), maxMsgSize) } - err = cdc.UnmarshalBinary(bz, &msg) + err = cdc.UnmarshalBinaryBare(bz, &msg) return } diff --git a/p2p/pex/pex_reactor_test.go b/p2p/pex/pex_reactor_test.go index 6d6e91c38..629c9397a 100644 --- a/p2p/pex/pex_reactor_test.go +++ b/p2p/pex/pex_reactor_test.go @@ -134,11 +134,11 @@ func TestPEXReactorReceive(t *testing.T) { size := book.Size() addrs := []*p2p.NetAddress{peer.NodeInfo().NetAddress()} - msg := cdc.MustMarshalBinary(&pexAddrsMessage{Addrs: addrs}) + msg := cdc.MustMarshalBinaryBare(&pexAddrsMessage{Addrs: addrs}) r.Receive(PexChannel, peer, msg) assert.Equal(t, size+1, book.Size()) - msg = cdc.MustMarshalBinary(&pexRequestMessage{}) + msg = cdc.MustMarshalBinaryBare(&pexRequestMessage{}) r.Receive(PexChannel, peer, msg) // should not panic. } @@ -154,7 +154,7 @@ func TestPEXReactorRequestMessageAbuse(t *testing.T) { assert.True(t, sw.Peers().Has(peer.ID())) id := string(peer.ID()) - msg := cdc.MustMarshalBinary(&pexRequestMessage{}) + msg := cdc.MustMarshalBinaryBare(&pexRequestMessage{}) // first time creates the entry r.Receive(PexChannel, peer, msg) @@ -191,7 +191,7 @@ func TestPEXReactorAddrsMessageAbuse(t *testing.T) { assert.True(t, sw.Peers().Has(peer.ID())) addrs := []*p2p.NetAddress{peer.NodeInfo().NetAddress()} - msg := cdc.MustMarshalBinary(&pexAddrsMessage{Addrs: addrs}) + msg := cdc.MustMarshalBinaryBare(&pexAddrsMessage{Addrs: addrs}) // receive some addrs. should clear the request r.Receive(PexChannel, peer, msg) @@ -303,7 +303,7 @@ func TestPEXReactorDoesNotAddPrivatePeersToAddrBook(t *testing.T) { size := book.Size() addrs := []*p2p.NetAddress{peer.NodeInfo().NetAddress()} - msg := cdc.MustMarshalBinary(&pexAddrsMessage{Addrs: addrs}) + msg := cdc.MustMarshalBinaryBare(&pexAddrsMessage{Addrs: addrs}) pexR.Receive(PexChannel, peer, msg) assert.Equal(t, size, book.Size()) diff --git a/rpc/lib/server/handlers.go b/rpc/lib/server/handlers.go index 1bfe52536..3ec5f81e3 100644 --- a/rpc/lib/server/handlers.go +++ b/rpc/lib/server/handlers.go @@ -18,9 +18,9 @@ import ( "github.com/pkg/errors" amino "github.com/tendermint/go-amino" - types "github.com/tendermint/tendermint/rpc/lib/types" cmn "github.com/tendermint/tendermint/libs/common" "github.com/tendermint/tendermint/libs/log" + types "github.com/tendermint/tendermint/rpc/lib/types" ) // RegisterRPCFuncs adds a route for each function in the funcMap, as well as general jsonrpc and websocket handlers for all functions. @@ -294,7 +294,7 @@ func httpParamsToArgs(rpcFunc *RPCFunc, cdc *amino.Codec, r *http.Request) ([]re continue } - v, err, ok := nonJSONToArg(cdc, argType, arg) + v, err, ok := nonJSONStringToArg(cdc, argType, arg) if err != nil { return nil, err } @@ -303,7 +303,7 @@ func httpParamsToArgs(rpcFunc *RPCFunc, cdc *amino.Codec, r *http.Request) ([]re continue } - values[i], err = _jsonStringToArg(cdc, argType, arg) + values[i], err = jsonStringToArg(cdc, argType, arg) if err != nil { return nil, err } @@ -312,26 +312,64 @@ func httpParamsToArgs(rpcFunc *RPCFunc, cdc *amino.Codec, r *http.Request) ([]re return values, nil } -func _jsonStringToArg(cdc *amino.Codec, ty reflect.Type, arg string) (reflect.Value, error) { - v := reflect.New(ty) - err := cdc.UnmarshalJSON([]byte(arg), v.Interface()) +func jsonStringToArg(cdc *amino.Codec, rt reflect.Type, arg string) (reflect.Value, error) { + rv := reflect.New(rt) + err := cdc.UnmarshalJSON([]byte(arg), rv.Interface()) if err != nil { - return v, err + return rv, err } - v = v.Elem() - return v, nil + rv = rv.Elem() + return rv, nil } -func nonJSONToArg(cdc *amino.Codec, ty reflect.Type, arg string) (reflect.Value, error, bool) { +func nonJSONStringToArg(cdc *amino.Codec, rt reflect.Type, arg string) (reflect.Value, error, bool) { + if rt.Kind() == reflect.Ptr { + rv_, err, ok := nonJSONStringToArg(cdc, rt.Elem(), arg) + if err != nil { + return reflect.Value{}, err, false + } else if ok { + rv := reflect.New(rt.Elem()) + rv.Elem().Set(rv_) + return rv, nil, true + } else { + return reflect.Value{}, nil, false + } + } else { + return _nonJSONStringToArg(cdc, rt, arg) + } +} + +// NOTE: rt.Kind() isn't a pointer. +func _nonJSONStringToArg(cdc *amino.Codec, rt reflect.Type, arg string) (reflect.Value, error, bool) { + isIntString := RE_INT.Match([]byte(arg)) isQuotedString := strings.HasPrefix(arg, `"`) && strings.HasSuffix(arg, `"`) isHexString := strings.HasPrefix(strings.ToLower(arg), "0x") - expectingString := ty.Kind() == reflect.String - expectingByteSlice := ty.Kind() == reflect.Slice && ty.Elem().Kind() == reflect.Uint8 + + var expectingString, expectingByteSlice, expectingInt bool + switch rt.Kind() { + case reflect.Int, reflect.Uint, reflect.Int8, reflect.Uint8, reflect.Int16, reflect.Uint16, reflect.Int32, reflect.Uint32, reflect.Int64, reflect.Uint64: + expectingInt = true + case reflect.String: + expectingString = true + case reflect.Slice: + expectingByteSlice = rt.Elem().Kind() == reflect.Uint8 + } + + if isIntString && expectingInt { + qarg := `"` + arg + `"` + // jsonStringToArg + rv, err := jsonStringToArg(cdc, rt, qarg) + if err != nil { + return rv, err, false + } else { + return rv, nil, true + } + } if isHexString { if !expectingString && !expectingByteSlice { err := errors.Errorf("Got a hex string arg, but expected '%s'", - ty.Kind().String()) + rt.Kind().String()) return reflect.ValueOf(nil), err, false } @@ -340,7 +378,7 @@ func nonJSONToArg(cdc *amino.Codec, ty reflect.Type, arg string) (reflect.Value, if err != nil { return reflect.ValueOf(nil), err, false } - if ty.Kind() == reflect.String { + if rt.Kind() == reflect.String { return reflect.ValueOf(string(value)), nil, true } return reflect.ValueOf([]byte(value)), nil, true diff --git a/rpc/lib/server/http_params.go b/rpc/lib/server/http_params.go index 565060678..3c948c0ba 100644 --- a/rpc/lib/server/http_params.go +++ b/rpc/lib/server/http_params.go @@ -15,6 +15,7 @@ var ( dotAtom = atom + `(?:\.` + atom + `)*` domain = `[A-Z0-9.-]+\.[A-Z]{2,4}` + RE_INT = regexp.MustCompile(`^-?[0-9]+$`) RE_HEX = regexp.MustCompile(`^(?i)[a-f0-9]+$`) RE_EMAIL = regexp.MustCompile(`^(?i)(` + dotAtom + `)@(` + dotAtom + `)$`) RE_ADDRESS = regexp.MustCompile(`^(?i)[a-z0-9]{25,34}$`) diff --git a/rpc/lib/server/parse_test.go b/rpc/lib/server/parse_test.go index f71316906..7b0aacdbe 100644 --- a/rpc/lib/server/parse_test.go +++ b/rpc/lib/server/parse_test.go @@ -2,6 +2,8 @@ package rpcserver import ( "encoding/json" + "fmt" + "net/http" "strconv" "testing" @@ -134,7 +136,7 @@ func TestParseJSONArray(t *testing.T) { } } -func TestParseRPC(t *testing.T) { +func TestParseJSONRPC(t *testing.T) { assert := assert.New(t) demo := func(height int, name string) {} @@ -172,5 +174,48 @@ func TestParseRPC(t *testing.T) { } } - +} + +func TestParseURI(t *testing.T) { + + demo := func(height int, name string) {} + call := NewRPCFunc(demo, "height,name") + cdc := amino.NewCodec() + + cases := []struct { + raw []string + height int64 + name string + fail bool + }{ + // can parse numbers unquoted and strings quoted + {[]string{"7", `"flew"`}, 7, "flew", false}, + {[]string{"22", `"john"`}, 22, "john", false}, + {[]string{"-10", `"bob"`}, -10, "bob", false}, + // can parse numbers quoted, too + {[]string{`"7"`, `"flew"`}, 7, "flew", false}, + {[]string{`"-10"`, `"bob"`}, -10, "bob", false}, + // cant parse strings uquoted + {[]string{`"-10"`, `bob`}, -10, "bob", true}, + } + for idx, tc := range cases { + i := strconv.Itoa(idx) + // data := []byte(tc.raw) + url := fmt.Sprintf( + "test.com/method?height=%v&name=%v", + tc.raw[0], tc.raw[1]) + req, err := http.NewRequest("GET", url, nil) + assert.NoError(t, err) + vals, err := httpParamsToArgs(call, cdc, req) + if tc.fail { + assert.NotNil(t, err, i) + } else { + assert.Nil(t, err, "%s: %+v", i, err) + if assert.Equal(t, 2, len(vals), i) { + assert.Equal(t, tc.height, vals[0].Int(), i) + assert.Equal(t, tc.name, vals[1].String(), i) + } + } + + } } diff --git a/scripts/dep_utils/parse.sh b/scripts/dep_utils/parse.sh deleted file mode 100644 index e6519efa0..000000000 --- a/scripts/dep_utils/parse.sh +++ /dev/null @@ -1,14 +0,0 @@ -#! /bin/bash - -set +u -if [[ "$DEP" == "" ]]; then - DEP=$GOPATH/src/github.com/tendermint/tendermint/Gopkg.lock -fi -set -u - - -set -euo pipefail - -LIB=$1 - -grep -A100 "$LIB" "$DEP" | grep revision | head -n1 | grep -o '"[^"]\+"' | cut -d '"' -f 2 diff --git a/scripts/install_abci_apps.sh b/scripts/install_abci_apps.sh deleted file mode 100644 index eb70070df..000000000 --- a/scripts/install_abci_apps.sh +++ /dev/null @@ -1,12 +0,0 @@ -#! /bin/bash - -# get the abci commit used by tendermint -COMMIT=$(bash scripts/dep_utils/parse.sh abci) -echo "Checking out vendored commit for abci: $COMMIT" - -go get -d github.com/tendermint/abci -cd "$GOPATH/src/github.com/tendermint/abci" || exit -git checkout "$COMMIT" -make get_tools -make get_vendor_deps -make install diff --git a/test/docker/Dockerfile b/test/docker/Dockerfile index bc211ea43..8b69d27fe 100644 --- a/test/docker/Dockerfile +++ b/test/docker/Dockerfile @@ -21,15 +21,12 @@ ADD Makefile Makefile RUN make get_tools RUN make get_vendor_deps -# Install the apps -ADD scripts scripts -RUN bash scripts/install_abci_apps.sh - # Now copy in the code # NOTE: this will overwrite whatever is in vendor/ COPY . $REPO RUN go install ./cmd/tendermint +RUN go install ./abci/cmd/abci-cli # expose the volume for debugging VOLUME $REPO diff --git a/version/version.go b/version/version.go index 68f1b4837..6ba0dcb42 100644 --- a/version/version.go +++ b/version/version.go @@ -4,13 +4,13 @@ package version const ( Maj = "0" Min = "22" - Fix = "1" + Fix = "2" ) var ( // Version is the current version of Tendermint // Must be a string because scripts like dist.sh read this file. - Version = "0.22.1" + Version = "0.22.2" // GitCommit is the current HEAD set using ldflags. GitCommit string