diff --git a/.circleci/config.yml b/.circleci/config.yml index 112e1c1bb..5a50a61c5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -32,6 +32,7 @@ jobs: command: | export PATH="$GOBIN:$PATH" make install + cd abci && make install - persist_to_workspace: root: /tmp/workspace paths: @@ -46,37 +47,6 @@ jobs: paths: - /go/src/github.com/tendermint/tendermint - setup_abci: - <<: *defaults - steps: - - attach_workspace: - at: /tmp/workspace - - restore_cache: - key: v2-pkg-cache - - restore_cache: - key: v2-tree-{{ .Environment.CIRCLE_SHA1 }} - - run: - name: Checkout abci - command: | - commit=$(bash scripts/dep_utils/parse.sh abci) - go get -v -u -d github.com/tendermint/abci/... - cd /go/src/github.com/tendermint/abci - git checkout "$commit" - - run: - working_directory: /go/src/github.com/tendermint/abci - name: Install abci - command: | - set -ex - export PATH="$GOBIN:$PATH" - make get_tools - make get_vendor_deps - make install - - run: ls -lah /tmp/workspace/bin - - persist_to_workspace: - root: /tmp/workspace - paths: - - "bin/abci*" - build_slate: <<: *defaults steps: @@ -109,6 +79,38 @@ jobs: export PATH="$GOBIN:$PATH" make metalinter + test_abci_apps: + <<: *defaults + steps: + - attach_workspace: + at: /tmp/workspace + - restore_cache: + key: v2-pkg-cache + - restore_cache: + key: v2-tree-{{ .Environment.CIRCLE_SHA1 }} + - run: + name: Run abci apps tests + command: | + export PATH="$GOBIN:$PATH" + bash abci/tests/test_app/test.sh + +# if this test fails, fix it and update the docs at: +# https://github.com/tendermint/tendermint/blob/develop/docs/abci-cli.md + test_abci_cli: + <<: *defaults + steps: + - attach_workspace: + at: /tmp/workspace + - restore_cache: + key: v2-pkg-cache + - restore_cache: + key: v2-tree-{{ .Environment.CIRCLE_SHA1 }} + - run: + name: Run abci-cli tests + command: | + export PATH="$GOBIN:$PATH" + bash abci/tests/test_cli/test.sh + test_apps: <<: *defaults steps: @@ -199,21 +201,24 @@ workflows: test-suite: jobs: - setup_dependencies - - setup_abci: + - lint: requires: - setup_dependencies - - lint: + - test_abci_apps: + requires: + - setup_dependencies + - test_abci_cli: requires: - setup_dependencies - test_apps: requires: - - setup_abci + - setup_dependencies - test_cover: requires: - setup_dependencies - test_persistence: requires: - - setup_abci + - setup_dependencies - test_p2p - upload_coverage: requires: diff --git a/.github/ISSUE_TEMPLATE b/.github/ISSUE_TEMPLATE index e0edb29ff..ed3caac5f 100644 --- a/.github/ISSUE_TEMPLATE +++ b/.github/ISSUE_TEMPLATE @@ -33,7 +33,7 @@ in a case of bug. **How to reproduce it** (as minimally and precisely as possible): -**Logs (you can paste a part showing an error or attach the whole file)**: +**Logs (you can paste a small part showing an error or link a pastebin, gist, etc. containing more of the log file)**: **Config (you can paste only the changes you've made)**: diff --git a/.gitignore b/.gitignore index 6d0587669..bcfd36db1 100644 --- a/.gitignore +++ b/.gitignore @@ -15,8 +15,9 @@ test/logs coverage.txt docs/_build docs/tools -docs/abci-spec.md *.log +abci-cli +abci/types/types.pb.go scripts/wal2json/wal2json scripts/cutWALUntil/cutWALUntil @@ -25,3 +26,4 @@ scripts/cutWALUntil/cutWALUntil *.iml libs/pubsub/query/fuzz_test/output +shunit2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 65815ccf3..9c9100157 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,50 @@ # Changelog +## 0.22.0 + +*July 2nd, 2018* + +BREAKING CHANGES: +- [config] Rename `skip_upnp` to `upnp`, and turn it off by default. +- [types] Update Amino to v0.10.1 + * Amino is now fully proto3 compatible for the basic types + * JSON-encoded types now use the type name instead of the prefix bytes + * Integers are encoded as strings +- [crypto] Update go-crypto to v0.10.0 and merge into `crypto` + * privKey.Sign returns error. + * ed25519 address changed to the first 20-bytes of the SHA256 of the raw pubkey bytes + * `tmlibs/merkle` -> `crypto/merkle`. Uses SHA256 instead of RIPEMD160 +- [tmlibs] Update to v0.9.0 and merge into `libs` + * remove `merkle` package (moved to `crypto/merkle`) +- [rpc] `syncing` is now called `catching_up`. + +FEATURES +- [cmd] Added metrics (served under `/metrics` using a Prometheus client; + disabled by default). See the new `instrumentation` section in the config and + [metrics](https://tendermint.readthedocs.io/projects/tools/en/develop/metrics.html) + guide. +- [p2p] Add IPv6 support to peering. +- [p2p] Add `external_address` to config to allow specifying the address for + peers to dial + +IMPROVEMENT +- [rpc/client] Supports https and wss now. +- [crypto] Make public key size into public constants +- [mempool] Log tx hash, not entire tx +- [abci] Merged in github.com/tendermint/abci +- [crypto] Merged in github.com/tendermint/go-crypto +- [libs] Merged in github.com/tendermint/tmlibs +- [docs] Move from .rst to .md + +BUG FIXES: +- [rpc] Limit maximum number of HTTP/WebSocket connections + (`rpc.max_open_connections`) and gRPC connections + (`rpc.grpc_max_open_connections`). Check out "Running In Production" guide if + you want to increase them. +- [rpc] Limit maximum request body size to 1MB (header is limited to 1MB). +- [consensus] Fix a halting bug where `create_empty_blocks=false` +- [p2p] Fix panic in seed mode + ## 0.21.0 *June 21th, 2018* diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5fd2d982f..3500732f5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -17,7 +17,7 @@ Instead, we use `git remote` to add the fork as a new remote for the original re For instance, to create a fork and work on a branch of it, I would: * Create the fork on github, using the fork button. - * Go to the original repo checked out locally (ie. `$GOPATH/src/github.com/tendermint/tendermint`) + * Go to the original repo checked out locally (i.e. `$GOPATH/src/github.com/tendermint/tendermint`) * `git remote rename origin upstream` * `git remote add origin git@github.com:ebuchman/basecoin.git` @@ -47,7 +47,7 @@ get_vendor_deps`). Even for dependencies under our control, dep helps us to keep multiple repos in sync as they evolve. Anything with an executable, such as apps, tools, and the core, should use dep. -Run `dep status` to get a list of vendored dependencies that may not be +Run `dep status` to get a list of vendor dependencies that may not be up-to-date. ## Vagrant @@ -85,7 +85,7 @@ especially `go-p2p` and `go-rpc`, as their versions are referenced in tendermint - the latest state of development is on `develop` - `develop` must never fail `make test` - no --force onto `develop` (except when reverting a broken commit, which should seldom happen) -- create a development branch either on github.com/tendermint/tendermint, or your fork (using `git add origin`) +- create a development branch either on github.com/tendermint/tendermint, or your fork (using `git remote add origin`) - before submitting a pull request, begin `git rebase` on top of `develop` ### Pull Merge Procedure: @@ -110,7 +110,7 @@ especially `go-p2p` and `go-rpc`, as their versions are referenced in tendermint - make the required changes - these changes should be small and an absolute necessity - add a note to CHANGELOG.md -- bumb versions +- bump versions - push to hotfix-vX.X.X to run the extended integration tests on the CI - merge hotfix-vX.X.X to master - merge hotfix-vX.X.X to develop diff --git a/Gopkg.lock b/Gopkg.lock index 8019acb52..17b74d74b 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1,12 +1,27 @@ # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. +[[projects]] + branch = "master" + name = "github.com/beorn7/perks" + packages = ["quantile"] + revision = "3a771d992973f24aa725d07868b467d1ddfceafb" + [[projects]] branch = "master" name = "github.com/btcsuite/btcd" packages = ["btcec"] revision = "86fed781132ac890ee03e906e4ecd5d6fa180c64" +[[projects]] + branch = "master" + name = "github.com/btcsuite/btcutil" + packages = [ + "base58", + "bech32" + ] + revision = "d4cc87b860166d00d6b5b9e0d3b3d71d6088d4d4" + [[projects]] name = "github.com/davecgh/go-spew" packages = ["spew"] @@ -20,10 +35,10 @@ revision = "95f809107225be108efcf10a3509e4ea6ceef3c4" [[projects]] + branch = "master" name = "github.com/fortytw2/leaktest" packages = ["."] - revision = "a5ef70473c97b71626b9abeda80ee92ba2a7de9e" - version = "v1.2.0" + revision = "b008db64ef8daabb22ff6daa557f33b41d8f6ccd" [[projects]] name = "github.com/fsnotify/fsnotify" @@ -36,7 +51,11 @@ packages = [ "log", "log/level", - "log/term" + "log/term", + "metrics", + "metrics/discard", + "metrics/internal/lv", + "metrics/prometheus" ] revision = "4dc7be5d2d12881735283bcab7352178e190fc71" version = "v0.6.0" @@ -97,7 +116,6 @@ ".", "hcl/ast", "hcl/parser", - "hcl/printer", "hcl/scanner", "hcl/strconv", "hcl/token", @@ -131,6 +149,12 @@ revision = "c2353362d570a7bfa228149c62842019201cfb71" version = "v1.8.0" +[[projects]] + name = "github.com/matttproud/golang_protobuf_extensions" + packages = ["pbutil"] + revision = "c12348ce28de40eed0136aa2b644d0ee0650e56c" + version = "v1.0.1" + [[projects]] branch = "master" name = "github.com/mitchellh/mapstructure" @@ -155,6 +179,42 @@ revision = "792786c7400a136282c1664665ae0a8db921c6c2" version = "v1.0.0" +[[projects]] + name = "github.com/prometheus/client_golang" + packages = [ + "prometheus", + "prometheus/promhttp" + ] + revision = "c5b7fccd204277076155f10851dad72b76a49317" + version = "v0.8.0" + +[[projects]] + branch = "master" + name = "github.com/prometheus/client_model" + packages = ["go"] + revision = "99fa1f4be8e564e8a6b613da7fa6f46c9edafc6c" + +[[projects]] + branch = "master" + name = "github.com/prometheus/common" + packages = [ + "expfmt", + "internal/bitbucket.org/ww/goautoneg", + "model" + ] + revision = "7600349dcfe1abd18d72d3a1770870d9800a7801" + +[[projects]] + branch = "master" + name = "github.com/prometheus/procfs" + packages = [ + ".", + "internal/util", + "nfs", + "xfs" + ] + revision = "40f013a808ec4fa79def444a1a56de4d1727efcb" + [[projects]] branch = "master" name = "github.com/rcrowley/go-metrics" @@ -179,8 +239,8 @@ [[projects]] name = "github.com/spf13/cobra" packages = ["."] - revision = "ef82de70bb3f60c65fb8eebacbb2d122ef517385" - version = "v0.0.3" + revision = "7b2c5ac9fc04fc5efafb60700713d4fa609b777b" + version = "v0.0.1" [[projects]] branch = "master" @@ -197,8 +257,8 @@ [[projects]] name = "github.com/spf13/viper" packages = ["."] - revision = "b5e8006cbee93ec955a89ab31e0e3ce3204f3736" - version = "v1.0.2" + revision = "25b30aa063fc18e48662b86996252eabdcf2f0c7" + version = "v1.0.0" [[projects]] name = "github.com/stretchr/testify" @@ -228,19 +288,6 @@ ] revision = "e2150783cd35f5b607daca48afd8c57ec54cc995" -[[projects]] - name = "github.com/tendermint/abci" - packages = [ - "client", - "example/code", - "example/counter", - "example/kvstore", - "server", - "types" - ] - revision = "198dccf0ddfd1bb176f87657e3286a05a6ed9540" - version = "v0.12.0" - [[projects]] branch = "master" name = "github.com/tendermint/ed25519" @@ -254,37 +301,20 @@ [[projects]] name = "github.com/tendermint/go-amino" packages = ["."] - revision = "ed62928576cfcaf887209dc96142cd79cdfff389" - version = "0.9.9" - -[[projects]] - name = "github.com/tendermint/go-crypto" - packages = ["."] - revision = "915416979bf70efa4bcbf1c6cd5d64c5fff9fc19" - version = "v0.6.2" - -[[projects]] - name = "github.com/tendermint/tmlibs" - packages = [ - "autofile", - "cli", - "cli/flags", - "clist", - "common", - "db", - "flowrate", - "log", - "merkle", - "test" - ] - revision = "692f1d86a6e2c0efa698fd1e4541b68c74ffaf38" - version = "v0.8.4" + revision = "2106ca61d91029c931fd54968c2bb02dc96b1412" + version = "0.10.1" [[projects]] branch = "master" name = "golang.org/x/crypto" packages = [ + "bcrypt", + "blowfish", + "chacha20poly1305", "curve25519", + "hkdf", + "internal/chacha20", + "internal/subtle", "nacl/box", "nacl/secretbox", "openpgp/armor", @@ -293,7 +323,7 @@ "ripemd160", "salsa20/salsa" ] - revision = "8ac0e0d97ce45cd83d1d7243c060cb8461dda5e9" + revision = "a49355c7e3f8fe157a85be2f77e6e269a0f89602" [[projects]] branch = "master" @@ -305,15 +335,19 @@ "http2/hpack", "idna", "internal/timeseries", + "netutil", "trace" ] - revision = "db08ff08e8622530d9ed3a0e8ac279f6d4c02196" + revision = "4cb1c02c05b0e749b0365f61ae859a8e0cfceed9" [[projects]] branch = "master" name = "golang.org/x/sys" - packages = ["unix"] - revision = "a9e25c09b96b8870693763211309e213c6ef299d" + packages = [ + "cpu", + "unix" + ] + revision = "7138fd3d9dc8335c567ca206f4333fb75eb05d56" [[projects]] name = "golang.org/x/text" @@ -346,9 +380,13 @@ packages = [ ".", "balancer", + "balancer/base", + "balancer/roundrobin", "codes", "connectivity", "credentials", + "encoding", + "encoding/proto", "grpclb/grpc_lb_v1/messages", "grpclog", "internal", @@ -357,13 +395,15 @@ "naming", "peer", "resolver", + "resolver/dns", + "resolver/passthrough", "stats", "status", "tap", "transport" ] - revision = "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" - version = "v1.7.5" + revision = "d11072e7ca9811b1100b80ca0269ac831f06d024" + version = "v1.11.3" [[projects]] name = "gopkg.in/yaml.v2" @@ -374,6 +414,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "62d0626fa963b25411234da915fb3b8bfc7aefffd76824d27ac1647ca2b5d489" + inputs-digest = "71753a9d4ece4252d23941f116f5ff66c0d5da730a099e5a9867491d223ed93b" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index b7393a475..28394b8b3 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -35,15 +35,15 @@ [[constraint]] name = "github.com/go-kit/kit" - version = "~0.6.0" + version = "=0.6.0" [[constraint]] name = "github.com/gogo/protobuf" - version = "~1.0.0" + version = "=1.0.0" [[constraint]] name = "github.com/golang/protobuf" - version = "~1.0.0" + version = "=1.0.0" [[constraint]] name = "github.com/gorilla/websocket" @@ -51,7 +51,7 @@ [[constraint]] name = "github.com/pkg/errors" - version = "~0.8.0" + version = "=0.8.0" [[constraint]] name = "github.com/rcrowley/go-metrics" @@ -59,35 +59,23 @@ [[constraint]] name = "github.com/spf13/cobra" - version = "~0.0.1" + version = "=0.0.1" [[constraint]] name = "github.com/spf13/viper" - version = "~1.0.0" + version = "=1.0.0" [[constraint]] name = "github.com/stretchr/testify" version = "~1.2.1" -[[constraint]] - name = "github.com/tendermint/abci" - version = "~0.12.0" - -[[constraint]] - name = "github.com/tendermint/go-crypto" - version = "~0.6.2" - [[constraint]] name = "github.com/tendermint/go-amino" - version = "=0.9.9" - -[[override]] - name = "github.com/tendermint/tmlibs" - version = "~0.8.4" + version = "~0.10.1" [[constraint]] name = "google.golang.org/grpc" - version = "~1.7.3" + version = "~1.11.3" # this got updated and broke, so locked to an old working commit ... [[override]] @@ -97,3 +85,11 @@ [prune] go-tests = true unused-packages = true + +[[constraint]] + name = "github.com/prometheus/client_golang" + version = "0.8.0" + +[[constraint]] + branch = "master" + name = "golang.org/x/net" diff --git a/Makefile b/Makefile old mode 100755 new mode 100644 diff --git a/README.md b/README.md index 52e93a7cd..2f7d13cdd 100644 --- a/README.md +++ b/README.md @@ -50,13 +50,13 @@ Go version | Go1.9 or higher ## Install -See the [install instructions](/docs/install.rst) +See the [install instructions](/docs/install.md) ## Quick Start -- [Single node](/docs/using-tendermint.rst) +- [Single node](/docs/using-tendermint.md) - [Local cluster using docker-compose](/networks/local) -- [Remote cluster using terraform and ansible](/docs/terraform-and-ansible.rst) +- [Remote cluster using terraform and ansible](/docs/terraform-and-ansible.md) - [Join the public testnet](https://cosmos.network/testnet) ## Resources @@ -72,10 +72,7 @@ Additional information about some - and eventually all - of the sub-projects bel ### Sub-projects -* [ABCI](http://github.com/tendermint/abci), the Application Blockchain Interface -* [Go-Wire](http://github.com/tendermint/go-wire), a deterministic serialization library -* [Go-Crypto](http://github.com/tendermint/go-crypto), an elliptic curve cryptography library -* [TmLibs](http://github.com/tendermint/tmlibs), an assortment of Go libraries used internally +* [Amino](http://github.com/tendermint/go-amino), a reflection-based improvement on proto3 * [IAVL](http://github.com/tendermint/iavl), Merkleized IAVL+ Tree implementation ### Tools @@ -119,8 +116,8 @@ CHANGELOG even if they don't lead to MINOR version bumps: - node Exported objects in these packages that are not covered by the versioning scheme -are explicitly marked by `// UNSTABLE` in their go doc comment and may change at any time. -Functions, types, and values in any other package may also change at any time. +are explicitly marked by `// UNSTABLE` in their go doc comment and may change at any +time without notice. Functions, types, and values in any other package may also change at any time. ### Upgrades diff --git a/abci/Dockerfile.develop b/abci/Dockerfile.develop new file mode 100644 index 000000000..c6ec05f69 --- /dev/null +++ b/abci/Dockerfile.develop @@ -0,0 +1,23 @@ +FROM golang:latest + +RUN mkdir -p /go/src/github.com/tendermint/abci +WORKDIR /go/src/github.com/tendermint/abci + +COPY Makefile /go/src/github.com/tendermint/abci/ + +# see make protoc for details on ldconfig +RUN make get_protoc && ldconfig + +# killall is used in tests +RUN apt-get update && apt-get install -y \ + psmisc \ + && rm -rf /var/lib/apt/lists/* + +COPY Gopkg.toml /go/src/github.com/tendermint/abci/ +COPY Gopkg.lock /go/src/github.com/tendermint/abci/ +RUN make get_tools + +# see https://github.com/golang/dep/issues/1312 +RUN dep ensure -vendor-only + +COPY . /go/src/github.com/tendermint/abci diff --git a/abci/Makefile b/abci/Makefile new file mode 100644 index 000000000..7d1c4b2e5 --- /dev/null +++ b/abci/Makefile @@ -0,0 +1,174 @@ +GOTOOLS = \ + github.com/mitchellh/gox \ + github.com/golang/dep/cmd/dep \ + gopkg.in/alecthomas/gometalinter.v2 \ + github.com/gogo/protobuf/protoc-gen-gogo \ + github.com/gogo/protobuf/gogoproto +GOTOOLS_CHECK = gox dep gometalinter.v2 protoc protoc-gen-gogo +PACKAGES=$(shell go list ./... | grep -v '/vendor/') +INCLUDE = -I=. -I=${GOPATH}/src -I=${GOPATH}/src/github.com/gogo/protobuf/protobuf + +all: check get_vendor_deps protoc build test install metalinter + +check: check_tools + + +######################################## +### Build + +protoc: + ## If you get the following error, + ## "error while loading shared libraries: libprotobuf.so.14: cannot open shared object file: No such file or directory" + ## See https://stackoverflow.com/a/25518702 + protoc $(INCLUDE) --gogo_out=plugins=grpc:. types/*.proto + @echo "--> adding nolint declarations to protobuf generated files" + @awk '/package types/ { print "//nolint: gas"; print; next }1' types/types.pb.go > types/types.pb.go.new + @mv types/types.pb.go.new types/types.pb.go + +build: + @go build -i ./cmd/... + +dist: + @bash scripts/dist.sh + @bash scripts/publish.sh + +install: + @go install ./cmd/... + + +######################################## +### Tools & dependencies + +check_tools: + @# https://stackoverflow.com/a/25668869 + @echo "Found tools: $(foreach tool,$(GOTOOLS_CHECK),\ + $(if $(shell which $(tool)),$(tool),$(error "No $(tool) in PATH")))" + +get_tools: + @echo "--> Installing tools" + go get -u -v $(GOTOOLS) + @gometalinter.v2 --install + +get_protoc: + @# https://github.com/google/protobuf/releases + curl -L https://github.com/google/protobuf/releases/download/v3.4.1/protobuf-cpp-3.4.1.tar.gz | tar xvz && \ + cd protobuf-3.4.1 && \ + DIST_LANG=cpp ./configure && \ + make && \ + make install && \ + cd .. && \ + rm -rf protobuf-3.4.1 + +update_tools: + @echo "--> Updating tools" + @go get -u $(GOTOOLS) + +get_vendor_deps: + @rm -rf vendor/ + @echo "--> Running dep ensure" + @dep ensure + + +######################################## +### Testing + +test: + @find . -path ./vendor -prune -o -name "*.sock" -exec rm {} \; + @echo "==> Running go test" + @go test $(PACKAGES) + +test_race: + @find . -path ./vendor -prune -o -name "*.sock" -exec rm {} \; + @echo "==> Running go test --race" + @go test -v -race $(PACKAGES) + +### three tests tested by Jenkins +test_cover: + @ bash tests/test_cover.sh + +test_apps: + # test the counter using a go test script + @ bash tests/test_app/test.sh + +test_cli: + # test the cli against the examples in the tutorial at: + # http://tendermint.readthedocs.io/projects/tools/en/master/abci-cli.html + # + # XXX: if this test fails, fix it and update the docs at: + # https://github.com/tendermint/tendermint/blob/develop/docs/abci-cli.rst + @ bash tests/test_cli/test.sh + +######################################## +### Formatting, linting, and vetting + +fmt: + @go fmt ./... + +metalinter: + @echo "==> Running linter" + gometalinter.v2 --vendor --deadline=600s --disable-all \ + --enable=maligned \ + --enable=deadcode \ + --enable=goconst \ + --enable=goimports \ + --enable=gosimple \ + --enable=ineffassign \ + --enable=megacheck \ + --enable=misspell \ + --enable=staticcheck \ + --enable=safesql \ + --enable=structcheck \ + --enable=unconvert \ + --enable=unused \ + --enable=varcheck \ + --enable=vetshadow \ + ./... + #--enable=gas \ + #--enable=dupl \ + #--enable=errcheck \ + #--enable=gocyclo \ + #--enable=golint \ <== comments on anything exported + #--enable=gotype \ + #--enable=interfacer \ + #--enable=unparam \ + #--enable=vet \ + +metalinter_all: + protoc $(INCLUDE) --lint_out=. types/*.proto + gometalinter.v2 --vendor --deadline=600s --enable-all --disable=lll ./... + + +######################################## +### Docker + +DEVDOC_SAVE = docker commit `docker ps -a -n 1 -q` devdoc:local + +docker_build: + docker build -t "tendermint/abci-dev" -f Dockerfile.develop . + +docker_run: + docker run -it -v "$(CURDIR):/go/src/github.com/tendermint/abci" -w "/go/src/github.com/tendermint/abci" "tendermint/abci-dev" /bin/bash + +docker_run_rm: + docker run -it --rm -v "$(CURDIR):/go/src/github.com/tendermint/abci" -w "/go/src/github.com/tendermint/abci" "tendermint/abci-dev" /bin/bash + +devdoc_init: + docker run -it -v "$(CURDIR):/go/src/github.com/tendermint/abci" -w "/go/src/github.com/tendermint/abci" tendermint/devdoc echo + # TODO make this safer + $(call DEVDOC_SAVE) + +devdoc: + docker run -it -v "$(CURDIR):/go/src/github.com/tendermint/abci" -w "/go/src/github.com/tendermint/abci" devdoc:local bash + +devdoc_save: + # TODO make this safer + $(call DEVDOC_SAVE) + +devdoc_clean: + docker rmi $$(docker images -f "dangling=true" -q) + + +# To avoid unintended conflicts with file names, always add to .PHONY +# unless there is a reason not to. +# https://www.gnu.org/software/make/manual/html_node/Phony-Targets.html +.PHONY: check protoc build dist install check_tools get_tools get_protoc update_tools get_vendor_deps test test_race fmt metalinter metalinter_all docker_build docker_run docker_run_rm devdoc_init devdoc devdoc_save devdoc_clean diff --git a/abci/README.md b/abci/README.md new file mode 100644 index 000000000..6de9f7069 --- /dev/null +++ b/abci/README.md @@ -0,0 +1,168 @@ +# Application BlockChain Interface (ABCI) + +[![CircleCI](https://circleci.com/gh/tendermint/abci.svg?style=svg)](https://circleci.com/gh/tendermint/abci) + +Blockchains are systems for multi-master state machine replication. +**ABCI** is an interface that defines the boundary between the replication engine (the blockchain), +and the state machine (the application). +Using a socket protocol, a consensus engine running in one process +can manage an application state running in another. + +Previously, the ABCI was referred to as TMSP. + +The community has provided a number of addtional implementations, see the [Tendermint Ecosystem](https://tendermint.com/ecosystem) + +## Specification + +A detailed description of the ABCI methods and message types is contained in: + +- [A prose specification](specification.md) +- [A protobuf file](https://github.com/tendermint/abci/blob/master/types/types.proto) +- [A Go interface](https://github.com/tendermint/abci/blob/master/types/application.go). + +For more background information on ABCI, motivations, and tendermint, please visit [the documentation](http://tendermint.readthedocs.io/en/master/). +The two guides to focus on are the `Application Development Guide` and `Using ABCI-CLI`. + + +## Protocl Buffers + +To compile the protobuf file, run: + +``` +make protoc +``` + +See `protoc --help` and [the Protocol Buffers site](https://developers.google.com/protocol-buffers) +for details on compiling for other languages. Note we also include a [GRPC](http://www.grpc.io/docs) +service definition. + +## Install ABCI-CLI + +The `abci-cli` is a simple tool for debugging ABCI servers and running some +example apps. To install it: + +``` +go get github.com/tendermint/abci +cd $GOPATH/src/github.com/tendermint/abci +make get_vendor_deps +make install +``` + +## Implementation + +We provide three implementations of the ABCI in Go: + +- Golang in-process +- ABCI-socket +- GRPC + +Note the GRPC version is maintained primarily to simplify onboarding and prototyping and is not receiving the same +attention to security and performance as the others + +### In Process + +The simplest implementation just uses function calls within Go. +This means ABCI applications written in Golang can be compiled with TendermintCore and run as a single binary. + +See the [examples](#examples) below for more information. + +### Socket (TSP) + +ABCI is best implemented as a streaming protocol. +The socket implementation provides for asynchronous, ordered message passing over unix or tcp. +Messages are serialized using Protobuf3 and length-prefixed with a [signed Varint](https://developers.google.com/protocol-buffers/docs/encoding?csw=1#signed-integers) + +For example, if the Protobuf3 encoded ABCI message is `0xDEADBEEF` (4 bytes), the length-prefixed message is `0x08DEADBEEF`, since `0x08` is the signed varint +encoding of `4`. If the Protobuf3 encoded ABCI message is 65535 bytes long, the length-prefixed message would be like `0xFEFF07...`. + +Note the benefit of using this `varint` encoding over the old version (where integers were encoded as `` is that +it is the standard way to encode integers in Protobuf. It is also generally shorter. + +### GRPC + +GRPC is an rpc framework native to Protocol Buffers with support in many languages. +Implementing the ABCI using GRPC can allow for faster prototyping, but is expected to be much slower than +the ordered, asynchronous socket protocol. The implementation has also not received as much testing or review. + +Note the length-prefixing used in the socket implementation does not apply for GRPC. + +## Usage + +The `abci-cli` tool wraps an ABCI client and can be used for probing/testing an ABCI server. +For instance, `abci-cli test` will run a test sequence against a listening server running the Counter application (see below). +It can also be used to run some example applications. +See [the documentation](http://tendermint.readthedocs.io/en/master/) for more details. + +### Examples + +Check out the variety of example applications in the [example directory](example/). +It also contains the code refered to by the `counter` and `kvstore` apps; these apps come +built into the `abci-cli` binary. + +#### Counter + +The `abci-cli counter` application illustrates nonce checking in transactions. It's code looks like: + +```golang +func cmdCounter(cmd *cobra.Command, args []string) error { + + app := counter.NewCounterApplication(flagSerial) + + logger := log.NewTMLogger(log.NewSyncWriter(os.Stdout)) + + // Start the listener + srv, err := server.NewServer(flagAddrC, flagAbci, app) + if err != nil { + return err + } + srv.SetLogger(logger.With("module", "abci-server")) + if err := srv.Start(); err != nil { + return err + } + + // Wait forever + cmn.TrapSignal(func() { + // Cleanup + srv.Stop() + }) + return nil +} +``` + +and can be found in [this file](cmd/abci-cli/abci-cli.go). + +#### kvstore + +The `abci-cli kvstore` application, which illustrates a simple key-value Merkle tree + +```golang +func cmdKVStore(cmd *cobra.Command, args []string) error { + logger := log.NewTMLogger(log.NewSyncWriter(os.Stdout)) + + // Create the application - in memory or persisted to disk + var app types.Application + if flagPersist == "" { + app = kvstore.NewKVStoreApplication() + } else { + app = kvstore.NewPersistentKVStoreApplication(flagPersist) + app.(*kvstore.PersistentKVStoreApplication).SetLogger(logger.With("module", "kvstore")) + } + + // Start the listener + srv, err := server.NewServer(flagAddrD, flagAbci, app) + if err != nil { + return err + } + srv.SetLogger(logger.With("module", "abci-server")) + if err := srv.Start(); err != nil { + return err + } + + // Wait forever + cmn.TrapSignal(func() { + // Cleanup + srv.Stop() + }) + return nil +} +``` diff --git a/abci/client/client.go b/abci/client/client.go new file mode 100644 index 000000000..558588107 --- /dev/null +++ b/abci/client/client.go @@ -0,0 +1,129 @@ +package abcicli + +import ( + "fmt" + "sync" + + "github.com/tendermint/tendermint/abci/types" + cmn "github.com/tendermint/tendermint/libs/common" +) + +const ( + dialRetryIntervalSeconds = 3 + echoRetryIntervalSeconds = 1 +) + +// Client defines an interface for an ABCI client. +// All `Async` methods return a `ReqRes` object. +// All `Sync` methods return the appropriate protobuf ResponseXxx struct and an error. +// Note these are client errors, eg. ABCI socket connectivity issues. +// Application-related errors are reflected in response via ABCI error codes and logs. +type Client interface { + cmn.Service + + SetResponseCallback(Callback) + Error() error + + FlushAsync() *ReqRes + EchoAsync(msg string) *ReqRes + InfoAsync(types.RequestInfo) *ReqRes + SetOptionAsync(types.RequestSetOption) *ReqRes + DeliverTxAsync(tx []byte) *ReqRes + CheckTxAsync(tx []byte) *ReqRes + QueryAsync(types.RequestQuery) *ReqRes + CommitAsync() *ReqRes + InitChainAsync(types.RequestInitChain) *ReqRes + BeginBlockAsync(types.RequestBeginBlock) *ReqRes + EndBlockAsync(types.RequestEndBlock) *ReqRes + + FlushSync() error + EchoSync(msg string) (*types.ResponseEcho, error) + InfoSync(types.RequestInfo) (*types.ResponseInfo, error) + SetOptionSync(types.RequestSetOption) (*types.ResponseSetOption, error) + DeliverTxSync(tx []byte) (*types.ResponseDeliverTx, error) + CheckTxSync(tx []byte) (*types.ResponseCheckTx, error) + QuerySync(types.RequestQuery) (*types.ResponseQuery, error) + CommitSync() (*types.ResponseCommit, error) + InitChainSync(types.RequestInitChain) (*types.ResponseInitChain, error) + BeginBlockSync(types.RequestBeginBlock) (*types.ResponseBeginBlock, error) + EndBlockSync(types.RequestEndBlock) (*types.ResponseEndBlock, error) +} + +//---------------------------------------- + +// NewClient returns a new ABCI client of the specified transport type. +// It returns an error if the transport is not "socket" or "grpc" +func NewClient(addr, transport string, mustConnect bool) (client Client, err error) { + switch transport { + case "socket": + client = NewSocketClient(addr, mustConnect) + case "grpc": + client = NewGRPCClient(addr, mustConnect) + default: + err = fmt.Errorf("Unknown abci transport %s", transport) + } + return +} + +//---------------------------------------- + +type Callback func(*types.Request, *types.Response) + +//---------------------------------------- + +type ReqRes struct { + *types.Request + *sync.WaitGroup + *types.Response // Not set atomically, so be sure to use WaitGroup. + + mtx sync.Mutex + done bool // Gets set to true once *after* WaitGroup.Done(). + cb func(*types.Response) // A single callback that may be set. +} + +func NewReqRes(req *types.Request) *ReqRes { + return &ReqRes{ + Request: req, + WaitGroup: waitGroup1(), + Response: nil, + + done: false, + cb: nil, + } +} + +// Sets the callback for this ReqRes atomically. +// If reqRes is already done, calls cb immediately. +// NOTE: reqRes.cb should not change if reqRes.done. +// NOTE: only one callback is supported. +func (reqRes *ReqRes) SetCallback(cb func(res *types.Response)) { + reqRes.mtx.Lock() + + if reqRes.done { + reqRes.mtx.Unlock() + cb(reqRes.Response) + return + } + + defer reqRes.mtx.Unlock() + reqRes.cb = cb +} + +func (reqRes *ReqRes) GetCallback() func(*types.Response) { + reqRes.mtx.Lock() + defer reqRes.mtx.Unlock() + return reqRes.cb +} + +// NOTE: it should be safe to read reqRes.cb without locks after this. +func (reqRes *ReqRes) SetDone() { + reqRes.mtx.Lock() + reqRes.done = true + reqRes.mtx.Unlock() +} + +func waitGroup1() (wg *sync.WaitGroup) { + wg = &sync.WaitGroup{} + wg.Add(1) + return +} diff --git a/abci/client/grpc_client.go b/abci/client/grpc_client.go new file mode 100644 index 000000000..502ee0fcd --- /dev/null +++ b/abci/client/grpc_client.go @@ -0,0 +1,301 @@ +package abcicli + +import ( + "fmt" + "net" + "sync" + "time" + + context "golang.org/x/net/context" + grpc "google.golang.org/grpc" + + "github.com/tendermint/tendermint/abci/types" + cmn "github.com/tendermint/tendermint/libs/common" +) + +var _ Client = (*grpcClient)(nil) + +// A stripped copy of the remoteClient that makes +// synchronous calls using grpc +type grpcClient struct { + cmn.BaseService + mustConnect bool + + client types.ABCIApplicationClient + + mtx sync.Mutex + addr string + err error + resCb func(*types.Request, *types.Response) // listens to all callbacks +} + +func NewGRPCClient(addr string, mustConnect bool) *grpcClient { + cli := &grpcClient{ + addr: addr, + mustConnect: mustConnect, + } + cli.BaseService = *cmn.NewBaseService(nil, "grpcClient", cli) + return cli +} + +func dialerFunc(addr string, timeout time.Duration) (net.Conn, error) { + return cmn.Connect(addr) +} + +func (cli *grpcClient) OnStart() error { + if err := cli.BaseService.OnStart(); err != nil { + return err + } +RETRY_LOOP: + for { + conn, err := grpc.Dial(cli.addr, grpc.WithInsecure(), grpc.WithDialer(dialerFunc)) + if err != nil { + if cli.mustConnect { + return err + } + cli.Logger.Error(fmt.Sprintf("abci.grpcClient failed to connect to %v. Retrying...\n", cli.addr)) + time.Sleep(time.Second * dialRetryIntervalSeconds) + continue RETRY_LOOP + } + + cli.Logger.Info("Dialed server. Waiting for echo.", "addr", cli.addr) + client := types.NewABCIApplicationClient(conn) + + ENSURE_CONNECTED: + for { + _, err := client.Echo(context.Background(), &types.RequestEcho{"hello"}, grpc.FailFast(true)) + if err == nil { + break ENSURE_CONNECTED + } + cli.Logger.Error("Echo failed", "err", err) + time.Sleep(time.Second * echoRetryIntervalSeconds) + } + + cli.client = client + return nil + } +} + +func (cli *grpcClient) OnStop() { + cli.BaseService.OnStop() + cli.mtx.Lock() + defer cli.mtx.Unlock() + // TODO: how to close conn? its not a net.Conn and grpc doesn't expose a Close() + /*if cli.client.conn != nil { + cli.client.conn.Close() + }*/ +} + +func (cli *grpcClient) StopForError(err error) { + cli.mtx.Lock() + if !cli.IsRunning() { + return + } + + if cli.err == nil { + cli.err = err + } + cli.mtx.Unlock() + + cli.Logger.Error(fmt.Sprintf("Stopping abci.grpcClient for error: %v", err.Error())) + cli.Stop() +} + +func (cli *grpcClient) Error() error { + cli.mtx.Lock() + defer cli.mtx.Unlock() + return cli.err +} + +// Set listener for all responses +// NOTE: callback may get internally generated flush responses. +func (cli *grpcClient) SetResponseCallback(resCb Callback) { + cli.mtx.Lock() + defer cli.mtx.Unlock() + cli.resCb = resCb +} + +//---------------------------------------- +// GRPC calls are synchronous, but some callbacks expect to be called asynchronously +// (eg. the mempool expects to be able to lock to remove bad txs from cache). +// To accommodate, we finish each call in its own go-routine, +// which is expensive, but easy - if you want something better, use the socket protocol! +// maybe one day, if people really want it, we use grpc streams, +// but hopefully not :D + +func (cli *grpcClient) EchoAsync(msg string) *ReqRes { + req := types.ToRequestEcho(msg) + res, err := cli.client.Echo(context.Background(), req.GetEcho(), grpc.FailFast(true)) + if err != nil { + cli.StopForError(err) + } + return cli.finishAsyncCall(req, &types.Response{&types.Response_Echo{res}}) +} + +func (cli *grpcClient) FlushAsync() *ReqRes { + req := types.ToRequestFlush() + res, err := cli.client.Flush(context.Background(), req.GetFlush(), grpc.FailFast(true)) + if err != nil { + cli.StopForError(err) + } + return cli.finishAsyncCall(req, &types.Response{&types.Response_Flush{res}}) +} + +func (cli *grpcClient) InfoAsync(params types.RequestInfo) *ReqRes { + req := types.ToRequestInfo(params) + res, err := cli.client.Info(context.Background(), req.GetInfo(), grpc.FailFast(true)) + if err != nil { + cli.StopForError(err) + } + return cli.finishAsyncCall(req, &types.Response{&types.Response_Info{res}}) +} + +func (cli *grpcClient) SetOptionAsync(params types.RequestSetOption) *ReqRes { + req := types.ToRequestSetOption(params) + res, err := cli.client.SetOption(context.Background(), req.GetSetOption(), grpc.FailFast(true)) + if err != nil { + cli.StopForError(err) + } + return cli.finishAsyncCall(req, &types.Response{&types.Response_SetOption{res}}) +} + +func (cli *grpcClient) DeliverTxAsync(tx []byte) *ReqRes { + req := types.ToRequestDeliverTx(tx) + res, err := cli.client.DeliverTx(context.Background(), req.GetDeliverTx(), grpc.FailFast(true)) + if err != nil { + cli.StopForError(err) + } + return cli.finishAsyncCall(req, &types.Response{&types.Response_DeliverTx{res}}) +} + +func (cli *grpcClient) CheckTxAsync(tx []byte) *ReqRes { + req := types.ToRequestCheckTx(tx) + res, err := cli.client.CheckTx(context.Background(), req.GetCheckTx(), grpc.FailFast(true)) + if err != nil { + cli.StopForError(err) + } + return cli.finishAsyncCall(req, &types.Response{&types.Response_CheckTx{res}}) +} + +func (cli *grpcClient) QueryAsync(params types.RequestQuery) *ReqRes { + req := types.ToRequestQuery(params) + res, err := cli.client.Query(context.Background(), req.GetQuery(), grpc.FailFast(true)) + if err != nil { + cli.StopForError(err) + } + return cli.finishAsyncCall(req, &types.Response{&types.Response_Query{res}}) +} + +func (cli *grpcClient) CommitAsync() *ReqRes { + req := types.ToRequestCommit() + res, err := cli.client.Commit(context.Background(), req.GetCommit(), grpc.FailFast(true)) + if err != nil { + cli.StopForError(err) + } + return cli.finishAsyncCall(req, &types.Response{&types.Response_Commit{res}}) +} + +func (cli *grpcClient) InitChainAsync(params types.RequestInitChain) *ReqRes { + req := types.ToRequestInitChain(params) + res, err := cli.client.InitChain(context.Background(), req.GetInitChain(), grpc.FailFast(true)) + if err != nil { + cli.StopForError(err) + } + return cli.finishAsyncCall(req, &types.Response{&types.Response_InitChain{res}}) +} + +func (cli *grpcClient) BeginBlockAsync(params types.RequestBeginBlock) *ReqRes { + req := types.ToRequestBeginBlock(params) + res, err := cli.client.BeginBlock(context.Background(), req.GetBeginBlock(), grpc.FailFast(true)) + if err != nil { + cli.StopForError(err) + } + return cli.finishAsyncCall(req, &types.Response{&types.Response_BeginBlock{res}}) +} + +func (cli *grpcClient) EndBlockAsync(params types.RequestEndBlock) *ReqRes { + req := types.ToRequestEndBlock(params) + res, err := cli.client.EndBlock(context.Background(), req.GetEndBlock(), grpc.FailFast(true)) + if err != nil { + cli.StopForError(err) + } + return cli.finishAsyncCall(req, &types.Response{&types.Response_EndBlock{res}}) +} + +func (cli *grpcClient) finishAsyncCall(req *types.Request, res *types.Response) *ReqRes { + reqres := NewReqRes(req) + reqres.Response = res // Set response + reqres.Done() // Release waiters + reqres.SetDone() // so reqRes.SetCallback will run the callback + + // go routine for callbacks + go func() { + // Notify reqRes listener if set + if cb := reqres.GetCallback(); cb != nil { + cb(res) + } + + // Notify client listener if set + if cli.resCb != nil { + cli.resCb(reqres.Request, res) + } + }() + return reqres +} + +//---------------------------------------- + +func (cli *grpcClient) FlushSync() error { + return nil +} + +func (cli *grpcClient) EchoSync(msg string) (*types.ResponseEcho, error) { + reqres := cli.EchoAsync(msg) + // StopForError should already have been called if error is set + return reqres.Response.GetEcho(), cli.Error() +} + +func (cli *grpcClient) InfoSync(req types.RequestInfo) (*types.ResponseInfo, error) { + reqres := cli.InfoAsync(req) + return reqres.Response.GetInfo(), cli.Error() +} + +func (cli *grpcClient) SetOptionSync(req types.RequestSetOption) (*types.ResponseSetOption, error) { + reqres := cli.SetOptionAsync(req) + return reqres.Response.GetSetOption(), cli.Error() +} + +func (cli *grpcClient) DeliverTxSync(tx []byte) (*types.ResponseDeliverTx, error) { + reqres := cli.DeliverTxAsync(tx) + return reqres.Response.GetDeliverTx(), cli.Error() +} + +func (cli *grpcClient) CheckTxSync(tx []byte) (*types.ResponseCheckTx, error) { + reqres := cli.CheckTxAsync(tx) + return reqres.Response.GetCheckTx(), cli.Error() +} + +func (cli *grpcClient) QuerySync(req types.RequestQuery) (*types.ResponseQuery, error) { + reqres := cli.QueryAsync(req) + return reqres.Response.GetQuery(), cli.Error() +} + +func (cli *grpcClient) CommitSync() (*types.ResponseCommit, error) { + reqres := cli.CommitAsync() + return reqres.Response.GetCommit(), cli.Error() +} + +func (cli *grpcClient) InitChainSync(params types.RequestInitChain) (*types.ResponseInitChain, error) { + reqres := cli.InitChainAsync(params) + return reqres.Response.GetInitChain(), cli.Error() +} + +func (cli *grpcClient) BeginBlockSync(params types.RequestBeginBlock) (*types.ResponseBeginBlock, error) { + reqres := cli.BeginBlockAsync(params) + return reqres.Response.GetBeginBlock(), cli.Error() +} + +func (cli *grpcClient) EndBlockSync(params types.RequestEndBlock) (*types.ResponseEndBlock, error) { + reqres := cli.EndBlockAsync(params) + return reqres.Response.GetEndBlock(), cli.Error() +} diff --git a/abci/client/local_client.go b/abci/client/local_client.go new file mode 100644 index 000000000..3d1f8d8e4 --- /dev/null +++ b/abci/client/local_client.go @@ -0,0 +1,230 @@ +package abcicli + +import ( + "sync" + + types "github.com/tendermint/tendermint/abci/types" + cmn "github.com/tendermint/tendermint/libs/common" +) + +var _ Client = (*localClient)(nil) + +type localClient struct { + cmn.BaseService + mtx *sync.Mutex + types.Application + Callback +} + +func NewLocalClient(mtx *sync.Mutex, app types.Application) *localClient { + if mtx == nil { + mtx = new(sync.Mutex) + } + cli := &localClient{ + mtx: mtx, + Application: app, + } + cli.BaseService = *cmn.NewBaseService(nil, "localClient", cli) + return cli +} + +func (app *localClient) SetResponseCallback(cb Callback) { + app.mtx.Lock() + defer app.mtx.Unlock() + app.Callback = cb +} + +// TODO: change types.Application to include Error()? +func (app *localClient) Error() error { + return nil +} + +func (app *localClient) FlushAsync() *ReqRes { + // Do nothing + return newLocalReqRes(types.ToRequestFlush(), nil) +} + +func (app *localClient) EchoAsync(msg string) *ReqRes { + return app.callback( + types.ToRequestEcho(msg), + types.ToResponseEcho(msg), + ) +} + +func (app *localClient) InfoAsync(req types.RequestInfo) *ReqRes { + app.mtx.Lock() + res := app.Application.Info(req) + app.mtx.Unlock() + return app.callback( + types.ToRequestInfo(req), + types.ToResponseInfo(res), + ) +} + +func (app *localClient) SetOptionAsync(req types.RequestSetOption) *ReqRes { + app.mtx.Lock() + res := app.Application.SetOption(req) + app.mtx.Unlock() + return app.callback( + types.ToRequestSetOption(req), + types.ToResponseSetOption(res), + ) +} + +func (app *localClient) DeliverTxAsync(tx []byte) *ReqRes { + app.mtx.Lock() + res := app.Application.DeliverTx(tx) + app.mtx.Unlock() + return app.callback( + types.ToRequestDeliverTx(tx), + types.ToResponseDeliverTx(res), + ) +} + +func (app *localClient) CheckTxAsync(tx []byte) *ReqRes { + app.mtx.Lock() + res := app.Application.CheckTx(tx) + app.mtx.Unlock() + return app.callback( + types.ToRequestCheckTx(tx), + types.ToResponseCheckTx(res), + ) +} + +func (app *localClient) QueryAsync(req types.RequestQuery) *ReqRes { + app.mtx.Lock() + res := app.Application.Query(req) + app.mtx.Unlock() + return app.callback( + types.ToRequestQuery(req), + types.ToResponseQuery(res), + ) +} + +func (app *localClient) CommitAsync() *ReqRes { + app.mtx.Lock() + res := app.Application.Commit() + app.mtx.Unlock() + return app.callback( + types.ToRequestCommit(), + types.ToResponseCommit(res), + ) +} + +func (app *localClient) InitChainAsync(req types.RequestInitChain) *ReqRes { + app.mtx.Lock() + res := app.Application.InitChain(req) + reqRes := app.callback( + types.ToRequestInitChain(req), + types.ToResponseInitChain(res), + ) + app.mtx.Unlock() + return reqRes +} + +func (app *localClient) BeginBlockAsync(req types.RequestBeginBlock) *ReqRes { + app.mtx.Lock() + res := app.Application.BeginBlock(req) + app.mtx.Unlock() + return app.callback( + types.ToRequestBeginBlock(req), + types.ToResponseBeginBlock(res), + ) +} + +func (app *localClient) EndBlockAsync(req types.RequestEndBlock) *ReqRes { + app.mtx.Lock() + res := app.Application.EndBlock(req) + app.mtx.Unlock() + return app.callback( + types.ToRequestEndBlock(req), + types.ToResponseEndBlock(res), + ) +} + +//------------------------------------------------------- + +func (app *localClient) FlushSync() error { + return nil +} + +func (app *localClient) EchoSync(msg string) (*types.ResponseEcho, error) { + return &types.ResponseEcho{msg}, nil +} + +func (app *localClient) InfoSync(req types.RequestInfo) (*types.ResponseInfo, error) { + app.mtx.Lock() + res := app.Application.Info(req) + app.mtx.Unlock() + return &res, nil +} + +func (app *localClient) SetOptionSync(req types.RequestSetOption) (*types.ResponseSetOption, error) { + app.mtx.Lock() + res := app.Application.SetOption(req) + app.mtx.Unlock() + return &res, nil +} + +func (app *localClient) DeliverTxSync(tx []byte) (*types.ResponseDeliverTx, error) { + app.mtx.Lock() + res := app.Application.DeliverTx(tx) + app.mtx.Unlock() + return &res, nil +} + +func (app *localClient) CheckTxSync(tx []byte) (*types.ResponseCheckTx, error) { + app.mtx.Lock() + res := app.Application.CheckTx(tx) + app.mtx.Unlock() + return &res, nil +} + +func (app *localClient) QuerySync(req types.RequestQuery) (*types.ResponseQuery, error) { + app.mtx.Lock() + res := app.Application.Query(req) + app.mtx.Unlock() + return &res, nil +} + +func (app *localClient) CommitSync() (*types.ResponseCommit, error) { + app.mtx.Lock() + res := app.Application.Commit() + app.mtx.Unlock() + return &res, nil +} + +func (app *localClient) InitChainSync(req types.RequestInitChain) (*types.ResponseInitChain, error) { + app.mtx.Lock() + res := app.Application.InitChain(req) + app.mtx.Unlock() + return &res, nil +} + +func (app *localClient) BeginBlockSync(req types.RequestBeginBlock) (*types.ResponseBeginBlock, error) { + app.mtx.Lock() + res := app.Application.BeginBlock(req) + app.mtx.Unlock() + return &res, nil +} + +func (app *localClient) EndBlockSync(req types.RequestEndBlock) (*types.ResponseEndBlock, error) { + app.mtx.Lock() + res := app.Application.EndBlock(req) + app.mtx.Unlock() + return &res, nil +} + +//------------------------------------------------------- + +func (app *localClient) callback(req *types.Request, res *types.Response) *ReqRes { + app.Callback(req, res) + return newLocalReqRes(req, res) +} + +func newLocalReqRes(req *types.Request, res *types.Response) *ReqRes { + reqRes := NewReqRes(req) + reqRes.Response = res + reqRes.SetDone() + return reqRes +} diff --git a/abci/client/socket_client.go b/abci/client/socket_client.go new file mode 100644 index 000000000..c3f88725c --- /dev/null +++ b/abci/client/socket_client.go @@ -0,0 +1,399 @@ +package abcicli + +import ( + "bufio" + "container/list" + "errors" + "fmt" + "net" + "reflect" + "sync" + "time" + + "github.com/tendermint/tendermint/abci/types" + cmn "github.com/tendermint/tendermint/libs/common" +) + +const reqQueueSize = 256 // TODO make configurable +// const maxResponseSize = 1048576 // 1MB TODO make configurable +const flushThrottleMS = 20 // Don't wait longer than... + +var _ Client = (*socketClient)(nil) + +// This is goroutine-safe, but users should beware that +// the application in general is not meant to be interfaced +// with concurrent callers. +type socketClient struct { + cmn.BaseService + + reqQueue chan *ReqRes + flushTimer *cmn.ThrottleTimer + mustConnect bool + + mtx sync.Mutex + addr string + conn net.Conn + err error + reqSent *list.List + resCb func(*types.Request, *types.Response) // listens to all callbacks + +} + +func NewSocketClient(addr string, mustConnect bool) *socketClient { + cli := &socketClient{ + reqQueue: make(chan *ReqRes, reqQueueSize), + flushTimer: cmn.NewThrottleTimer("socketClient", flushThrottleMS), + mustConnect: mustConnect, + + addr: addr, + reqSent: list.New(), + resCb: nil, + } + cli.BaseService = *cmn.NewBaseService(nil, "socketClient", cli) + return cli +} + +func (cli *socketClient) OnStart() error { + if err := cli.BaseService.OnStart(); err != nil { + return err + } + + var err error + var conn net.Conn +RETRY_LOOP: + for { + conn, err = cmn.Connect(cli.addr) + if err != nil { + if cli.mustConnect { + return err + } + cli.Logger.Error(fmt.Sprintf("abci.socketClient failed to connect to %v. Retrying...", cli.addr)) + time.Sleep(time.Second * dialRetryIntervalSeconds) + continue RETRY_LOOP + } + cli.conn = conn + + go cli.sendRequestsRoutine(conn) + go cli.recvResponseRoutine(conn) + + return nil + } +} + +func (cli *socketClient) OnStop() { + cli.BaseService.OnStop() + + cli.mtx.Lock() + defer cli.mtx.Unlock() + if cli.conn != nil { + cli.conn.Close() + } + + cli.flushQueue() +} + +// Stop the client and set the error +func (cli *socketClient) StopForError(err error) { + if !cli.IsRunning() { + return + } + + cli.mtx.Lock() + if cli.err == nil { + cli.err = err + } + cli.mtx.Unlock() + + cli.Logger.Error(fmt.Sprintf("Stopping abci.socketClient for error: %v", err.Error())) + cli.Stop() +} + +func (cli *socketClient) Error() error { + cli.mtx.Lock() + defer cli.mtx.Unlock() + return cli.err +} + +// Set listener for all responses +// NOTE: callback may get internally generated flush responses. +func (cli *socketClient) SetResponseCallback(resCb Callback) { + cli.mtx.Lock() + defer cli.mtx.Unlock() + cli.resCb = resCb +} + +//---------------------------------------- + +func (cli *socketClient) sendRequestsRoutine(conn net.Conn) { + + w := bufio.NewWriter(conn) + for { + select { + case <-cli.flushTimer.Ch: + select { + case cli.reqQueue <- NewReqRes(types.ToRequestFlush()): + default: + // Probably will fill the buffer, or retry later. + } + case <-cli.Quit(): + return + case reqres := <-cli.reqQueue: + cli.willSendReq(reqres) + err := types.WriteMessage(reqres.Request, w) + if err != nil { + cli.StopForError(fmt.Errorf("Error writing msg: %v", err)) + return + } + // cli.Logger.Debug("Sent request", "requestType", reflect.TypeOf(reqres.Request), "request", reqres.Request) + if _, ok := reqres.Request.Value.(*types.Request_Flush); ok { + err = w.Flush() + if err != nil { + cli.StopForError(fmt.Errorf("Error flushing writer: %v", err)) + return + } + } + } + } +} + +func (cli *socketClient) recvResponseRoutine(conn net.Conn) { + + r := bufio.NewReader(conn) // Buffer reads + for { + var res = &types.Response{} + err := types.ReadMessage(r, res) + if err != nil { + cli.StopForError(err) + return + } + switch r := res.Value.(type) { + case *types.Response_Exception: + // XXX After setting cli.err, release waiters (e.g. reqres.Done()) + cli.StopForError(errors.New(r.Exception.Error)) + return + default: + // cli.Logger.Debug("Received response", "responseType", reflect.TypeOf(res), "response", res) + err := cli.didRecvResponse(res) + if err != nil { + cli.StopForError(err) + return + } + } + } +} + +func (cli *socketClient) willSendReq(reqres *ReqRes) { + cli.mtx.Lock() + defer cli.mtx.Unlock() + cli.reqSent.PushBack(reqres) +} + +func (cli *socketClient) didRecvResponse(res *types.Response) error { + cli.mtx.Lock() + defer cli.mtx.Unlock() + + // Get the first ReqRes + next := cli.reqSent.Front() + if next == nil { + return fmt.Errorf("Unexpected result type %v when nothing expected", reflect.TypeOf(res.Value)) + } + reqres := next.Value.(*ReqRes) + if !resMatchesReq(reqres.Request, res) { + return fmt.Errorf("Unexpected result type %v when response to %v expected", + reflect.TypeOf(res.Value), reflect.TypeOf(reqres.Request.Value)) + } + + reqres.Response = res // Set response + reqres.Done() // Release waiters + cli.reqSent.Remove(next) // Pop first item from linked list + + // Notify reqRes listener if set + if cb := reqres.GetCallback(); cb != nil { + cb(res) + } + + // Notify client listener if set + if cli.resCb != nil { + cli.resCb(reqres.Request, res) + } + + return nil +} + +//---------------------------------------- + +func (cli *socketClient) EchoAsync(msg string) *ReqRes { + return cli.queueRequest(types.ToRequestEcho(msg)) +} + +func (cli *socketClient) FlushAsync() *ReqRes { + return cli.queueRequest(types.ToRequestFlush()) +} + +func (cli *socketClient) InfoAsync(req types.RequestInfo) *ReqRes { + return cli.queueRequest(types.ToRequestInfo(req)) +} + +func (cli *socketClient) SetOptionAsync(req types.RequestSetOption) *ReqRes { + return cli.queueRequest(types.ToRequestSetOption(req)) +} + +func (cli *socketClient) DeliverTxAsync(tx []byte) *ReqRes { + return cli.queueRequest(types.ToRequestDeliverTx(tx)) +} + +func (cli *socketClient) CheckTxAsync(tx []byte) *ReqRes { + return cli.queueRequest(types.ToRequestCheckTx(tx)) +} + +func (cli *socketClient) QueryAsync(req types.RequestQuery) *ReqRes { + return cli.queueRequest(types.ToRequestQuery(req)) +} + +func (cli *socketClient) CommitAsync() *ReqRes { + return cli.queueRequest(types.ToRequestCommit()) +} + +func (cli *socketClient) InitChainAsync(req types.RequestInitChain) *ReqRes { + return cli.queueRequest(types.ToRequestInitChain(req)) +} + +func (cli *socketClient) BeginBlockAsync(req types.RequestBeginBlock) *ReqRes { + return cli.queueRequest(types.ToRequestBeginBlock(req)) +} + +func (cli *socketClient) EndBlockAsync(req types.RequestEndBlock) *ReqRes { + return cli.queueRequest(types.ToRequestEndBlock(req)) +} + +//---------------------------------------- + +func (cli *socketClient) FlushSync() error { + reqRes := cli.queueRequest(types.ToRequestFlush()) + if err := cli.Error(); err != nil { + return err + } + reqRes.Wait() // NOTE: if we don't flush the queue, its possible to get stuck here + return cli.Error() +} + +func (cli *socketClient) EchoSync(msg string) (*types.ResponseEcho, error) { + reqres := cli.queueRequest(types.ToRequestEcho(msg)) + cli.FlushSync() + return reqres.Response.GetEcho(), cli.Error() +} + +func (cli *socketClient) InfoSync(req types.RequestInfo) (*types.ResponseInfo, error) { + reqres := cli.queueRequest(types.ToRequestInfo(req)) + cli.FlushSync() + return reqres.Response.GetInfo(), cli.Error() +} + +func (cli *socketClient) SetOptionSync(req types.RequestSetOption) (*types.ResponseSetOption, error) { + reqres := cli.queueRequest(types.ToRequestSetOption(req)) + cli.FlushSync() + return reqres.Response.GetSetOption(), cli.Error() +} + +func (cli *socketClient) DeliverTxSync(tx []byte) (*types.ResponseDeliverTx, error) { + reqres := cli.queueRequest(types.ToRequestDeliverTx(tx)) + cli.FlushSync() + return reqres.Response.GetDeliverTx(), cli.Error() +} + +func (cli *socketClient) CheckTxSync(tx []byte) (*types.ResponseCheckTx, error) { + reqres := cli.queueRequest(types.ToRequestCheckTx(tx)) + cli.FlushSync() + return reqres.Response.GetCheckTx(), cli.Error() +} + +func (cli *socketClient) QuerySync(req types.RequestQuery) (*types.ResponseQuery, error) { + reqres := cli.queueRequest(types.ToRequestQuery(req)) + cli.FlushSync() + return reqres.Response.GetQuery(), cli.Error() +} + +func (cli *socketClient) CommitSync() (*types.ResponseCommit, error) { + reqres := cli.queueRequest(types.ToRequestCommit()) + cli.FlushSync() + return reqres.Response.GetCommit(), cli.Error() +} + +func (cli *socketClient) InitChainSync(req types.RequestInitChain) (*types.ResponseInitChain, error) { + reqres := cli.queueRequest(types.ToRequestInitChain(req)) + cli.FlushSync() + return reqres.Response.GetInitChain(), cli.Error() +} + +func (cli *socketClient) BeginBlockSync(req types.RequestBeginBlock) (*types.ResponseBeginBlock, error) { + reqres := cli.queueRequest(types.ToRequestBeginBlock(req)) + cli.FlushSync() + return reqres.Response.GetBeginBlock(), cli.Error() +} + +func (cli *socketClient) EndBlockSync(req types.RequestEndBlock) (*types.ResponseEndBlock, error) { + reqres := cli.queueRequest(types.ToRequestEndBlock(req)) + cli.FlushSync() + return reqres.Response.GetEndBlock(), cli.Error() +} + +//---------------------------------------- + +func (cli *socketClient) queueRequest(req *types.Request) *ReqRes { + reqres := NewReqRes(req) + + // TODO: set cli.err if reqQueue times out + cli.reqQueue <- reqres + + // Maybe auto-flush, or unset auto-flush + switch req.Value.(type) { + case *types.Request_Flush: + cli.flushTimer.Unset() + default: + cli.flushTimer.Set() + } + + return reqres +} + +func (cli *socketClient) flushQueue() { +LOOP: + for { + select { + case reqres := <-cli.reqQueue: + reqres.Done() + default: + break LOOP + } + } +} + +//---------------------------------------- + +func resMatchesReq(req *types.Request, res *types.Response) (ok bool) { + switch req.Value.(type) { + case *types.Request_Echo: + _, ok = res.Value.(*types.Response_Echo) + case *types.Request_Flush: + _, ok = res.Value.(*types.Response_Flush) + case *types.Request_Info: + _, ok = res.Value.(*types.Response_Info) + case *types.Request_SetOption: + _, ok = res.Value.(*types.Response_SetOption) + case *types.Request_DeliverTx: + _, ok = res.Value.(*types.Response_DeliverTx) + case *types.Request_CheckTx: + _, ok = res.Value.(*types.Response_CheckTx) + case *types.Request_Commit: + _, ok = res.Value.(*types.Response_Commit) + case *types.Request_Query: + _, ok = res.Value.(*types.Response_Query) + case *types.Request_InitChain: + _, ok = res.Value.(*types.Response_InitChain) + case *types.Request_BeginBlock: + _, ok = res.Value.(*types.Response_BeginBlock) + case *types.Request_EndBlock: + _, ok = res.Value.(*types.Response_EndBlock) + } + return ok +} diff --git a/abci/client/socket_client_test.go b/abci/client/socket_client_test.go new file mode 100644 index 000000000..5a9187fb4 --- /dev/null +++ b/abci/client/socket_client_test.go @@ -0,0 +1,28 @@ +package abcicli_test + +import ( + "errors" + "testing" + "time" + + "github.com/tendermint/tendermint/abci/client" +) + +func TestSocketClientStopForErrorDeadlock(t *testing.T) { + c := abcicli.NewSocketClient(":80", false) + err := errors.New("foo-tendermint") + + // See Issue https://github.com/tendermint/abci/issues/114 + doneChan := make(chan bool) + go func() { + defer close(doneChan) + c.StopForError(err) + c.StopForError(err) + }() + + select { + case <-doneChan: + case <-time.After(time.Second * 4): + t.Fatalf("Test took too long, potential deadlock still exists") + } +} diff --git a/abci/cmd/abci-cli/abci-cli.go b/abci/cmd/abci-cli/abci-cli.go new file mode 100644 index 000000000..e20244011 --- /dev/null +++ b/abci/cmd/abci-cli/abci-cli.go @@ -0,0 +1,765 @@ +package main + +import ( + "bufio" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "strings" + + "github.com/spf13/cobra" + + cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/libs/log" + + abcicli "github.com/tendermint/tendermint/abci/client" + "github.com/tendermint/tendermint/abci/example/code" + "github.com/tendermint/tendermint/abci/example/counter" + "github.com/tendermint/tendermint/abci/example/kvstore" + "github.com/tendermint/tendermint/abci/server" + servertest "github.com/tendermint/tendermint/abci/tests/server" + "github.com/tendermint/tendermint/abci/types" + "github.com/tendermint/tendermint/abci/version" +) + +// client is a global variable so it can be reused by the console +var ( + client abcicli.Client + logger log.Logger +) + +// flags +var ( + // global + flagAddress string + flagAbci string + flagVerbose bool // for the println output + flagLogLevel string // for the logger + + // query + flagPath string + flagHeight int + flagProve bool + + // counter + flagSerial bool + + // kvstore + flagPersist string +) + +var RootCmd = &cobra.Command{ + Use: "abci-cli", + Short: "the ABCI CLI tool wraps an ABCI client", + Long: "the ABCI CLI tool wraps an ABCI client and is used for testing ABCI servers", + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + + switch cmd.Use { + case "counter", "kvstore", "dummy": // for the examples apps, don't pre-run + return nil + case "version": // skip running for version command + return nil + } + + if logger == nil { + allowLevel, err := log.AllowLevel(flagLogLevel) + if err != nil { + return err + } + logger = log.NewFilter(log.NewTMLogger(log.NewSyncWriter(os.Stdout)), allowLevel) + } + if client == nil { + var err error + client, err = abcicli.NewClient(flagAddress, flagAbci, false) + if err != nil { + return err + } + client.SetLogger(logger.With("module", "abci-client")) + if err := client.Start(); err != nil { + return err + } + } + return nil + }, +} + +// Structure for data passed to print response. +type response struct { + // generic abci response + Data []byte + Code uint32 + Info string + Log string + + Query *queryResponse +} + +type queryResponse struct { + Key []byte + Value []byte + Height int64 + Proof []byte +} + +func Execute() error { + addGlobalFlags() + addCommands() + return RootCmd.Execute() +} + +func addGlobalFlags() { + RootCmd.PersistentFlags().StringVarP(&flagAddress, "address", "", "tcp://0.0.0.0:26658", "address of application socket") + RootCmd.PersistentFlags().StringVarP(&flagAbci, "abci", "", "socket", "either socket or grpc") + RootCmd.PersistentFlags().BoolVarP(&flagVerbose, "verbose", "v", false, "print the command and results as if it were a console session") + RootCmd.PersistentFlags().StringVarP(&flagLogLevel, "log_level", "", "debug", "set the logger level") +} + +func addQueryFlags() { + queryCmd.PersistentFlags().StringVarP(&flagPath, "path", "", "/store", "path to prefix query with") + queryCmd.PersistentFlags().IntVarP(&flagHeight, "height", "", 0, "height to query the blockchain at") + queryCmd.PersistentFlags().BoolVarP(&flagProve, "prove", "", false, "whether or not to return a merkle proof of the query result") +} + +func addCounterFlags() { + counterCmd.PersistentFlags().BoolVarP(&flagSerial, "serial", "", false, "enforce incrementing (serial) transactions") +} + +func addDummyFlags() { + dummyCmd.PersistentFlags().StringVarP(&flagPersist, "persist", "", "", "directory to use for a database") +} + +func addKVStoreFlags() { + kvstoreCmd.PersistentFlags().StringVarP(&flagPersist, "persist", "", "", "directory to use for a database") +} + +func addCommands() { + RootCmd.AddCommand(batchCmd) + RootCmd.AddCommand(consoleCmd) + RootCmd.AddCommand(echoCmd) + RootCmd.AddCommand(infoCmd) + RootCmd.AddCommand(setOptionCmd) + RootCmd.AddCommand(deliverTxCmd) + RootCmd.AddCommand(checkTxCmd) + RootCmd.AddCommand(commitCmd) + RootCmd.AddCommand(versionCmd) + RootCmd.AddCommand(testCmd) + addQueryFlags() + RootCmd.AddCommand(queryCmd) + + // examples + addCounterFlags() + RootCmd.AddCommand(counterCmd) + // deprecated, left for backwards compatibility + addDummyFlags() + RootCmd.AddCommand(dummyCmd) + // replaces dummy, see issue #196 + addKVStoreFlags() + RootCmd.AddCommand(kvstoreCmd) +} + +var batchCmd = &cobra.Command{ + Use: "batch", + Short: "run a batch of abci commands against an application", + Long: `run a batch of abci commands against an application + +This command is run by piping in a file containing a series of commands +you'd like to run: + + abci-cli batch < example.file + +where example.file looks something like: + + set_option serial on + check_tx 0x00 + check_tx 0xff + deliver_tx 0x00 + check_tx 0x00 + deliver_tx 0x01 + deliver_tx 0x04 + info +`, + Args: cobra.ExactArgs(0), + RunE: func(cmd *cobra.Command, args []string) error { + return cmdBatch(cmd, args) + }, +} + +var consoleCmd = &cobra.Command{ + Use: "console", + Short: "start an interactive ABCI console for multiple commands", + Long: `start an interactive ABCI console for multiple commands + +This command opens an interactive console for running any of the other commands +without opening a new connection each time +`, + Args: cobra.ExactArgs(0), + ValidArgs: []string{"echo", "info", "set_option", "deliver_tx", "check_tx", "commit", "query"}, + RunE: func(cmd *cobra.Command, args []string) error { + return cmdConsole(cmd, args) + }, +} + +var echoCmd = &cobra.Command{ + Use: "echo", + Short: "have the application echo a message", + Long: "have the application echo a message", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return cmdEcho(cmd, args) + }, +} +var infoCmd = &cobra.Command{ + Use: "info", + Short: "get some info about the application", + Long: "get some info about the application", + Args: cobra.ExactArgs(0), + RunE: func(cmd *cobra.Command, args []string) error { + return cmdInfo(cmd, args) + }, +} +var setOptionCmd = &cobra.Command{ + Use: "set_option", + Short: "set an option on the application", + Long: "set an option on the application", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + return cmdSetOption(cmd, args) + }, +} + +var deliverTxCmd = &cobra.Command{ + Use: "deliver_tx", + Short: "deliver a new transaction to the application", + Long: "deliver a new transaction to the application", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return cmdDeliverTx(cmd, args) + }, +} + +var checkTxCmd = &cobra.Command{ + Use: "check_tx", + Short: "validate a transaction", + Long: "validate a transaction", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return cmdCheckTx(cmd, args) + }, +} + +var commitCmd = &cobra.Command{ + Use: "commit", + Short: "commit the application state and return the Merkle root hash", + Long: "commit the application state and return the Merkle root hash", + Args: cobra.ExactArgs(0), + RunE: func(cmd *cobra.Command, args []string) error { + return cmdCommit(cmd, args) + }, +} + +var versionCmd = &cobra.Command{ + Use: "version", + Short: "print ABCI console version", + Long: "print ABCI console version", + Args: cobra.ExactArgs(0), + RunE: func(cmd *cobra.Command, args []string) error { + fmt.Println(version.Version) + return nil + }, +} + +var queryCmd = &cobra.Command{ + Use: "query", + Short: "query the application state", + Long: "query the application state", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return cmdQuery(cmd, args) + }, +} + +var counterCmd = &cobra.Command{ + Use: "counter", + Short: "ABCI demo example", + Long: "ABCI demo example", + Args: cobra.ExactArgs(0), + RunE: func(cmd *cobra.Command, args []string) error { + return cmdCounter(cmd, args) + }, +} + +// deprecated, left for backwards compatibility +var dummyCmd = &cobra.Command{ + Use: "dummy", + Deprecated: "use: [abci-cli kvstore] instead", + Short: "ABCI demo example", + Long: "ABCI demo example", + Args: cobra.ExactArgs(0), + RunE: func(cmd *cobra.Command, args []string) error { + return cmdKVStore(cmd, args) + }, +} + +var kvstoreCmd = &cobra.Command{ + Use: "kvstore", + Short: "ABCI demo example", + Long: "ABCI demo example", + Args: cobra.ExactArgs(0), + RunE: func(cmd *cobra.Command, args []string) error { + return cmdKVStore(cmd, args) + }, +} + +var testCmd = &cobra.Command{ + Use: "test", + Short: "run integration tests", + Long: "run integration tests", + Args: cobra.ExactArgs(0), + RunE: func(cmd *cobra.Command, args []string) error { + return cmdTest(cmd, args) + }, +} + +// Generates new Args array based off of previous call args to maintain flag persistence +func persistentArgs(line []byte) []string { + + // generate the arguments to run from original os.Args + // to maintain flag arguments + args := os.Args + args = args[:len(args)-1] // remove the previous command argument + + if len(line) > 0 { // prevents introduction of extra space leading to argument parse errors + args = append(args, strings.Split(string(line), " ")...) + } + return args +} + +//-------------------------------------------------------------------------------- + +func compose(fs []func() error) error { + if len(fs) == 0 { + return nil + } else { + err := fs[0]() + if err == nil { + return compose(fs[1:]) + } else { + return err + } + } +} + +func cmdTest(cmd *cobra.Command, args []string) error { + return compose( + []func() error{ + func() error { return servertest.InitChain(client) }, + func() error { return servertest.SetOption(client, "serial", "on") }, + func() error { return servertest.Commit(client, nil) }, + func() error { return servertest.DeliverTx(client, []byte("abc"), code.CodeTypeBadNonce, nil) }, + func() error { return servertest.Commit(client, nil) }, + func() error { return servertest.DeliverTx(client, []byte{0x00}, code.CodeTypeOK, nil) }, + func() error { return servertest.Commit(client, []byte{0, 0, 0, 0, 0, 0, 0, 1}) }, + func() error { return servertest.DeliverTx(client, []byte{0x00}, code.CodeTypeBadNonce, nil) }, + func() error { return servertest.DeliverTx(client, []byte{0x01}, code.CodeTypeOK, nil) }, + func() error { return servertest.DeliverTx(client, []byte{0x00, 0x02}, code.CodeTypeOK, nil) }, + func() error { return servertest.DeliverTx(client, []byte{0x00, 0x03}, code.CodeTypeOK, nil) }, + func() error { return servertest.DeliverTx(client, []byte{0x00, 0x00, 0x04}, code.CodeTypeOK, nil) }, + func() error { + return servertest.DeliverTx(client, []byte{0x00, 0x00, 0x06}, code.CodeTypeBadNonce, nil) + }, + func() error { return servertest.Commit(client, []byte{0, 0, 0, 0, 0, 0, 0, 5}) }, + }) +} + +func cmdBatch(cmd *cobra.Command, args []string) error { + bufReader := bufio.NewReader(os.Stdin) + for { + + line, more, err := bufReader.ReadLine() + if more { + return errors.New("Input line is too long") + } else if err == io.EOF { + break + } else if len(line) == 0 { + continue + } else if err != nil { + return err + } + + cmdArgs := persistentArgs(line) + if err := muxOnCommands(cmd, cmdArgs); err != nil { + return err + } + fmt.Println() + } + return nil +} + +func cmdConsole(cmd *cobra.Command, args []string) error { + for { + fmt.Printf("> ") + bufReader := bufio.NewReader(os.Stdin) + line, more, err := bufReader.ReadLine() + if more { + return errors.New("Input is too long") + } else if err != nil { + return err + } + + pArgs := persistentArgs(line) + if err := muxOnCommands(cmd, pArgs); err != nil { + return err + } + } + return nil +} + +func muxOnCommands(cmd *cobra.Command, pArgs []string) error { + if len(pArgs) < 2 { + return errors.New("expecting persistent args of the form: abci-cli [command] <...>") + } + + // TODO: this parsing is fragile + args := []string{} + for i := 0; i < len(pArgs); i++ { + arg := pArgs[i] + + // check for flags + if strings.HasPrefix(arg, "-") { + // if it has an equal, we can just skip + if strings.Contains(arg, "=") { + continue + } + // if its a boolean, we can just skip + _, err := cmd.Flags().GetBool(strings.TrimLeft(arg, "-")) + if err == nil { + continue + } + + // otherwise, we need to skip the next one too + i += 1 + continue + } + + // append the actual arg + args = append(args, arg) + } + var subCommand string + var actualArgs []string + if len(args) > 1 { + subCommand = args[1] + } + if len(args) > 2 { + actualArgs = args[2:] + } + cmd.Use = subCommand // for later print statements ... + + switch strings.ToLower(subCommand) { + case "check_tx": + return cmdCheckTx(cmd, actualArgs) + case "commit": + return cmdCommit(cmd, actualArgs) + case "deliver_tx": + return cmdDeliverTx(cmd, actualArgs) + case "echo": + return cmdEcho(cmd, actualArgs) + case "info": + return cmdInfo(cmd, actualArgs) + case "query": + return cmdQuery(cmd, actualArgs) + case "set_option": + return cmdSetOption(cmd, actualArgs) + default: + return cmdUnimplemented(cmd, pArgs) + } +} + +func cmdUnimplemented(cmd *cobra.Command, args []string) error { + // TODO: Print out all the sub-commands available + msg := "unimplemented command" + if err := cmd.Help(); err != nil { + msg = err.Error() + } + if len(args) > 0 { + msg += fmt.Sprintf(" args: [%s]", strings.Join(args, " ")) + } + printResponse(cmd, args, response{ + Code: codeBad, + Log: msg, + }) + return nil +} + +// Have the application echo a message +func cmdEcho(cmd *cobra.Command, args []string) error { + msg := "" + if len(args) > 0 { + msg = args[0] + } + res, err := client.EchoSync(msg) + if err != nil { + return err + } + printResponse(cmd, args, response{ + Data: []byte(res.Message), + }) + return nil +} + +// Get some info from the application +func cmdInfo(cmd *cobra.Command, args []string) error { + var version string + if len(args) == 1 { + version = args[0] + } + res, err := client.InfoSync(types.RequestInfo{version}) + if err != nil { + return err + } + printResponse(cmd, args, response{ + Data: []byte(res.Data), + }) + return nil +} + +const codeBad uint32 = 10 + +// Set an option on the application +func cmdSetOption(cmd *cobra.Command, args []string) error { + if len(args) < 2 { + printResponse(cmd, args, response{ + Code: codeBad, + Log: "want at least arguments of the form: ", + }) + return nil + } + + key, val := args[0], args[1] + _, err := client.SetOptionSync(types.RequestSetOption{key, val}) + if err != nil { + return err + } + printResponse(cmd, args, response{Log: "OK (SetOption doesn't return anything.)"}) // NOTE: Nothing to show... + return nil +} + +// Append a new tx to application +func cmdDeliverTx(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + printResponse(cmd, args, response{ + Code: codeBad, + Log: "want the tx", + }) + return nil + } + txBytes, err := stringOrHexToBytes(args[0]) + if err != nil { + return err + } + res, err := client.DeliverTxSync(txBytes) + if err != nil { + return err + } + printResponse(cmd, args, response{ + Code: res.Code, + Data: res.Data, + Info: res.Info, + Log: res.Log, + }) + return nil +} + +// Validate a tx +func cmdCheckTx(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + printResponse(cmd, args, response{ + Code: codeBad, + Info: "want the tx", + }) + return nil + } + txBytes, err := stringOrHexToBytes(args[0]) + if err != nil { + return err + } + res, err := client.CheckTxSync(txBytes) + if err != nil { + return err + } + printResponse(cmd, args, response{ + Code: res.Code, + Data: res.Data, + Info: res.Info, + Log: res.Log, + }) + return nil +} + +// Get application Merkle root hash +func cmdCommit(cmd *cobra.Command, args []string) error { + res, err := client.CommitSync() + if err != nil { + return err + } + printResponse(cmd, args, response{ + Data: res.Data, + }) + return nil +} + +// Query application state +func cmdQuery(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + printResponse(cmd, args, response{ + Code: codeBad, + Info: "want the query", + Log: "", + }) + return nil + } + queryBytes, err := stringOrHexToBytes(args[0]) + if err != nil { + return err + } + + resQuery, err := client.QuerySync(types.RequestQuery{ + Data: queryBytes, + Path: flagPath, + Height: int64(flagHeight), + Prove: flagProve, + }) + if err != nil { + return err + } + printResponse(cmd, args, response{ + Code: resQuery.Code, + Info: resQuery.Info, + Log: resQuery.Log, + Query: &queryResponse{ + Key: resQuery.Key, + Value: resQuery.Value, + Height: resQuery.Height, + Proof: resQuery.Proof, + }, + }) + return nil +} + +func cmdCounter(cmd *cobra.Command, args []string) error { + + app := counter.NewCounterApplication(flagSerial) + + logger := log.NewTMLogger(log.NewSyncWriter(os.Stdout)) + + // Start the listener + srv, err := server.NewServer(flagAddress, flagAbci, app) + if err != nil { + return err + } + srv.SetLogger(logger.With("module", "abci-server")) + if err := srv.Start(); err != nil { + return err + } + + // Wait forever + cmn.TrapSignal(func() { + // Cleanup + srv.Stop() + }) + return nil +} + +func cmdKVStore(cmd *cobra.Command, args []string) error { + logger := log.NewTMLogger(log.NewSyncWriter(os.Stdout)) + + // Create the application - in memory or persisted to disk + var app types.Application + if flagPersist == "" { + app = kvstore.NewKVStoreApplication() + } else { + app = kvstore.NewPersistentKVStoreApplication(flagPersist) + app.(*kvstore.PersistentKVStoreApplication).SetLogger(logger.With("module", "kvstore")) + } + + // Start the listener + srv, err := server.NewServer(flagAddress, flagAbci, app) + if err != nil { + return err + } + srv.SetLogger(logger.With("module", "abci-server")) + if err := srv.Start(); err != nil { + return err + } + + // Wait forever + cmn.TrapSignal(func() { + // Cleanup + srv.Stop() + }) + return nil +} + +//-------------------------------------------------------------------------------- + +func printResponse(cmd *cobra.Command, args []string, rsp response) { + + if flagVerbose { + fmt.Println(">", cmd.Use, strings.Join(args, " ")) + } + + // Always print the status code. + if rsp.Code == types.CodeTypeOK { + fmt.Printf("-> code: OK\n") + } else { + fmt.Printf("-> code: %d\n", rsp.Code) + + } + + if len(rsp.Data) != 0 { + // Do no print this line when using the commit command + // because the string comes out as gibberish + if cmd.Use != "commit" { + fmt.Printf("-> data: %s\n", rsp.Data) + } + fmt.Printf("-> data.hex: 0x%X\n", rsp.Data) + } + if rsp.Log != "" { + fmt.Printf("-> log: %s\n", rsp.Log) + } + + if rsp.Query != nil { + fmt.Printf("-> height: %d\n", rsp.Query.Height) + if rsp.Query.Key != nil { + fmt.Printf("-> key: %s\n", rsp.Query.Key) + fmt.Printf("-> key.hex: %X\n", rsp.Query.Key) + } + if rsp.Query.Value != nil { + fmt.Printf("-> value: %s\n", rsp.Query.Value) + fmt.Printf("-> value.hex: %X\n", rsp.Query.Value) + } + if rsp.Query.Proof != nil { + fmt.Printf("-> proof: %X\n", rsp.Query.Proof) + } + } +} + +// NOTE: s is interpreted as a string unless prefixed with 0x +func stringOrHexToBytes(s string) ([]byte, error) { + if len(s) > 2 && strings.ToLower(s[:2]) == "0x" { + b, err := hex.DecodeString(s[2:]) + if err != nil { + err = fmt.Errorf("Error decoding hex argument: %s", err.Error()) + return nil, err + } + return b, nil + } + + if !strings.HasPrefix(s, "\"") || !strings.HasSuffix(s, "\"") { + err := fmt.Errorf("Invalid string arg: \"%s\". Must be quoted or a \"0x\"-prefixed hex string", s) + return nil, err + } + + return []byte(s[1 : len(s)-1]), nil +} diff --git a/abci/cmd/abci-cli/main.go b/abci/cmd/abci-cli/main.go new file mode 100644 index 000000000..a927e7ed8 --- /dev/null +++ b/abci/cmd/abci-cli/main.go @@ -0,0 +1,14 @@ +package main + +import ( + "fmt" + "os" +) + +func main() { + err := Execute() + if err != nil { + fmt.Print(err) + os.Exit(1) + } +} diff --git a/abci/example/code/code.go b/abci/example/code/code.go new file mode 100644 index 000000000..94e9d015e --- /dev/null +++ b/abci/example/code/code.go @@ -0,0 +1,9 @@ +package code + +// Return codes for the examples +const ( + CodeTypeOK uint32 = 0 + CodeTypeEncodingError uint32 = 1 + CodeTypeBadNonce uint32 = 2 + CodeTypeUnauthorized uint32 = 3 +) diff --git a/abci/example/counter/counter.go b/abci/example/counter/counter.go new file mode 100644 index 000000000..857e82baf --- /dev/null +++ b/abci/example/counter/counter.go @@ -0,0 +1,104 @@ +package counter + +import ( + "encoding/binary" + "fmt" + + "github.com/tendermint/tendermint/abci/example/code" + "github.com/tendermint/tendermint/abci/types" + cmn "github.com/tendermint/tendermint/libs/common" +) + +type CounterApplication struct { + types.BaseApplication + + hashCount int + txCount int + serial bool +} + +func NewCounterApplication(serial bool) *CounterApplication { + return &CounterApplication{serial: serial} +} + +func (app *CounterApplication) Info(req types.RequestInfo) types.ResponseInfo { + return types.ResponseInfo{Data: cmn.Fmt("{\"hashes\":%v,\"txs\":%v}", app.hashCount, app.txCount)} +} + +func (app *CounterApplication) SetOption(req types.RequestSetOption) types.ResponseSetOption { + key, value := req.Key, req.Value + if key == "serial" && value == "on" { + app.serial = true + } else { + /* + TODO Panic and have the ABCI server pass an exception. + The client can call SetOptionSync() and get an `error`. + return types.ResponseSetOption{ + Error: cmn.Fmt("Unknown key (%s) or value (%s)", key, value), + } + */ + return types.ResponseSetOption{} + } + + return types.ResponseSetOption{} +} + +func (app *CounterApplication) DeliverTx(tx []byte) types.ResponseDeliverTx { + if app.serial { + if len(tx) > 8 { + return types.ResponseDeliverTx{ + Code: code.CodeTypeEncodingError, + Log: fmt.Sprintf("Max tx size is 8 bytes, got %d", len(tx))} + } + tx8 := make([]byte, 8) + copy(tx8[len(tx8)-len(tx):], tx) + txValue := binary.BigEndian.Uint64(tx8) + if txValue != uint64(app.txCount) { + return types.ResponseDeliverTx{ + Code: code.CodeTypeBadNonce, + Log: fmt.Sprintf("Invalid nonce. Expected %v, got %v", app.txCount, txValue)} + } + } + app.txCount++ + return types.ResponseDeliverTx{Code: code.CodeTypeOK} +} + +func (app *CounterApplication) CheckTx(tx []byte) types.ResponseCheckTx { + if app.serial { + if len(tx) > 8 { + return types.ResponseCheckTx{ + Code: code.CodeTypeEncodingError, + Log: fmt.Sprintf("Max tx size is 8 bytes, got %d", len(tx))} + } + tx8 := make([]byte, 8) + copy(tx8[len(tx8)-len(tx):], tx) + txValue := binary.BigEndian.Uint64(tx8) + if txValue < uint64(app.txCount) { + return types.ResponseCheckTx{ + Code: code.CodeTypeBadNonce, + Log: fmt.Sprintf("Invalid nonce. Expected >= %v, got %v", app.txCount, txValue)} + } + } + return types.ResponseCheckTx{Code: code.CodeTypeOK} +} + +func (app *CounterApplication) Commit() (resp types.ResponseCommit) { + app.hashCount++ + if app.txCount == 0 { + return types.ResponseCommit{} + } + hash := make([]byte, 8) + binary.BigEndian.PutUint64(hash, uint64(app.txCount)) + return types.ResponseCommit{Data: hash} +} + +func (app *CounterApplication) Query(reqQuery types.RequestQuery) types.ResponseQuery { + switch reqQuery.Path { + case "hash": + return types.ResponseQuery{Value: []byte(cmn.Fmt("%v", app.hashCount))} + case "tx": + return types.ResponseQuery{Value: []byte(cmn.Fmt("%v", app.txCount))} + default: + return types.ResponseQuery{Log: cmn.Fmt("Invalid query path. Expected hash or tx, got %v", reqQuery.Path)} + } +} diff --git a/abci/example/example.go b/abci/example/example.go new file mode 100644 index 000000000..ee491c1b5 --- /dev/null +++ b/abci/example/example.go @@ -0,0 +1,3 @@ +package example + +// so the go tool doesn't return errors about no buildable go files ... diff --git a/abci/example/example_test.go b/abci/example/example_test.go new file mode 100644 index 000000000..bbb53b5af --- /dev/null +++ b/abci/example/example_test.go @@ -0,0 +1,154 @@ +package example + +import ( + "fmt" + "net" + "reflect" + "testing" + "time" + + "google.golang.org/grpc" + + "golang.org/x/net/context" + + cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/libs/log" + + abcicli "github.com/tendermint/tendermint/abci/client" + "github.com/tendermint/tendermint/abci/example/code" + "github.com/tendermint/tendermint/abci/example/kvstore" + abciserver "github.com/tendermint/tendermint/abci/server" + "github.com/tendermint/tendermint/abci/types" +) + +func TestKVStore(t *testing.T) { + fmt.Println("### Testing KVStore") + testStream(t, kvstore.NewKVStoreApplication()) +} + +func TestBaseApp(t *testing.T) { + fmt.Println("### Testing BaseApp") + testStream(t, types.NewBaseApplication()) +} + +func TestGRPC(t *testing.T) { + fmt.Println("### Testing GRPC") + testGRPCSync(t, types.NewGRPCApplication(types.NewBaseApplication())) +} + +func testStream(t *testing.T, app types.Application) { + numDeliverTxs := 200000 + + // Start the listener + server := abciserver.NewSocketServer("unix://test.sock", app) + server.SetLogger(log.TestingLogger().With("module", "abci-server")) + if err := server.Start(); err != nil { + t.Fatalf("Error starting socket server: %v", err.Error()) + } + defer server.Stop() + + // Connect to the socket + client := abcicli.NewSocketClient("unix://test.sock", false) + client.SetLogger(log.TestingLogger().With("module", "abci-client")) + if err := client.Start(); err != nil { + t.Fatalf("Error starting socket client: %v", err.Error()) + } + defer client.Stop() + + done := make(chan struct{}) + counter := 0 + client.SetResponseCallback(func(req *types.Request, res *types.Response) { + // Process response + switch r := res.Value.(type) { + case *types.Response_DeliverTx: + counter++ + if r.DeliverTx.Code != code.CodeTypeOK { + t.Error("DeliverTx failed with ret_code", r.DeliverTx.Code) + } + if counter > numDeliverTxs { + t.Fatalf("Too many DeliverTx responses. Got %d, expected %d", counter, numDeliverTxs) + } + if counter == numDeliverTxs { + go func() { + time.Sleep(time.Second * 2) // Wait for a bit to allow counter overflow + close(done) + }() + return + } + case *types.Response_Flush: + // ignore + default: + t.Error("Unexpected response type", reflect.TypeOf(res.Value)) + } + }) + + // Write requests + for counter := 0; counter < numDeliverTxs; counter++ { + // Send request + reqRes := client.DeliverTxAsync([]byte("test")) + _ = reqRes + // check err ? + + // Sometimes send flush messages + if counter%123 == 0 { + client.FlushAsync() + // check err ? + } + } + + // Send final flush message + client.FlushAsync() + + <-done +} + +//------------------------- +// test grpc + +func dialerFunc(addr string, timeout time.Duration) (net.Conn, error) { + return cmn.Connect(addr) +} + +func testGRPCSync(t *testing.T, app *types.GRPCApplication) { + numDeliverTxs := 2000 + + // Start the listener + server := abciserver.NewGRPCServer("unix://test.sock", app) + server.SetLogger(log.TestingLogger().With("module", "abci-server")) + if err := server.Start(); err != nil { + t.Fatalf("Error starting GRPC server: %v", err.Error()) + } + defer server.Stop() + + // Connect to the socket + conn, err := grpc.Dial("unix://test.sock", grpc.WithInsecure(), grpc.WithDialer(dialerFunc)) + if err != nil { + t.Fatalf("Error dialing GRPC server: %v", err.Error()) + } + defer conn.Close() + + client := types.NewABCIApplicationClient(conn) + + // Write requests + for counter := 0; counter < numDeliverTxs; counter++ { + // Send request + response, err := client.DeliverTx(context.Background(), &types.RequestDeliverTx{[]byte("test")}) + if err != nil { + t.Fatalf("Error in GRPC DeliverTx: %v", err.Error()) + } + counter++ + if response.Code != code.CodeTypeOK { + t.Error("DeliverTx failed with ret_code", response.Code) + } + if counter > numDeliverTxs { + t.Fatal("Too many DeliverTx responses") + } + t.Log("response", counter) + if counter == numDeliverTxs { + go func() { + time.Sleep(time.Second * 2) // Wait for a bit to allow counter overflow + }() + } + + } +} diff --git a/abci/example/js/.gitignore b/abci/example/js/.gitignore new file mode 100644 index 000000000..3c3629e64 --- /dev/null +++ b/abci/example/js/.gitignore @@ -0,0 +1 @@ +node_modules diff --git a/abci/example/js/README.md b/abci/example/js/README.md new file mode 100644 index 000000000..1bef9cbf5 --- /dev/null +++ b/abci/example/js/README.md @@ -0,0 +1 @@ +This example has been moved here: https://github.com/tendermint/js-abci/tree/master/example diff --git a/abci/example/kvstore/README.md b/abci/example/kvstore/README.md new file mode 100644 index 000000000..e988eadb0 --- /dev/null +++ b/abci/example/kvstore/README.md @@ -0,0 +1,31 @@ +# KVStore + +There are two app's here: the KVStoreApplication and the PersistentKVStoreApplication. + +## KVStoreApplication + +The KVStoreApplication is a simple merkle key-value store. +Transactions of the form `key=value` are stored as key-value pairs in the tree. +Transactions without an `=` sign set the value to the key. +The app has no replay protection (other than what the mempool provides). + +## PersistentKVStoreApplication + +The PersistentKVStoreApplication wraps the KVStoreApplication +and provides two additional features: + +1) persistence of state across app restarts (using Tendermint's ABCI-Handshake mechanism) +2) validator set changes + +The state is persisted in leveldb along with the last block committed, +and the Handshake allows any necessary blocks to be replayed. +Validator set changes are effected using the following transaction format: + +``` +val:pubkey1/power1,addr2/power2,addr3/power3" +``` + +where `power1` is the new voting power for the validator with `pubkey1` (possibly a new one). +There is no sybil protection against new validators joining. +Validators can be removed by setting their power to `0`. + diff --git a/abci/example/kvstore/helpers.go b/abci/example/kvstore/helpers.go new file mode 100644 index 000000000..0e69fab9f --- /dev/null +++ b/abci/example/kvstore/helpers.go @@ -0,0 +1,38 @@ +package kvstore + +import ( + "github.com/tendermint/tendermint/abci/types" + cmn "github.com/tendermint/tendermint/libs/common" +) + +// RandVal creates one random validator, with a key derived +// from the input value +func RandVal(i int) types.Validator { + addr := cmn.RandBytes(20) + pubkey := cmn.RandBytes(32) + power := cmn.RandUint16() + 1 + v := types.Ed25519Validator(pubkey, int64(power)) + v.Address = addr + return v +} + +// RandVals returns a list of cnt validators for initializing +// the application. Note that the keys are deterministically +// derived from the index in the array, while the power is +// random (Change this if not desired) +func RandVals(cnt int) []types.Validator { + res := make([]types.Validator, cnt) + for i := 0; i < cnt; i++ { + res[i] = RandVal(i) + } + return res +} + +// InitKVStore initializes the kvstore app with some data, +// which allows tests to pass and is fine as long as you +// don't make any tx that modify the validator state +func InitKVStore(app *PersistentKVStoreApplication) { + app.InitChain(types.RequestInitChain{ + Validators: RandVals(1), + }) +} diff --git a/abci/example/kvstore/kvstore.go b/abci/example/kvstore/kvstore.go new file mode 100644 index 000000000..0f72b44ea --- /dev/null +++ b/abci/example/kvstore/kvstore.go @@ -0,0 +1,126 @@ +package kvstore + +import ( + "bytes" + "encoding/binary" + "encoding/json" + "fmt" + + "github.com/tendermint/tendermint/abci/example/code" + "github.com/tendermint/tendermint/abci/types" + cmn "github.com/tendermint/tendermint/libs/common" + dbm "github.com/tendermint/tendermint/libs/db" +) + +var ( + stateKey = []byte("stateKey") + kvPairPrefixKey = []byte("kvPairKey:") +) + +type State struct { + db dbm.DB + Size int64 `json:"size"` + Height int64 `json:"height"` + AppHash []byte `json:"app_hash"` +} + +func loadState(db dbm.DB) State { + stateBytes := db.Get(stateKey) + var state State + if len(stateBytes) != 0 { + err := json.Unmarshal(stateBytes, &state) + if err != nil { + panic(err) + } + } + state.db = db + return state +} + +func saveState(state State) { + stateBytes, err := json.Marshal(state) + if err != nil { + panic(err) + } + state.db.Set(stateKey, stateBytes) +} + +func prefixKey(key []byte) []byte { + return append(kvPairPrefixKey, key...) +} + +//--------------------------------------------------- + +var _ types.Application = (*KVStoreApplication)(nil) + +type KVStoreApplication struct { + types.BaseApplication + + state State +} + +func NewKVStoreApplication() *KVStoreApplication { + state := loadState(dbm.NewMemDB()) + return &KVStoreApplication{state: state} +} + +func (app *KVStoreApplication) Info(req types.RequestInfo) (resInfo types.ResponseInfo) { + return types.ResponseInfo{Data: fmt.Sprintf("{\"size\":%v}", app.state.Size)} +} + +// tx is either "key=value" or just arbitrary bytes +func (app *KVStoreApplication) DeliverTx(tx []byte) types.ResponseDeliverTx { + var key, value []byte + parts := bytes.Split(tx, []byte("=")) + if len(parts) == 2 { + key, value = parts[0], parts[1] + } else { + key, value = tx, tx + } + app.state.db.Set(prefixKey(key), value) + app.state.Size += 1 + + tags := []cmn.KVPair{ + {[]byte("app.creator"), []byte("jae")}, + {[]byte("app.key"), key}, + } + return types.ResponseDeliverTx{Code: code.CodeTypeOK, Tags: tags} +} + +func (app *KVStoreApplication) CheckTx(tx []byte) types.ResponseCheckTx { + return types.ResponseCheckTx{Code: code.CodeTypeOK} +} + +func (app *KVStoreApplication) Commit() types.ResponseCommit { + // Using a memdb - just return the big endian size of the db + appHash := make([]byte, 8) + binary.PutVarint(appHash, app.state.Size) + app.state.AppHash = appHash + app.state.Height += 1 + saveState(app.state) + return types.ResponseCommit{Data: appHash} +} + +func (app *KVStoreApplication) Query(reqQuery types.RequestQuery) (resQuery types.ResponseQuery) { + if reqQuery.Prove { + value := app.state.db.Get(prefixKey(reqQuery.Data)) + resQuery.Index = -1 // TODO make Proof return index + resQuery.Key = reqQuery.Data + resQuery.Value = value + if value != nil { + resQuery.Log = "exists" + } else { + resQuery.Log = "does not exist" + } + return + } else { + value := app.state.db.Get(prefixKey(reqQuery.Data)) + resQuery.Value = value + if value != nil { + resQuery.Log = "exists" + } else { + resQuery.Log = "does not exist" + } + return + } +} diff --git a/abci/example/kvstore/kvstore_test.go b/abci/example/kvstore/kvstore_test.go new file mode 100644 index 000000000..2d8f81272 --- /dev/null +++ b/abci/example/kvstore/kvstore_test.go @@ -0,0 +1,310 @@ +package kvstore + +import ( + "bytes" + "io/ioutil" + "sort" + "testing" + + "github.com/stretchr/testify/require" + + cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/libs/log" + + abcicli "github.com/tendermint/tendermint/abci/client" + "github.com/tendermint/tendermint/abci/example/code" + abciserver "github.com/tendermint/tendermint/abci/server" + "github.com/tendermint/tendermint/abci/types" +) + +func testKVStore(t *testing.T, app types.Application, tx []byte, key, value string) { + ar := app.DeliverTx(tx) + require.False(t, ar.IsErr(), ar) + // repeating tx doesn't raise error + ar = app.DeliverTx(tx) + require.False(t, ar.IsErr(), ar) + + // make sure query is fine + resQuery := app.Query(types.RequestQuery{ + Path: "/store", + Data: []byte(key), + }) + require.Equal(t, code.CodeTypeOK, resQuery.Code) + require.Equal(t, value, string(resQuery.Value)) + + // make sure proof is fine + resQuery = app.Query(types.RequestQuery{ + Path: "/store", + Data: []byte(key), + Prove: true, + }) + require.EqualValues(t, code.CodeTypeOK, resQuery.Code) + require.Equal(t, value, string(resQuery.Value)) +} + +func TestKVStoreKV(t *testing.T) { + kvstore := NewKVStoreApplication() + key := "abc" + value := key + tx := []byte(key) + testKVStore(t, kvstore, tx, key, value) + + value = "def" + tx = []byte(key + "=" + value) + testKVStore(t, kvstore, tx, key, value) +} + +func TestPersistentKVStoreKV(t *testing.T) { + dir, err := ioutil.TempDir("/tmp", "abci-kvstore-test") // TODO + if err != nil { + t.Fatal(err) + } + kvstore := NewPersistentKVStoreApplication(dir) + key := "abc" + value := key + tx := []byte(key) + testKVStore(t, kvstore, tx, key, value) + + value = "def" + tx = []byte(key + "=" + value) + testKVStore(t, kvstore, tx, key, value) +} + +func TestPersistentKVStoreInfo(t *testing.T) { + dir, err := ioutil.TempDir("/tmp", "abci-kvstore-test") // TODO + if err != nil { + t.Fatal(err) + } + kvstore := NewPersistentKVStoreApplication(dir) + InitKVStore(kvstore) + height := int64(0) + + resInfo := kvstore.Info(types.RequestInfo{}) + if resInfo.LastBlockHeight != height { + t.Fatalf("expected height of %d, got %d", height, resInfo.LastBlockHeight) + } + + // make and apply block + height = int64(1) + hash := []byte("foo") + header := types.Header{ + Height: int64(height), + } + kvstore.BeginBlock(types.RequestBeginBlock{hash, header, nil, nil}) + kvstore.EndBlock(types.RequestEndBlock{header.Height}) + kvstore.Commit() + + resInfo = kvstore.Info(types.RequestInfo{}) + if resInfo.LastBlockHeight != height { + t.Fatalf("expected height of %d, got %d", height, resInfo.LastBlockHeight) + } + +} + +// add a validator, remove a validator, update a validator +func TestValUpdates(t *testing.T) { + dir, err := ioutil.TempDir("/tmp", "abci-kvstore-test") // TODO + if err != nil { + t.Fatal(err) + } + kvstore := NewPersistentKVStoreApplication(dir) + + // init with some validators + total := 10 + nInit := 5 + vals := RandVals(total) + // iniitalize with the first nInit + kvstore.InitChain(types.RequestInitChain{ + Validators: vals[:nInit], + }) + + vals1, vals2 := vals[:nInit], kvstore.Validators() + valsEqual(t, vals1, vals2) + + var v1, v2, v3 types.Validator + + // add some validators + v1, v2 = vals[nInit], vals[nInit+1] + diff := []types.Validator{v1, v2} + tx1 := MakeValSetChangeTx(v1.PubKey, v1.Power) + tx2 := MakeValSetChangeTx(v2.PubKey, v2.Power) + + makeApplyBlock(t, kvstore, 1, diff, tx1, tx2) + + vals1, vals2 = vals[:nInit+2], kvstore.Validators() + valsEqual(t, vals1, vals2) + + // remove some validators + v1, v2, v3 = vals[nInit-2], vals[nInit-1], vals[nInit] + v1.Power = 0 + v2.Power = 0 + v3.Power = 0 + diff = []types.Validator{v1, v2, v3} + tx1 = MakeValSetChangeTx(v1.PubKey, v1.Power) + tx2 = MakeValSetChangeTx(v2.PubKey, v2.Power) + tx3 := MakeValSetChangeTx(v3.PubKey, v3.Power) + + makeApplyBlock(t, kvstore, 2, diff, tx1, tx2, tx3) + + vals1 = append(vals[:nInit-2], vals[nInit+1]) + vals2 = kvstore.Validators() + valsEqual(t, vals1, vals2) + + // update some validators + v1 = vals[0] + if v1.Power == 5 { + v1.Power = 6 + } else { + v1.Power = 5 + } + diff = []types.Validator{v1} + tx1 = MakeValSetChangeTx(v1.PubKey, v1.Power) + + makeApplyBlock(t, kvstore, 3, diff, tx1) + + vals1 = append([]types.Validator{v1}, vals1[1:]...) + vals2 = kvstore.Validators() + valsEqual(t, vals1, vals2) + +} + +func makeApplyBlock(t *testing.T, kvstore types.Application, heightInt int, diff []types.Validator, txs ...[]byte) { + // make and apply block + height := int64(heightInt) + hash := []byte("foo") + header := types.Header{ + Height: height, + } + + kvstore.BeginBlock(types.RequestBeginBlock{hash, header, nil, nil}) + for _, tx := range txs { + if r := kvstore.DeliverTx(tx); r.IsErr() { + t.Fatal(r) + } + } + resEndBlock := kvstore.EndBlock(types.RequestEndBlock{header.Height}) + kvstore.Commit() + + valsEqual(t, diff, resEndBlock.ValidatorUpdates) + +} + +// order doesn't matter +func valsEqual(t *testing.T, vals1, vals2 []types.Validator) { + if len(vals1) != len(vals2) { + t.Fatalf("vals dont match in len. got %d, expected %d", len(vals2), len(vals1)) + } + sort.Sort(types.Validators(vals1)) + sort.Sort(types.Validators(vals2)) + for i, v1 := range vals1 { + v2 := vals2[i] + if !bytes.Equal(v1.PubKey.Data, v2.PubKey.Data) || + v1.Power != v2.Power { + t.Fatalf("vals dont match at index %d. got %X/%d , expected %X/%d", i, v2.PubKey, v2.Power, v1.PubKey, v1.Power) + } + } +} + +func makeSocketClientServer(app types.Application, name string) (abcicli.Client, cmn.Service, error) { + // Start the listener + socket := cmn.Fmt("unix://%s.sock", name) + logger := log.TestingLogger() + + server := abciserver.NewSocketServer(socket, app) + server.SetLogger(logger.With("module", "abci-server")) + if err := server.Start(); err != nil { + return nil, nil, err + } + + // Connect to the socket + client := abcicli.NewSocketClient(socket, false) + client.SetLogger(logger.With("module", "abci-client")) + if err := client.Start(); err != nil { + server.Stop() + return nil, nil, err + } + + return client, server, nil +} + +func makeGRPCClientServer(app types.Application, name string) (abcicli.Client, cmn.Service, error) { + // Start the listener + socket := cmn.Fmt("unix://%s.sock", name) + logger := log.TestingLogger() + + gapp := types.NewGRPCApplication(app) + server := abciserver.NewGRPCServer(socket, gapp) + server.SetLogger(logger.With("module", "abci-server")) + if err := server.Start(); err != nil { + return nil, nil, err + } + + client := abcicli.NewGRPCClient(socket, true) + client.SetLogger(logger.With("module", "abci-client")) + if err := client.Start(); err != nil { + server.Stop() + return nil, nil, err + } + return client, server, nil +} + +func TestClientServer(t *testing.T) { + // set up socket app + kvstore := NewKVStoreApplication() + client, server, err := makeSocketClientServer(kvstore, "kvstore-socket") + require.Nil(t, err) + defer server.Stop() + defer client.Stop() + + runClientTests(t, client) + + // set up grpc app + kvstore = NewKVStoreApplication() + gclient, gserver, err := makeGRPCClientServer(kvstore, "kvstore-grpc") + require.Nil(t, err) + defer gserver.Stop() + defer gclient.Stop() + + runClientTests(t, gclient) +} + +func runClientTests(t *testing.T, client abcicli.Client) { + // run some tests.... + key := "abc" + value := key + tx := []byte(key) + testClient(t, client, tx, key, value) + + value = "def" + tx = []byte(key + "=" + value) + testClient(t, client, tx, key, value) +} + +func testClient(t *testing.T, app abcicli.Client, tx []byte, key, value string) { + ar, err := app.DeliverTxSync(tx) + require.NoError(t, err) + require.False(t, ar.IsErr(), ar) + // repeating tx doesn't raise error + ar, err = app.DeliverTxSync(tx) + require.NoError(t, err) + require.False(t, ar.IsErr(), ar) + + // make sure query is fine + resQuery, err := app.QuerySync(types.RequestQuery{ + Path: "/store", + Data: []byte(key), + }) + require.Nil(t, err) + require.Equal(t, code.CodeTypeOK, resQuery.Code) + require.Equal(t, value, string(resQuery.Value)) + + // make sure proof is fine + resQuery, err = app.QuerySync(types.RequestQuery{ + Path: "/store", + Data: []byte(key), + Prove: true, + }) + require.Nil(t, err) + require.Equal(t, code.CodeTypeOK, resQuery.Code) + require.Equal(t, value, string(resQuery.Value)) +} diff --git a/abci/example/kvstore/persistent_kvstore.go b/abci/example/kvstore/persistent_kvstore.go new file mode 100644 index 000000000..12ccbab78 --- /dev/null +++ b/abci/example/kvstore/persistent_kvstore.go @@ -0,0 +1,200 @@ +package kvstore + +import ( + "bytes" + "encoding/hex" + "fmt" + "strconv" + "strings" + + "github.com/tendermint/tendermint/abci/example/code" + "github.com/tendermint/tendermint/abci/types" + cmn "github.com/tendermint/tendermint/libs/common" + dbm "github.com/tendermint/tendermint/libs/db" + "github.com/tendermint/tendermint/libs/log" +) + +const ( + ValidatorSetChangePrefix string = "val:" +) + +//----------------------------------------- + +var _ types.Application = (*PersistentKVStoreApplication)(nil) + +type PersistentKVStoreApplication struct { + app *KVStoreApplication + + // validator set + ValUpdates []types.Validator + + logger log.Logger +} + +func NewPersistentKVStoreApplication(dbDir string) *PersistentKVStoreApplication { + name := "kvstore" + db, err := dbm.NewGoLevelDB(name, dbDir) + if err != nil { + panic(err) + } + + state := loadState(db) + + return &PersistentKVStoreApplication{ + app: &KVStoreApplication{state: state}, + logger: log.NewNopLogger(), + } +} + +func (app *PersistentKVStoreApplication) SetLogger(l log.Logger) { + app.logger = l +} + +func (app *PersistentKVStoreApplication) Info(req types.RequestInfo) types.ResponseInfo { + res := app.app.Info(req) + res.LastBlockHeight = app.app.state.Height + res.LastBlockAppHash = app.app.state.AppHash + return res +} + +func (app *PersistentKVStoreApplication) SetOption(req types.RequestSetOption) types.ResponseSetOption { + return app.app.SetOption(req) +} + +// tx is either "val:pubkey/power" or "key=value" or just arbitrary bytes +func (app *PersistentKVStoreApplication) DeliverTx(tx []byte) types.ResponseDeliverTx { + // if it starts with "val:", update the validator set + // format is "val:pubkey/power" + if isValidatorTx(tx) { + // update validators in the merkle tree + // and in app.ValUpdates + return app.execValidatorTx(tx) + } + + // otherwise, update the key-value store + return app.app.DeliverTx(tx) +} + +func (app *PersistentKVStoreApplication) CheckTx(tx []byte) types.ResponseCheckTx { + return app.app.CheckTx(tx) +} + +// Commit will panic if InitChain was not called +func (app *PersistentKVStoreApplication) Commit() types.ResponseCommit { + return app.app.Commit() +} + +func (app *PersistentKVStoreApplication) Query(reqQuery types.RequestQuery) types.ResponseQuery { + return app.app.Query(reqQuery) +} + +// Save the validators in the merkle tree +func (app *PersistentKVStoreApplication) InitChain(req types.RequestInitChain) types.ResponseInitChain { + for _, v := range req.Validators { + r := app.updateValidator(v) + if r.IsErr() { + app.logger.Error("Error updating validators", "r", r) + } + } + return types.ResponseInitChain{} +} + +// Track the block hash and header information +func (app *PersistentKVStoreApplication) BeginBlock(req types.RequestBeginBlock) types.ResponseBeginBlock { + // reset valset changes + app.ValUpdates = make([]types.Validator, 0) + return types.ResponseBeginBlock{} +} + +// Update the validator set +func (app *PersistentKVStoreApplication) EndBlock(req types.RequestEndBlock) types.ResponseEndBlock { + return types.ResponseEndBlock{ValidatorUpdates: app.ValUpdates} +} + +//--------------------------------------------- +// update validators + +func (app *PersistentKVStoreApplication) Validators() (validators []types.Validator) { + itr := app.app.state.db.Iterator(nil, nil) + for ; itr.Valid(); itr.Next() { + if isValidatorTx(itr.Key()) { + validator := new(types.Validator) + err := types.ReadMessage(bytes.NewBuffer(itr.Value()), validator) + if err != nil { + panic(err) + } + validators = append(validators, *validator) + } + } + return +} + +func MakeValSetChangeTx(pubkey types.PubKey, power int64) []byte { + return []byte(cmn.Fmt("val:%X/%d", pubkey.Data, power)) +} + +func isValidatorTx(tx []byte) bool { + return strings.HasPrefix(string(tx), ValidatorSetChangePrefix) +} + +// format is "val:pubkey/power" +// pubkey is raw 32-byte ed25519 key +func (app *PersistentKVStoreApplication) execValidatorTx(tx []byte) types.ResponseDeliverTx { + tx = tx[len(ValidatorSetChangePrefix):] + + //get the pubkey and power + pubKeyAndPower := strings.Split(string(tx), "/") + if len(pubKeyAndPower) != 2 { + return types.ResponseDeliverTx{ + Code: code.CodeTypeEncodingError, + Log: fmt.Sprintf("Expected 'pubkey/power'. Got %v", pubKeyAndPower)} + } + pubkeyS, powerS := pubKeyAndPower[0], pubKeyAndPower[1] + + // decode the pubkey + pubkey, err := hex.DecodeString(pubkeyS) + if err != nil { + return types.ResponseDeliverTx{ + Code: code.CodeTypeEncodingError, + Log: fmt.Sprintf("Pubkey (%s) is invalid hex", pubkeyS)} + } + + // decode the power + power, err := strconv.ParseInt(powerS, 10, 64) + if err != nil { + return types.ResponseDeliverTx{ + Code: code.CodeTypeEncodingError, + Log: fmt.Sprintf("Power (%s) is not an int", powerS)} + } + + // update + return app.updateValidator(types.Ed25519Validator(pubkey, int64(power))) +} + +// add, update, or remove a validator +func (app *PersistentKVStoreApplication) updateValidator(v types.Validator) types.ResponseDeliverTx { + key := []byte("val:" + string(v.PubKey.Data)) + if v.Power == 0 { + // remove validator + if !app.app.state.db.Has(key) { + return types.ResponseDeliverTx{ + Code: code.CodeTypeUnauthorized, + Log: fmt.Sprintf("Cannot remove non-existent validator %X", key)} + } + app.app.state.db.Delete(key) + } else { + // add or update validator + value := bytes.NewBuffer(make([]byte, 0)) + if err := types.WriteMessage(&v, value); err != nil { + return types.ResponseDeliverTx{ + Code: code.CodeTypeEncodingError, + Log: fmt.Sprintf("Error encoding validator: %v", err)} + } + app.app.state.db.Set(key, value.Bytes()) + } + + // we only update the changes array if we successfully updated the tree + app.ValUpdates = append(app.ValUpdates, v) + + return types.ResponseDeliverTx{Code: code.CodeTypeOK} +} diff --git a/abci/example/python/abci/__init__.py b/abci/example/python/abci/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/abci/example/python/abci/msg.py b/abci/example/python/abci/msg.py new file mode 100644 index 000000000..7329f5852 --- /dev/null +++ b/abci/example/python/abci/msg.py @@ -0,0 +1,50 @@ +from wire import decode_string + +# map type_byte to message name +message_types = { + 0x01: "echo", + 0x02: "flush", + 0x03: "info", + 0x04: "set_option", + 0x21: "deliver_tx", + 0x22: "check_tx", + 0x23: "commit", + 0x24: "add_listener", + 0x25: "rm_listener", +} + +# return the decoded arguments of abci messages + +class RequestDecoder(): + + def __init__(self, reader): + self.reader = reader + + def echo(self): + return decode_string(self.reader) + + def flush(self): + return + + def info(self): + return + + def set_option(self): + return decode_string(self.reader), decode_string(self.reader) + + def deliver_tx(self): + return decode_string(self.reader) + + def check_tx(self): + return decode_string(self.reader) + + def commit(self): + return + + def add_listener(self): + # TODO + return + + def rm_listener(self): + # TODO + return diff --git a/abci/example/python/abci/reader.py b/abci/example/python/abci/reader.py new file mode 100644 index 000000000..6c0dad94e --- /dev/null +++ b/abci/example/python/abci/reader.py @@ -0,0 +1,56 @@ + +# Simple read() method around a bytearray + + +class BytesBuffer(): + + def __init__(self, b): + self.buf = b + self.readCount = 0 + + def count(self): + return self.readCount + + def reset_count(self): + self.readCount = 0 + + def size(self): + return len(self.buf) + + def peek(self): + return self.buf[0] + + def write(self, b): + # b should be castable to byte array + self.buf += bytearray(b) + + def read(self, n): + if len(self.buf) < n: + print "reader err: buf less than n" + # TODO: exception + return + self.readCount += n + r = self.buf[:n] + self.buf = self.buf[n:] + return r + +# Buffer bytes off a tcp connection and read them off in chunks + + +class ConnReader(): + + def __init__(self, conn): + self.conn = conn + self.buf = bytearray() + + # blocking + def read(self, n): + while n > len(self.buf): + moreBuf = self.conn.recv(1024) + if not moreBuf: + raise IOError("dead connection") + self.buf = self.buf + bytearray(moreBuf) + + r = self.buf[:n] + self.buf = self.buf[n:] + return r diff --git a/abci/example/python/abci/server.py b/abci/example/python/abci/server.py new file mode 100644 index 000000000..40d50896c --- /dev/null +++ b/abci/example/python/abci/server.py @@ -0,0 +1,202 @@ +import socket +import select +import sys + +from wire import decode_varint, encode +from reader import BytesBuffer +from msg import RequestDecoder, message_types + +# hold the asyncronous state of a connection +# ie. we may not get enough bytes on one read to decode the message + +class Connection(): + + def __init__(self, fd, app): + self.fd = fd + self.app = app + self.recBuf = BytesBuffer(bytearray()) + self.resBuf = BytesBuffer(bytearray()) + self.msgLength = 0 + self.decoder = RequestDecoder(self.recBuf) + self.inProgress = False # are we in the middle of a message + + def recv(this): + data = this.fd.recv(1024) + if not data: # what about len(data) == 0 + raise IOError("dead connection") + this.recBuf.write(data) + +# ABCI server responds to messges by calling methods on the app + +class ABCIServer(): + + def __init__(self, app, port=5410): + self.app = app + # map conn file descriptors to (app, reqBuf, resBuf, msgDecoder) + self.appMap = {} + + self.port = port + self.listen_backlog = 10 + + self.listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self.listener.setblocking(0) + self.listener.bind(('', port)) + + self.listener.listen(self.listen_backlog) + + self.shutdown = False + + self.read_list = [self.listener] + self.write_list = [] + + def handle_new_connection(self, r): + new_fd, new_addr = r.accept() + new_fd.setblocking(0) # non-blocking + self.read_list.append(new_fd) + self.write_list.append(new_fd) + print 'new connection to', new_addr + + self.appMap[new_fd] = Connection(new_fd, self.app) + + def handle_conn_closed(self, r): + self.read_list.remove(r) + self.write_list.remove(r) + r.close() + print "connection closed" + + def handle_recv(self, r): + # app, recBuf, resBuf, conn + conn = self.appMap[r] + while True: + try: + print "recv loop" + # check if we need more data first + if conn.inProgress: + if (conn.msgLength == 0 or conn.recBuf.size() < conn.msgLength): + conn.recv() + else: + if conn.recBuf.size() == 0: + conn.recv() + + conn.inProgress = True + + # see if we have enough to get the message length + if conn.msgLength == 0: + ll = conn.recBuf.peek() + if conn.recBuf.size() < 1 + ll: + # we don't have enough bytes to read the length yet + return + print "decoding msg length" + conn.msgLength = decode_varint(conn.recBuf) + + # see if we have enough to decode the message + if conn.recBuf.size() < conn.msgLength: + return + + # now we can decode the message + + # first read the request type and get the particular msg + # decoder + typeByte = conn.recBuf.read(1) + typeByte = int(typeByte[0]) + resTypeByte = typeByte + 0x10 + req_type = message_types[typeByte] + + if req_type == "flush": + # messages are length prefixed + conn.resBuf.write(encode(1)) + conn.resBuf.write([resTypeByte]) + conn.fd.send(str(conn.resBuf.buf)) + conn.msgLength = 0 + conn.inProgress = False + conn.resBuf = BytesBuffer(bytearray()) + return + + decoder = getattr(conn.decoder, req_type) + + print "decoding args" + req_args = decoder() + print "got args", req_args + + # done decoding message + conn.msgLength = 0 + conn.inProgress = False + + req_f = getattr(conn.app, req_type) + if req_args is None: + res = req_f() + elif isinstance(req_args, tuple): + res = req_f(*req_args) + else: + res = req_f(req_args) + + if isinstance(res, tuple): + res, ret_code = res + else: + ret_code = res + res = None + + print "called", req_type, "ret code:", ret_code + if ret_code != 0: + print "non-zero retcode:", ret_code + + if req_type in ("echo", "info"): # these dont return a ret code + enc = encode(res) + # messages are length prefixed + conn.resBuf.write(encode(len(enc) + 1)) + conn.resBuf.write([resTypeByte]) + conn.resBuf.write(enc) + else: + enc, encRet = encode(res), encode(ret_code) + # messages are length prefixed + conn.resBuf.write(encode(len(enc) + len(encRet) + 1)) + conn.resBuf.write([resTypeByte]) + conn.resBuf.write(encRet) + conn.resBuf.write(enc) + except TypeError as e: + print "TypeError on reading from connection:", e + self.handle_conn_closed(r) + return + except ValueError as e: + print "ValueError on reading from connection:", e + self.handle_conn_closed(r) + return + except IOError as e: + print "IOError on reading from connection:", e + self.handle_conn_closed(r) + return + except Exception as e: + # sys.exc_info()[0] # TODO better + print "error reading from connection", str(e) + self.handle_conn_closed(r) + return + + def main_loop(self): + while not self.shutdown: + r_list, w_list, _ = select.select( + self.read_list, self.write_list, [], 2.5) + + for r in r_list: + if (r == self.listener): + try: + self.handle_new_connection(r) + # undo adding to read list ... + except NameError as e: + print "Could not connect due to NameError:", e + except TypeError as e: + print "Could not connect due to TypeError:", e + except: + print "Could not connect due to unexpected error:", sys.exc_info()[0] + else: + self.handle_recv(r) + + def handle_shutdown(self): + for r in self.read_list: + r.close() + for w in self.write_list: + try: + w.close() + except Exception as e: + print(e) # TODO: add logging + self.shutdown = True diff --git a/abci/example/python/abci/wire.py b/abci/example/python/abci/wire.py new file mode 100644 index 000000000..1a07e89f1 --- /dev/null +++ b/abci/example/python/abci/wire.py @@ -0,0 +1,115 @@ + +# the decoder works off a reader +# the encoder returns bytearray + + +def hex2bytes(h): + return bytearray(h.decode('hex')) + + +def bytes2hex(b): + if type(b) in (str, unicode): + return "".join([hex(ord(c))[2:].zfill(2) for c in b]) + else: + return bytes2hex(b.decode()) + + +# expects uvarint64 (no crazy big nums!) +def uvarint_size(i): + if i == 0: + return 0 + for j in xrange(1, 8): + if i < 1 << j * 8: + return j + return 8 + +# expects i < 2**size + + +def encode_big_endian(i, size): + if size == 0: + return bytearray() + return encode_big_endian(i / 256, size - 1) + bytearray([i % 256]) + + +def decode_big_endian(reader, size): + if size == 0: + return 0 + firstByte = reader.read(1)[0] + return firstByte * (256 ** (size - 1)) + decode_big_endian(reader, size - 1) + +# ints are max 16 bytes long + + +def encode_varint(i): + negate = False + if i < 0: + negate = True + i = -i + size = uvarint_size(i) + if size == 0: + return bytearray([0]) + big_end = encode_big_endian(i, size) + if negate: + size += 0xF0 + return bytearray([size]) + big_end + +# returns the int and whats left of the byte array + + +def decode_varint(reader): + size = reader.read(1)[0] + if size == 0: + return 0 + + negate = True if size > int(0xF0) else False + if negate: + size = size - 0xF0 + i = decode_big_endian(reader, size) + if negate: + i = i * (-1) + return i + + +def encode_string(s): + size = encode_varint(len(s)) + return size + bytearray(s) + + +def decode_string(reader): + length = decode_varint(reader) + return str(reader.read(length)) + + +def encode_list(s): + b = bytearray() + map(b.extend, map(encode, s)) + return encode_varint(len(s)) + b + + +def encode(s): + if s is None: + return bytearray() + if isinstance(s, int): + return encode_varint(s) + elif isinstance(s, str): + return encode_string(s) + elif isinstance(s, list): + return encode_list(s) + else: + print "UNSUPPORTED TYPE!", type(s), s + + +if __name__ == '__main__': + ns = [100, 100, 1000, 256] + ss = [2, 5, 5, 2] + bs = map(encode_big_endian, ns, ss) + ds = map(decode_big_endian, bs, ss) + print ns + print [i[0] for i in ds] + + ss = ["abc", "hi there jim", "ok now what"] + e = map(encode_string, ss) + d = map(decode_string, e) + print ss + print [i[0] for i in d] diff --git a/abci/example/python/app.py b/abci/example/python/app.py new file mode 100644 index 000000000..1c041be6c --- /dev/null +++ b/abci/example/python/app.py @@ -0,0 +1,82 @@ +import sys + +from abci.wire import hex2bytes, decode_big_endian, encode_big_endian +from abci.server import ABCIServer +from abci.reader import BytesBuffer + + +class CounterApplication(): + + def __init__(self): + sys.exit("The python example is out of date. Upgrading the Python examples is currently left as an exercise to you.") + self.hashCount = 0 + self.txCount = 0 + self.serial = False + + def echo(self, msg): + return msg, 0 + + def info(self): + return ["hashes:%d, txs:%d" % (self.hashCount, self.txCount)], 0 + + def set_option(self, key, value): + if key == "serial" and value == "on": + self.serial = True + return 0 + + def deliver_tx(self, txBytes): + if self.serial: + txByteArray = bytearray(txBytes) + if len(txBytes) >= 2 and txBytes[:2] == "0x": + txByteArray = hex2bytes(txBytes[2:]) + txValue = decode_big_endian( + BytesBuffer(txByteArray), len(txBytes)) + if txValue != self.txCount: + return None, 6 + self.txCount += 1 + return None, 0 + + def check_tx(self, txBytes): + if self.serial: + txByteArray = bytearray(txBytes) + if len(txBytes) >= 2 and txBytes[:2] == "0x": + txByteArray = hex2bytes(txBytes[2:]) + txValue = decode_big_endian( + BytesBuffer(txByteArray), len(txBytes)) + if txValue < self.txCount: + return 6 + return 0 + + def commit(self): + self.hashCount += 1 + if self.txCount == 0: + return "", 0 + h = encode_big_endian(self.txCount, 8) + h.reverse() + return str(h), 0 + + def add_listener(self): + return 0 + + def rm_listener(self): + return 0 + + def event(self): + return + + +if __name__ == '__main__': + l = len(sys.argv) + if l == 1: + port = 26658 + elif l == 2: + port = int(sys.argv[1]) + else: + print "too many arguments" + quit() + + print 'ABCI Demo APP (Python)' + + app = CounterApplication() + server = ABCIServer(app, port) + server.main_loop() diff --git a/abci/example/python3/abci/__init__.py b/abci/example/python3/abci/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/abci/example/python3/abci/msg.py b/abci/example/python3/abci/msg.py new file mode 100644 index 000000000..807c4b6b0 --- /dev/null +++ b/abci/example/python3/abci/msg.py @@ -0,0 +1,50 @@ +from .wire import decode_string + +# map type_byte to message name +message_types = { + 0x01: "echo", + 0x02: "flush", + 0x03: "info", + 0x04: "set_option", + 0x21: "deliver_tx", + 0x22: "check_tx", + 0x23: "commit", + 0x24: "add_listener", + 0x25: "rm_listener", +} + +# return the decoded arguments of abci messages + +class RequestDecoder(): + + def __init__(self, reader): + self.reader = reader + + def echo(self): + return decode_string(self.reader) + + def flush(self): + return + + def info(self): + return + + def set_option(self): + return decode_string(self.reader), decode_string(self.reader) + + def deliver_tx(self): + return decode_string(self.reader) + + def check_tx(self): + return decode_string(self.reader) + + def commit(self): + return + + def add_listener(self): + # TODO + return + + def rm_listener(self): + # TODO + return diff --git a/abci/example/python3/abci/reader.py b/abci/example/python3/abci/reader.py new file mode 100644 index 000000000..c016ac604 --- /dev/null +++ b/abci/example/python3/abci/reader.py @@ -0,0 +1,56 @@ + +# Simple read() method around a bytearray + + +class BytesBuffer(): + + def __init__(self, b): + self.buf = b + self.readCount = 0 + + def count(self): + return self.readCount + + def reset_count(self): + self.readCount = 0 + + def size(self): + return len(self.buf) + + def peek(self): + return self.buf[0] + + def write(self, b): + # b should be castable to byte array + self.buf += bytearray(b) + + def read(self, n): + if len(self.buf) < n: + print("reader err: buf less than n") + # TODO: exception + return + self.readCount += n + r = self.buf[:n] + self.buf = self.buf[n:] + return r + +# Buffer bytes off a tcp connection and read them off in chunks + + +class ConnReader(): + + def __init__(self, conn): + self.conn = conn + self.buf = bytearray() + + # blocking + def read(self, n): + while n > len(self.buf): + moreBuf = self.conn.recv(1024) + if not moreBuf: + raise IOError("dead connection") + self.buf = self.buf + bytearray(moreBuf) + + r = self.buf[:n] + self.buf = self.buf[n:] + return r diff --git a/abci/example/python3/abci/server.py b/abci/example/python3/abci/server.py new file mode 100644 index 000000000..04063262d --- /dev/null +++ b/abci/example/python3/abci/server.py @@ -0,0 +1,196 @@ +import socket +import select +import sys +import logging + +from .wire import decode_varint, encode +from .reader import BytesBuffer +from .msg import RequestDecoder, message_types + +# hold the asyncronous state of a connection +# ie. we may not get enough bytes on one read to decode the message + +logger = logging.getLogger(__name__) + +class Connection(): + + def __init__(self, fd, app): + self.fd = fd + self.app = app + self.recBuf = BytesBuffer(bytearray()) + self.resBuf = BytesBuffer(bytearray()) + self.msgLength = 0 + self.decoder = RequestDecoder(self.recBuf) + self.inProgress = False # are we in the middle of a message + + def recv(this): + data = this.fd.recv(1024) + if not data: # what about len(data) == 0 + raise IOError("dead connection") + this.recBuf.write(data) + +# ABCI server responds to messges by calling methods on the app + +class ABCIServer(): + + def __init__(self, app, port=5410): + self.app = app + # map conn file descriptors to (app, reqBuf, resBuf, msgDecoder) + self.appMap = {} + + self.port = port + self.listen_backlog = 10 + + self.listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self.listener.setblocking(0) + self.listener.bind(('', port)) + + self.listener.listen(self.listen_backlog) + + self.shutdown = False + + self.read_list = [self.listener] + self.write_list = [] + + def handle_new_connection(self, r): + new_fd, new_addr = r.accept() + new_fd.setblocking(0) # non-blocking + self.read_list.append(new_fd) + self.write_list.append(new_fd) + print('new connection to', new_addr) + + self.appMap[new_fd] = Connection(new_fd, self.app) + + def handle_conn_closed(self, r): + self.read_list.remove(r) + self.write_list.remove(r) + r.close() + print("connection closed") + + def handle_recv(self, r): + # app, recBuf, resBuf, conn + conn = self.appMap[r] + while True: + try: + print("recv loop") + # check if we need more data first + if conn.inProgress: + if (conn.msgLength == 0 or conn.recBuf.size() < conn.msgLength): + conn.recv() + else: + if conn.recBuf.size() == 0: + conn.recv() + + conn.inProgress = True + + # see if we have enough to get the message length + if conn.msgLength == 0: + ll = conn.recBuf.peek() + if conn.recBuf.size() < 1 + ll: + # we don't have enough bytes to read the length yet + return + print("decoding msg length") + conn.msgLength = decode_varint(conn.recBuf) + + # see if we have enough to decode the message + if conn.recBuf.size() < conn.msgLength: + return + + # now we can decode the message + + # first read the request type and get the particular msg + # decoder + typeByte = conn.recBuf.read(1) + typeByte = int(typeByte[0]) + resTypeByte = typeByte + 0x10 + req_type = message_types[typeByte] + + if req_type == "flush": + # messages are length prefixed + conn.resBuf.write(encode(1)) + conn.resBuf.write([resTypeByte]) + conn.fd.send(conn.resBuf.buf) + conn.msgLength = 0 + conn.inProgress = False + conn.resBuf = BytesBuffer(bytearray()) + return + + decoder = getattr(conn.decoder, req_type) + + print("decoding args") + req_args = decoder() + print("got args", req_args) + + # done decoding message + conn.msgLength = 0 + conn.inProgress = False + + req_f = getattr(conn.app, req_type) + if req_args is None: + res = req_f() + elif isinstance(req_args, tuple): + res = req_f(*req_args) + else: + res = req_f(req_args) + + if isinstance(res, tuple): + res, ret_code = res + else: + ret_code = res + res = None + + print("called", req_type, "ret code:", ret_code, 'res:', res) + if ret_code != 0: + print("non-zero retcode:", ret_code) + + if req_type in ("echo", "info"): # these dont return a ret code + enc = encode(res) + # messages are length prefixed + conn.resBuf.write(encode(len(enc) + 1)) + conn.resBuf.write([resTypeByte]) + conn.resBuf.write(enc) + else: + enc, encRet = encode(res), encode(ret_code) + # messages are length prefixed + conn.resBuf.write(encode(len(enc) + len(encRet) + 1)) + conn.resBuf.write([resTypeByte]) + conn.resBuf.write(encRet) + conn.resBuf.write(enc) + except IOError as e: + print("IOError on reading from connection:", e) + self.handle_conn_closed(r) + return + except Exception as e: + logger.exception("error reading from connection") + self.handle_conn_closed(r) + return + + def main_loop(self): + while not self.shutdown: + r_list, w_list, _ = select.select( + self.read_list, self.write_list, [], 2.5) + + for r in r_list: + if (r == self.listener): + try: + self.handle_new_connection(r) + # undo adding to read list ... + except NameError as e: + print("Could not connect due to NameError:", e) + except TypeError as e: + print("Could not connect due to TypeError:", e) + except: + print("Could not connect due to unexpected error:", sys.exc_info()[0]) + else: + self.handle_recv(r) + + def handle_shutdown(self): + for r in self.read_list: + r.close() + for w in self.write_list: + try: + w.close() + except Exception as e: + print(e) # TODO: add logging + self.shutdown = True diff --git a/abci/example/python3/abci/wire.py b/abci/example/python3/abci/wire.py new file mode 100644 index 000000000..72f5fab8b --- /dev/null +++ b/abci/example/python3/abci/wire.py @@ -0,0 +1,119 @@ + +# the decoder works off a reader +# the encoder returns bytearray + + +def hex2bytes(h): + return bytearray(h.decode('hex')) + + +def bytes2hex(b): + if type(b) in (str, str): + return "".join([hex(ord(c))[2:].zfill(2) for c in b]) + else: + return bytes2hex(b.decode()) + + +# expects uvarint64 (no crazy big nums!) +def uvarint_size(i): + if i == 0: + return 0 + for j in range(1, 8): + if i < 1 << j * 8: + return j + return 8 + +# expects i < 2**size + + +def encode_big_endian(i, size): + if size == 0: + return bytearray() + return encode_big_endian(i // 256, size - 1) + bytearray([i % 256]) + + +def decode_big_endian(reader, size): + if size == 0: + return 0 + firstByte = reader.read(1)[0] + return firstByte * (256 ** (size - 1)) + decode_big_endian(reader, size - 1) + +# ints are max 16 bytes long + + +def encode_varint(i): + negate = False + if i < 0: + negate = True + i = -i + size = uvarint_size(i) + if size == 0: + return bytearray([0]) + big_end = encode_big_endian(i, size) + if negate: + size += 0xF0 + return bytearray([size]) + big_end + +# returns the int and whats left of the byte array + + +def decode_varint(reader): + size = reader.read(1)[0] + if size == 0: + return 0 + + negate = True if size > int(0xF0) else False + if negate: + size = size - 0xF0 + i = decode_big_endian(reader, size) + if negate: + i = i * (-1) + return i + + +def encode_string(s): + size = encode_varint(len(s)) + return size + bytearray(s, 'utf8') + + +def decode_string(reader): + length = decode_varint(reader) + raw_data = reader.read(length) + return raw_data.decode() + + +def encode_list(s): + b = bytearray() + list(map(b.extend, list(map(encode, s)))) + return encode_varint(len(s)) + b + + +def encode(s): + print('encoding', repr(s)) + if s is None: + return bytearray() + if isinstance(s, int): + return encode_varint(s) + elif isinstance(s, str): + return encode_string(s) + elif isinstance(s, list): + return encode_list(s) + elif isinstance(s, bytearray): + return encode_string(s) + else: + print("UNSUPPORTED TYPE!", type(s), s) + + +if __name__ == '__main__': + ns = [100, 100, 1000, 256] + ss = [2, 5, 5, 2] + bs = list(map(encode_big_endian, ns, ss)) + ds = list(map(decode_big_endian, bs, ss)) + print(ns) + print([i[0] for i in ds]) + + ss = ["abc", "hi there jim", "ok now what"] + e = list(map(encode_string, ss)) + d = list(map(decode_string, e)) + print(ss) + print([i[0] for i in d]) diff --git a/abci/example/python3/app.py b/abci/example/python3/app.py new file mode 100644 index 000000000..9f051b1e2 --- /dev/null +++ b/abci/example/python3/app.py @@ -0,0 +1,82 @@ +import sys + +from abci.wire import hex2bytes, decode_big_endian, encode_big_endian +from abci.server import ABCIServer +from abci.reader import BytesBuffer + + +class CounterApplication(): + + def __init__(self): + sys.exit("The python example is out of date. Upgrading the Python examples is currently left as an exercise to you.") + self.hashCount = 0 + self.txCount = 0 + self.serial = False + + def echo(self, msg): + return msg, 0 + + def info(self): + return ["hashes:%d, txs:%d" % (self.hashCount, self.txCount)], 0 + + def set_option(self, key, value): + if key == "serial" and value == "on": + self.serial = True + return 0 + + def deliver_tx(self, txBytes): + if self.serial: + txByteArray = bytearray(txBytes) + if len(txBytes) >= 2 and txBytes[:2] == "0x": + txByteArray = hex2bytes(txBytes[2:]) + txValue = decode_big_endian( + BytesBuffer(txByteArray), len(txBytes)) + if txValue != self.txCount: + return None, 6 + self.txCount += 1 + return None, 0 + + def check_tx(self, txBytes): + if self.serial: + txByteArray = bytearray(txBytes) + if len(txBytes) >= 2 and txBytes[:2] == "0x": + txByteArray = hex2bytes(txBytes[2:]) + txValue = decode_big_endian( + BytesBuffer(txByteArray), len(txBytes)) + if txValue < self.txCount: + return 6 + return 0 + + def commit(self): + self.hashCount += 1 + if self.txCount == 0: + return "", 0 + h = encode_big_endian(self.txCount, 8) + h.reverse() + return h.decode(), 0 + + def add_listener(self): + return 0 + + def rm_listener(self): + return 0 + + def event(self): + return + + +if __name__ == '__main__': + l = len(sys.argv) + if l == 1: + port = 26658 + elif l == 2: + port = int(sys.argv[1]) + else: + print("too many arguments") + quit() + + print('ABCI Demo APP (Python)') + + app = CounterApplication() + server = ABCIServer(app, port) + server.main_loop() diff --git a/abci/scripts/abci-builder/Dockerfile b/abci/scripts/abci-builder/Dockerfile new file mode 100644 index 000000000..1182085b4 --- /dev/null +++ b/abci/scripts/abci-builder/Dockerfile @@ -0,0 +1,12 @@ +FROM golang:1.9.2 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + zip \ + && rm -rf /var/lib/apt/lists/* + +# We want to ensure that release builds never have any cgo dependencies so we +# switch that off at the highest level. +ENV CGO_ENABLED 0 + +RUN mkdir -p $GOPATH/src/github.com/tendermint/abci +WORKDIR $GOPATH/src/github.com/tendermint/abci diff --git a/abci/scripts/dist.sh b/abci/scripts/dist.sh new file mode 100755 index 000000000..d94ce20f7 --- /dev/null +++ b/abci/scripts/dist.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +set -e + +REPO_NAME="abci" + +# Get the version from the environment, or try to figure it out. +if [ -z $VERSION ]; then + VERSION=$(awk -F\" '/Version =/ { print $2; exit }' < version/version.go) +fi +if [ -z "$VERSION" ]; then + echo "Please specify a version." + exit 1 +fi +echo "==> Building version $VERSION..." + +# Get the parent directory of where this script is. +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done +DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )" + +# Change into that dir because we expect that. +cd "$DIR" + +# Delete the old dir +echo "==> Removing old directory..." +rm -rf build/pkg +mkdir -p build/pkg + + +# Do a hermetic build inside a Docker container. +docker build -t tendermint/${REPO_NAME}-builder scripts/${REPO_NAME}-builder/ +docker run --rm -e "BUILD_TAGS=$BUILD_TAGS" -v "$(pwd)":/go/src/github.com/tendermint/${REPO_NAME} tendermint/${REPO_NAME}-builder ./scripts/dist_build.sh + +# Add $REPO_NAME and $VERSION prefix to package name. +rm -rf ./build/dist +mkdir -p ./build/dist +for FILENAME in $(find ./build/pkg -mindepth 1 -maxdepth 1 -type f); do + FILENAME=$(basename "$FILENAME") + cp "./build/pkg/${FILENAME}" "./build/dist/${REPO_NAME}_${VERSION}_${FILENAME}" +done + +# Make the checksums. +pushd ./build/dist +shasum -a256 ./* > "./${REPO_NAME}_${VERSION}_SHA256SUMS" +popd + +# Done +echo +echo "==> Results:" +ls -hl ./build/dist + +exit 0 diff --git a/abci/scripts/dist_build.sh b/abci/scripts/dist_build.sh new file mode 100755 index 000000000..c45c752ef --- /dev/null +++ b/abci/scripts/dist_build.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +set -e + +# Get the parent directory of where this script is. +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done +DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )" + +# Change into that dir because we expect that. +cd "$DIR" + +# Get the git commit +GIT_COMMIT="$(git rev-parse --short HEAD)" +GIT_DESCRIBE="$(git describe --tags --always)" +GIT_IMPORT="github.com/tendermint/abci/version" + +# Determine the arch/os combos we're building for +XC_ARCH=${XC_ARCH:-"386 amd64 arm"} +XC_OS=${XC_OS:-"solaris darwin freebsd linux windows"} + +# Make sure build tools are available. +make get_tools + +# Get VENDORED dependencies +make get_vendor_deps + +BINARY="abci-cli" + +# Build! +echo "==> Building..." +"$(which gox)" \ + -os="${XC_OS}" \ + -arch="${XC_ARCH}" \ + -osarch="!darwin/arm !solaris/amd64 !freebsd/amd64" \ + -ldflags "-X ${GIT_IMPORT}.GitCommit='${GIT_COMMIT}' -X ${GIT_IMPORT}.GitDescribe='${GIT_DESCRIBE}'" \ + -output "build/pkg/{{.OS}}_{{.Arch}}/$BINARY" \ + -tags="${BUILD_TAGS}" \ + github.com/tendermint/abci/cmd/$BINARY + +# Zip all the files. +echo "==> Packaging..." +for PLATFORM in $(find ./build/pkg -mindepth 1 -maxdepth 1 -type d); do + OSARCH=$(basename "${PLATFORM}") + echo "--> ${OSARCH}" + + pushd "$PLATFORM" >/dev/null 2>&1 + zip "../${OSARCH}.zip" ./* + popd >/dev/null 2>&1 +done + + + +exit 0 diff --git a/abci/scripts/publish.sh b/abci/scripts/publish.sh new file mode 100644 index 000000000..715f6c11b --- /dev/null +++ b/abci/scripts/publish.sh @@ -0,0 +1,7 @@ +#! /bin/bash + +# Get the version from the environment, or try to figure it out. +if [ -z $VERSION ]; then + VERSION=$(awk -F\" '/Version =/ { print $2; exit }' < version/version.go) +fi +aws s3 cp --recursive build/dist s3://tendermint/binaries/abci/v${VERSION} --acl public-read diff --git a/abci/server/grpc_server.go b/abci/server/grpc_server.go new file mode 100644 index 000000000..ccbe609cc --- /dev/null +++ b/abci/server/grpc_server.go @@ -0,0 +1,57 @@ +package server + +import ( + "net" + + "google.golang.org/grpc" + + "github.com/tendermint/tendermint/abci/types" + cmn "github.com/tendermint/tendermint/libs/common" +) + +type GRPCServer struct { + cmn.BaseService + + proto string + addr string + listener net.Listener + server *grpc.Server + + app types.ABCIApplicationServer +} + +// NewGRPCServer returns a new gRPC ABCI server +func NewGRPCServer(protoAddr string, app types.ABCIApplicationServer) cmn.Service { + proto, addr := cmn.ProtocolAndAddress(protoAddr) + s := &GRPCServer{ + proto: proto, + addr: addr, + listener: nil, + app: app, + } + s.BaseService = *cmn.NewBaseService(nil, "ABCIServer", s) + return s +} + +// OnStart starts the gRPC service +func (s *GRPCServer) OnStart() error { + if err := s.BaseService.OnStart(); err != nil { + return err + } + ln, err := net.Listen(s.proto, s.addr) + if err != nil { + return err + } + s.Logger.Info("Listening", "proto", s.proto, "addr", s.addr) + s.listener = ln + s.server = grpc.NewServer() + types.RegisterABCIApplicationServer(s.server, s.app) + go s.server.Serve(s.listener) + return nil +} + +// OnStop stops the gRPC server +func (s *GRPCServer) OnStop() { + s.BaseService.OnStop() + s.server.Stop() +} diff --git a/abci/server/server.go b/abci/server/server.go new file mode 100644 index 000000000..ada514fa8 --- /dev/null +++ b/abci/server/server.go @@ -0,0 +1,31 @@ +/* +Package server is used to start a new ABCI server. + +It contains two server implementation: + * gRPC server + * socket server + +*/ + +package server + +import ( + "fmt" + + "github.com/tendermint/tendermint/abci/types" + cmn "github.com/tendermint/tendermint/libs/common" +) + +func NewServer(protoAddr, transport string, app types.Application) (cmn.Service, error) { + var s cmn.Service + var err error + switch transport { + case "socket": + s = NewSocketServer(protoAddr, app) + case "grpc": + s = NewGRPCServer(protoAddr, types.NewGRPCApplication(app)) + default: + err = fmt.Errorf("Unknown server type %s", transport) + } + return s, err +} diff --git a/abci/server/socket_server.go b/abci/server/socket_server.go new file mode 100644 index 000000000..4b92f04cf --- /dev/null +++ b/abci/server/socket_server.go @@ -0,0 +1,226 @@ +package server + +import ( + "bufio" + "fmt" + "io" + "net" + "sync" + + "github.com/tendermint/tendermint/abci/types" + cmn "github.com/tendermint/tendermint/libs/common" +) + +// var maxNumberConnections = 2 + +type SocketServer struct { + cmn.BaseService + + proto string + addr string + listener net.Listener + + connsMtx sync.Mutex + conns map[int]net.Conn + nextConnID int + + appMtx sync.Mutex + app types.Application +} + +func NewSocketServer(protoAddr string, app types.Application) cmn.Service { + proto, addr := cmn.ProtocolAndAddress(protoAddr) + s := &SocketServer{ + proto: proto, + addr: addr, + listener: nil, + app: app, + conns: make(map[int]net.Conn), + } + s.BaseService = *cmn.NewBaseService(nil, "ABCIServer", s) + return s +} + +func (s *SocketServer) OnStart() error { + if err := s.BaseService.OnStart(); err != nil { + return err + } + ln, err := net.Listen(s.proto, s.addr) + if err != nil { + return err + } + s.listener = ln + go s.acceptConnectionsRoutine() + return nil +} + +func (s *SocketServer) OnStop() { + s.BaseService.OnStop() + if err := s.listener.Close(); err != nil { + s.Logger.Error("Error closing listener", "err", err) + } + + s.connsMtx.Lock() + defer s.connsMtx.Unlock() + for id, conn := range s.conns { + delete(s.conns, id) + if err := conn.Close(); err != nil { + s.Logger.Error("Error closing connection", "id", id, "conn", conn, "err", err) + } + } +} + +func (s *SocketServer) addConn(conn net.Conn) int { + s.connsMtx.Lock() + defer s.connsMtx.Unlock() + + connID := s.nextConnID + s.nextConnID++ + s.conns[connID] = conn + + return connID +} + +// deletes conn even if close errs +func (s *SocketServer) rmConn(connID int) error { + s.connsMtx.Lock() + defer s.connsMtx.Unlock() + + conn, ok := s.conns[connID] + if !ok { + return fmt.Errorf("Connection %d does not exist", connID) + } + + delete(s.conns, connID) + return conn.Close() +} + +func (s *SocketServer) acceptConnectionsRoutine() { + for { + // Accept a connection + s.Logger.Info("Waiting for new connection...") + conn, err := s.listener.Accept() + if err != nil { + if !s.IsRunning() { + return // Ignore error from listener closing. + } + s.Logger.Error("Failed to accept connection: " + err.Error()) + continue + } + + s.Logger.Info("Accepted a new connection") + + connID := s.addConn(conn) + + closeConn := make(chan error, 2) // Push to signal connection closed + responses := make(chan *types.Response, 1000) // A channel to buffer responses + + // Read requests from conn and deal with them + go s.handleRequests(closeConn, conn, responses) + // Pull responses from 'responses' and write them to conn. + go s.handleResponses(closeConn, conn, responses) + + // Wait until signal to close connection + go s.waitForClose(closeConn, connID) + } +} + +func (s *SocketServer) waitForClose(closeConn chan error, connID int) { + err := <-closeConn + if err == io.EOF { + s.Logger.Error("Connection was closed by client") + } else if err != nil { + s.Logger.Error("Connection error", "error", err) + } else { + // never happens + s.Logger.Error("Connection was closed.") + } + + // Close the connection + if err := s.rmConn(connID); err != nil { + s.Logger.Error("Error in closing connection", "error", err) + } +} + +// Read requests from conn and deal with them +func (s *SocketServer) handleRequests(closeConn chan error, conn net.Conn, responses chan<- *types.Response) { + var count int + var bufReader = bufio.NewReader(conn) + for { + + var req = &types.Request{} + err := types.ReadMessage(bufReader, req) + if err != nil { + if err == io.EOF { + closeConn <- err + } else { + closeConn <- fmt.Errorf("Error reading message: %v", err.Error()) + } + return + } + s.appMtx.Lock() + count++ + s.handleRequest(req, responses) + s.appMtx.Unlock() + } +} + +func (s *SocketServer) handleRequest(req *types.Request, responses chan<- *types.Response) { + switch r := req.Value.(type) { + case *types.Request_Echo: + responses <- types.ToResponseEcho(r.Echo.Message) + case *types.Request_Flush: + responses <- types.ToResponseFlush() + case *types.Request_Info: + res := s.app.Info(*r.Info) + responses <- types.ToResponseInfo(res) + case *types.Request_SetOption: + res := s.app.SetOption(*r.SetOption) + responses <- types.ToResponseSetOption(res) + case *types.Request_DeliverTx: + res := s.app.DeliverTx(r.DeliverTx.Tx) + responses <- types.ToResponseDeliverTx(res) + case *types.Request_CheckTx: + res := s.app.CheckTx(r.CheckTx.Tx) + responses <- types.ToResponseCheckTx(res) + case *types.Request_Commit: + res := s.app.Commit() + responses <- types.ToResponseCommit(res) + case *types.Request_Query: + res := s.app.Query(*r.Query) + responses <- types.ToResponseQuery(res) + case *types.Request_InitChain: + res := s.app.InitChain(*r.InitChain) + responses <- types.ToResponseInitChain(res) + case *types.Request_BeginBlock: + res := s.app.BeginBlock(*r.BeginBlock) + responses <- types.ToResponseBeginBlock(res) + case *types.Request_EndBlock: + res := s.app.EndBlock(*r.EndBlock) + responses <- types.ToResponseEndBlock(res) + default: + responses <- types.ToResponseException("Unknown request") + } +} + +// Pull responses from 'responses' and write them to conn. +func (s *SocketServer) handleResponses(closeConn chan error, conn net.Conn, responses <-chan *types.Response) { + var count int + var bufWriter = bufio.NewWriter(conn) + for { + var res = <-responses + err := types.WriteMessage(res, bufWriter) + if err != nil { + closeConn <- fmt.Errorf("Error writing message: %v", err.Error()) + return + } + if _, ok := res.Value.(*types.Response_Flush); ok { + err = bufWriter.Flush() + if err != nil { + closeConn <- fmt.Errorf("Error flushing write buffer: %v", err.Error()) + return + } + } + count++ + } +} diff --git a/abci/specification.rst b/abci/specification.rst new file mode 100644 index 000000000..8d8530def --- /dev/null +++ b/abci/specification.rst @@ -0,0 +1,294 @@ +ABCI Specification +================== + +NOTE: this file has moved to `specification.md <./specification.md>`__. It is left to prevent link breakages for the forseable future. It can safely be deleted in a few months. + +Message Types +~~~~~~~~~~~~~ + +ABCI requests/responses are defined as simple Protobuf messages in `this +schema +file `__. +TendermintCore sends the requests, and the ABCI application sends the +responses. Here, we provide an overview of the messages types and how they +are used by Tendermint. Then we describe each request-response pair as a +function with arguments and return values, and add some notes on usage. + +Some messages (``Echo, Info, InitChain, BeginBlock, EndBlock, Commit``), don't +return errors because an error would indicate a critical failure in the +application and there's nothing Tendermint can do. The problem should be +addressed and both Tendermint and the application restarted. All other +messages (``SetOption, Query, CheckTx, DeliverTx``) return an +application-specific response ``Code uint32``, where only ``0`` is reserved for +``OK``. + +Some messages (``SetOption, Query, CheckTx, DeliverTx``) return +non-deterministic data in the form of ``Info`` and ``Log``. The ``Log`` is +intended for the literal output from the application's logger, while the +``Info`` is any additional info that should be returned. + +The first time a new blockchain is started, Tendermint calls ``InitChain``. +From then on, the Block Execution Sequence that causes the committed state to +be updated is as follows: + +``BeginBlock, [DeliverTx], EndBlock, Commit`` + +where one ``DeliverTx`` is called for each transaction in the block. +Cryptographic commitments to the results of DeliverTx, EndBlock, and +Commit are included in the header of the next block. + +Tendermint opens three connections to the application to handle the different message +types: + +- ``Consensus Connection - InitChain, BeginBlock, DeliverTx, EndBlock, Commit`` + +- ``Mempool Connection - CheckTx`` + +- ``Info Connection - Info, SetOption, Query`` + +The ``Flush`` message is used on every connection, and the ``Echo`` message +is only used for debugging. + +Note that messages may be sent concurrently across all connections - +a typical application will thus maintain a distinct state for each +connection. They may be referred to as the ``DeliverTx state``, the +``CheckTx state``, and the ``Commit state`` respectively. + +See below for more details on the message types and how they are used. + +Echo +^^^^ + +- **Arguments**: + + - ``Message (string)``: A string to echo back + +- **Returns**: + + - ``Message (string)``: The input string + +- **Usage**: + + - Echo a string to test an abci client/server implementation + +Flush +^^^^^ + +- **Usage**: + + - Signals that messages queued on the client should be flushed to + the server. It is called periodically by the client implementation + to ensure asynchronous requests are actually sent, and is called + immediately to make a synchronous request, which returns when the + Flush response comes back. + +Info +^^^^ + +- **Arguments**: + + - ``Version (string)``: The Tendermint version + +- **Returns**: + + - ``Data (string)``: Some arbitrary information + - ``Version (Version)``: Version information + - ``LastBlockHeight (int64)``: Latest block for which the app has + called Commit + - ``LastBlockAppHash ([]byte)``: Latest result of Commit + +- **Usage**: + + - Return information about the application state. + - Used to sync Tendermint with the application during a handshake that + happens on startup. + - Tendermint expects ``LastBlockAppHash`` and ``LastBlockHeight`` to be + updated during ``Commit``, ensuring that ``Commit`` is never called twice + for the same block height. + +SetOption +^^^^^^^^^ + +- **Arguments**: + + - ``Key (string)``: Key to set + - ``Value (string)``: Value to set for key + +- **Returns**: + + - ``Code (uint32)``: Response code + - ``Log (string)``: The output of the application's logger. May be non-deterministic. + - ``Info (string)``: Additional information. May be non-deterministic. + +- **Usage**: + + - Set non-consensus critical application specific options. + - e.g. Key="min-fee", Value="100fermion" could set the minimum fee required for CheckTx + (but not DeliverTx - that would be consensus critical). + +InitChain +^^^^^^^^^ + +- **Arguments**: + + - ``Validators ([]Validator)``: Initial genesis validators + - ``AppStateBytes ([]byte)``: Serialized initial application state + +- **Usage**: + + - Called once upon genesis. + +Query +^^^^^ + +- **Arguments**: + + - ``Data ([]byte)``: Raw query bytes. Can be used with or in lieu of + Path. + - ``Path (string)``: Path of request, like an HTTP GET path. Can be + used with or in liue of Data. + - Apps MUST interpret '/store' as a query by key on the underlying + store. The key SHOULD be specified in the Data field. + - Apps SHOULD allow queries over specific types like '/accounts/...' + or '/votes/...' + - ``Height (int64)``: The block height for which you want the query + (default=0 returns data for the latest committed block). Note that + this is the height of the block containing the application's + Merkle root hash, which represents the state as it was after + committing the block at Height-1 + - ``Prove (bool)``: Return Merkle proof with response if possible + +- **Returns**: + + - ``Code (uint32)``: Response code. + - ``Log (string)``: The output of the application's logger. May be non-deterministic. + - ``Info (string)``: Additional information. May be non-deterministic. + - ``Index (int64)``: The index of the key in the tree. + - ``Key ([]byte)``: The key of the matching data. + - ``Value ([]byte)``: The value of the matching data. + - ``Proof ([]byte)``: Proof for the data, if requested. + - ``Height (int64)``: The block height from which data was derived. + Note that this is the height of the block containing the + application's Merkle root hash, which represents the state as it + was after committing the block at Height-1 + +- **Usage**: + + - Query for data from the application at current or past height. + - Optionally return Merkle proof. + +BeginBlock +^^^^^^^^^^ + +- **Arguments**: + + - ``Hash ([]byte)``: The block's hash. This can be derived from the + block header. + - ``Header (struct{})``: The block header + - ``AbsentValidators ([]int32)``: List of indices of validators not + included in the LastCommit + - ``ByzantineValidators ([]Evidence)``: List of evidence of + validators that acted maliciously + +- **Usage**: + + - Signals the beginning of a new block. Called prior to any DeliverTxs. + - The header is expected to at least contain the Height. + - The ``AbsentValidators`` and ``ByzantineValidators`` can be used to + determine rewards and punishments for the validators. + +CheckTx +^^^^^^^ + +- **Arguments**: + + - ``Tx ([]byte)``: The request transaction bytes + +- **Returns**: + + - ``Code (uint32)``: Response code + - ``Data ([]byte)``: Result bytes, if any. + - ``Log (string)``: The output of the application's logger. May be non-deterministic. + - ``Info (string)``: Additional information. May be non-deterministic. + - ``GasWanted (int64)``: Amount of gas request for transaction. + - ``GasUsed (int64)``: Amount of gas consumed by transaction. + - ``Tags ([]cmn.KVPair)``: Key-Value tags for filtering and indexing transactions (eg. by account). + - ``Fee (cmn.KI64Pair)``: Fee paid for the transaction. + +- **Usage**: Validate a mempool transaction, prior to broadcasting or + proposing. CheckTx should perform stateful but light-weight checks + of the validity of the transaction (like checking signatures and account balances), + but need not execute in full (like running a smart contract). + + Tendermint runs CheckTx and DeliverTx concurrently with eachother, + though on distinct ABCI connections - the mempool connection and the consensus + connection, respectively. + + The application should maintain a separate state to support CheckTx. + This state can be reset to the latest committed state during ``Commit``, + where Tendermint ensures the mempool is locked and not sending new ``CheckTx``. + After ``Commit``, the mempool will rerun CheckTx on all remaining + transactions, throwing out any that are no longer valid. + + Keys and values in Tags must be UTF-8 encoded strings (e.g. "account.owner": "Bob", "balance": "100.0", "date": "2018-01-02") + + +DeliverTx +^^^^^^^^^ + +- **Arguments**: + + - ``Tx ([]byte)``: The request transaction bytes. + +- **Returns**: + + - ``Code (uint32)``: Response code. + - ``Data ([]byte)``: Result bytes, if any. + - ``Log (string)``: The output of the application's logger. May be non-deterministic. + - ``Info (string)``: Additional information. May be non-deterministic. + - ``GasWanted (int64)``: Amount of gas requested for transaction. + - ``GasUsed (int64)``: Amount of gas consumed by transaction. + - ``Tags ([]cmn.KVPair)``: Key-Value tags for filtering and indexing transactions (eg. by account). + - ``Fee (cmn.KI64Pair)``: Fee paid for the transaction. + +- **Usage**: + + - Deliver a transaction to be executed in full by the application. If the transaction is valid, + returns CodeType.OK. + - Keys and values in Tags must be UTF-8 encoded strings (e.g. "account.owner": "Bob", "balance": "100.0", "time": "2018-01-02T12:30:00Z") + +EndBlock +^^^^^^^^ + +- **Arguments**: + + - ``Height (int64)``: Height of the block just executed. + +- **Returns**: + + - ``ValidatorUpdates ([]Validator)``: Changes to validator set (set + voting power to 0 to remove). + - ``ConsensusParamUpdates (ConsensusParams)``: Changes to + consensus-critical time, size, and other parameters. + +- **Usage**: + + - Signals the end of a block. + - Called prior to each Commit, after all transactions. + - Validator set and consensus params are updated with the result. + - Validator pubkeys are expected to be go-wire encoded. + +Commit +^^^^^^ + +- **Returns**: + + - ``Data ([]byte)``: The Merkle root hash + +- **Usage**: + + - Persist the application state. + - Return a Merkle root hash of the application state. + - It's critical that all application instances return the same hash. If not, + they will not be able to agree on the next block, because the hash is + included in the next block! diff --git a/abci/tests/benchmarks/blank.go b/abci/tests/benchmarks/blank.go new file mode 100644 index 000000000..20f08f14b --- /dev/null +++ b/abci/tests/benchmarks/blank.go @@ -0,0 +1 @@ +package benchmarks diff --git a/abci/tests/benchmarks/parallel/parallel.go b/abci/tests/benchmarks/parallel/parallel.go new file mode 100644 index 000000000..78b69ed12 --- /dev/null +++ b/abci/tests/benchmarks/parallel/parallel.go @@ -0,0 +1,55 @@ +package main + +import ( + "bufio" + "fmt" + "log" + + "github.com/tendermint/tendermint/abci/types" + cmn "github.com/tendermint/tendermint/libs/common" +) + +func main() { + + conn, err := cmn.Connect("unix://test.sock") + if err != nil { + log.Fatal(err.Error()) + } + + // Read a bunch of responses + go func() { + counter := 0 + for { + var res = &types.Response{} + err := types.ReadMessage(conn, res) + if err != nil { + log.Fatal(err.Error()) + } + counter++ + if counter%1000 == 0 { + fmt.Println("Read", counter) + } + } + }() + + // Write a bunch of requests + counter := 0 + for i := 0; ; i++ { + var bufWriter = bufio.NewWriter(conn) + var req = types.ToRequestEcho("foobar") + + err := types.WriteMessage(req, bufWriter) + if err != nil { + log.Fatal(err.Error()) + } + err = bufWriter.Flush() + if err != nil { + log.Fatal(err.Error()) + } + + counter++ + if counter%1000 == 0 { + fmt.Println("Write", counter) + } + } +} diff --git a/abci/tests/benchmarks/simple/simple.go b/abci/tests/benchmarks/simple/simple.go new file mode 100644 index 000000000..b0819799b --- /dev/null +++ b/abci/tests/benchmarks/simple/simple.go @@ -0,0 +1,69 @@ +package main + +import ( + "bufio" + "fmt" + "log" + "net" + "reflect" + + "github.com/tendermint/tendermint/abci/types" + cmn "github.com/tendermint/tendermint/libs/common" +) + +func main() { + + conn, err := cmn.Connect("unix://test.sock") + if err != nil { + log.Fatal(err.Error()) + } + + // Make a bunch of requests + counter := 0 + for i := 0; ; i++ { + req := types.ToRequestEcho("foobar") + _, err := makeRequest(conn, req) + if err != nil { + log.Fatal(err.Error()) + } + counter++ + if counter%1000 == 0 { + fmt.Println(counter) + } + } +} + +func makeRequest(conn net.Conn, req *types.Request) (*types.Response, error) { + var bufWriter = bufio.NewWriter(conn) + + // Write desired request + err := types.WriteMessage(req, bufWriter) + if err != nil { + return nil, err + } + err = types.WriteMessage(types.ToRequestFlush(), bufWriter) + if err != nil { + return nil, err + } + err = bufWriter.Flush() + if err != nil { + return nil, err + } + + // Read desired response + var res = &types.Response{} + err = types.ReadMessage(conn, res) + if err != nil { + return nil, err + } + var resFlush = &types.Response{} + err = types.ReadMessage(conn, resFlush) + if err != nil { + return nil, err + } + if _, ok := resFlush.Value.(*types.Response_Flush); !ok { + return nil, fmt.Errorf("Expected flush response but got something else: %v", reflect.TypeOf(resFlush)) + } + + return res, nil +} diff --git a/abci/tests/client_server_test.go b/abci/tests/client_server_test.go new file mode 100644 index 000000000..f76c9baf1 --- /dev/null +++ b/abci/tests/client_server_test.go @@ -0,0 +1,27 @@ +package tests + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + abciclient "github.com/tendermint/tendermint/abci/client" + "github.com/tendermint/tendermint/abci/example/kvstore" + abciserver "github.com/tendermint/tendermint/abci/server" +) + +func TestClientServerNoAddrPrefix(t *testing.T) { + addr := "localhost:26658" + transport := "socket" + app := kvstore.NewKVStoreApplication() + + server, err := abciserver.NewServer(addr, transport, app) + assert.NoError(t, err, "expected no error on NewServer") + err = server.Start() + assert.NoError(t, err, "expected no error on server.Start") + + client, err := abciclient.NewClient(addr, transport, true) + assert.NoError(t, err, "expected no error on NewClient") + err = client.Start() + assert.NoError(t, err, "expected no error on client.Start") +} diff --git a/abci/tests/server/client.go b/abci/tests/server/client.go new file mode 100644 index 000000000..f67297cd7 --- /dev/null +++ b/abci/tests/server/client.go @@ -0,0 +1,96 @@ +package testsuite + +import ( + "bytes" + "errors" + "fmt" + + abcicli "github.com/tendermint/tendermint/abci/client" + "github.com/tendermint/tendermint/abci/types" + cmn "github.com/tendermint/tendermint/libs/common" +) + +func InitChain(client abcicli.Client) error { + total := 10 + vals := make([]types.Validator, total) + for i := 0; i < total; i++ { + pubkey := cmn.RandBytes(33) + power := cmn.RandInt() + vals[i] = types.Ed25519Validator(pubkey, int64(power)) + } + _, err := client.InitChainSync(types.RequestInitChain{ + Validators: vals, + }) + if err != nil { + fmt.Printf("Failed test: InitChain - %v\n", err) + return err + } + fmt.Println("Passed test: InitChain") + return nil +} + +func SetOption(client abcicli.Client, key, value string) error { + _, err := client.SetOptionSync(types.RequestSetOption{Key: key, Value: value}) + if err != nil { + fmt.Println("Failed test: SetOption") + fmt.Printf("error while setting %v=%v: \nerror: %v\n", key, value, err) + return err + } + fmt.Println("Passed test: SetOption") + return nil +} + +func Commit(client abcicli.Client, hashExp []byte) error { + res, err := client.CommitSync() + data := res.Data + if err != nil { + fmt.Println("Failed test: Commit") + fmt.Printf("error while committing: %v\n", err) + return err + } + if !bytes.Equal(data, hashExp) { + fmt.Println("Failed test: Commit") + fmt.Printf("Commit hash was unexpected. Got %X expected %X\n", data, hashExp) + return errors.New("CommitTx failed") + } + fmt.Println("Passed test: Commit") + return nil +} + +func DeliverTx(client abcicli.Client, txBytes []byte, codeExp uint32, dataExp []byte) error { + res, _ := client.DeliverTxSync(txBytes) + code, data, log := res.Code, res.Data, res.Log + if code != codeExp { + fmt.Println("Failed test: DeliverTx") + fmt.Printf("DeliverTx response code was unexpected. Got %v expected %v. Log: %v\n", + code, codeExp, log) + return errors.New("DeliverTx error") + } + if !bytes.Equal(data, dataExp) { + fmt.Println("Failed test: DeliverTx") + fmt.Printf("DeliverTx response data was unexpected. Got %X expected %X\n", + data, dataExp) + return errors.New("DeliverTx error") + } + fmt.Println("Passed test: DeliverTx") + return nil +} + +func CheckTx(client abcicli.Client, txBytes []byte, codeExp uint32, dataExp []byte) error { + res, _ := client.CheckTxSync(txBytes) + code, data, log := res.Code, res.Data, res.Log + if code != codeExp { + fmt.Println("Failed test: CheckTx") + fmt.Printf("CheckTx response code was unexpected. Got %v expected %v. Log: %v\n", + code, codeExp, log) + return errors.New("CheckTx") + } + if !bytes.Equal(data, dataExp) { + fmt.Println("Failed test: CheckTx") + fmt.Printf("CheckTx response data was unexpected. Got %X expected %X\n", + data, dataExp) + return errors.New("CheckTx") + } + fmt.Println("Passed test: CheckTx") + return nil +} diff --git a/abci/tests/test_app/app.go b/abci/tests/test_app/app.go new file mode 100644 index 000000000..a33f4ee9e --- /dev/null +++ b/abci/tests/test_app/app.go @@ -0,0 +1,78 @@ +package main + +import ( + "bytes" + "fmt" + "os" + + abcicli "github.com/tendermint/tendermint/abci/client" + "github.com/tendermint/tendermint/abci/types" + "github.com/tendermint/tendermint/libs/log" +) + +func startClient(abciType string) abcicli.Client { + // Start client + client, err := abcicli.NewClient("tcp://127.0.0.1:26658", abciType, true) + if err != nil { + panic(err.Error()) + } + logger := log.NewTMLogger(log.NewSyncWriter(os.Stdout)) + client.SetLogger(logger.With("module", "abcicli")) + if err := client.Start(); err != nil { + panicf("connecting to abci_app: %v", err.Error()) + } + + return client +} + +func setOption(client abcicli.Client, key, value string) { + _, err := client.SetOptionSync(types.RequestSetOption{key, value}) + if err != nil { + panicf("setting %v=%v: \nerr: %v", key, value, err) + } +} + +func commit(client abcicli.Client, hashExp []byte) { + res, err := client.CommitSync() + if err != nil { + panicf("client error: %v", err) + } + if !bytes.Equal(res.Data, hashExp) { + panicf("Commit hash was unexpected. Got %X expected %X", res.Data, hashExp) + } +} + +func deliverTx(client abcicli.Client, txBytes []byte, codeExp uint32, dataExp []byte) { + res, err := client.DeliverTxSync(txBytes) + if err != nil { + panicf("client error: %v", err) + } + if res.Code != codeExp { + panicf("DeliverTx response code was unexpected. Got %v expected %v. Log: %v", res.Code, codeExp, res.Log) + } + if !bytes.Equal(res.Data, dataExp) { + panicf("DeliverTx response data was unexpected. Got %X expected %X", res.Data, dataExp) + } +} + +/*func checkTx(client abcicli.Client, txBytes []byte, codeExp uint32, dataExp []byte) { + res, err := client.CheckTxSync(txBytes) + if err != nil { + panicf("client error: %v", err) + } + if res.IsErr() { + panicf("checking tx %X: %v\nlog: %v", txBytes, res.Log) + } + if res.Code != codeExp { + panicf("CheckTx response code was unexpected. Got %v expected %v. Log: %v", + res.Code, codeExp, res.Log) + } + if !bytes.Equal(res.Data, dataExp) { + panicf("CheckTx response data was unexpected. Got %X expected %X", + res.Data, dataExp) + } +}*/ + +func panicf(format string, a ...interface{}) { + panic(fmt.Sprintf(format, a...)) +} diff --git a/abci/tests/test_app/main.go b/abci/tests/test_app/main.go new file mode 100644 index 000000000..8f45cec3c --- /dev/null +++ b/abci/tests/test_app/main.go @@ -0,0 +1,84 @@ +package main + +import ( + "fmt" + "log" + "os" + "os/exec" + "time" + + "github.com/tendermint/tendermint/abci/example/code" + "github.com/tendermint/tendermint/abci/types" +) + +var abciType string + +func init() { + abciType = os.Getenv("ABCI") + if abciType == "" { + abciType = "socket" + } +} + +func main() { + testCounter() +} + +const ( + maxABCIConnectTries = 10 +) + +func ensureABCIIsUp(typ string, n int) error { + var err error + cmdString := "abci-cli echo hello" + if typ == "grpc" { + cmdString = "abci-cli --abci grpc echo hello" + } + + for i := 0; i < n; i++ { + cmd := exec.Command("bash", "-c", cmdString) // nolint: gas + _, err = cmd.CombinedOutput() + if err == nil { + break + } + <-time.After(500 * time.Millisecond) + } + return err +} + +func testCounter() { + abciApp := os.Getenv("ABCI_APP") + if abciApp == "" { + panic("No ABCI_APP specified") + } + + fmt.Printf("Running %s test with abci=%s\n", abciApp, abciType) + cmd := exec.Command("bash", "-c", fmt.Sprintf("abci-cli %s", abciApp)) // nolint: gas + cmd.Stdout = os.Stdout + if err := cmd.Start(); err != nil { + log.Fatalf("starting %q err: %v", abciApp, err) + } + defer cmd.Wait() + defer cmd.Process.Kill() + + if err := ensureABCIIsUp(abciType, maxABCIConnectTries); err != nil { + log.Fatalf("echo failed: %v", err) + } + + client := startClient(abciType) + defer client.Stop() + + setOption(client, "serial", "on") + commit(client, nil) + deliverTx(client, []byte("abc"), code.CodeTypeBadNonce, nil) + commit(client, nil) + deliverTx(client, []byte{0x00}, types.CodeTypeOK, nil) + commit(client, []byte{0, 0, 0, 0, 0, 0, 0, 1}) + deliverTx(client, []byte{0x00}, code.CodeTypeBadNonce, nil) + deliverTx(client, []byte{0x01}, types.CodeTypeOK, nil) + deliverTx(client, []byte{0x00, 0x02}, types.CodeTypeOK, nil) + deliverTx(client, []byte{0x00, 0x03}, types.CodeTypeOK, nil) + deliverTx(client, []byte{0x00, 0x00, 0x04}, types.CodeTypeOK, nil) + deliverTx(client, []byte{0x00, 0x00, 0x06}, code.CodeTypeBadNonce, nil) + commit(client, []byte{0, 0, 0, 0, 0, 0, 0, 5}) +} diff --git a/abci/tests/test_app/test.sh b/abci/tests/test_app/test.sh new file mode 100755 index 000000000..230c94163 --- /dev/null +++ b/abci/tests/test_app/test.sh @@ -0,0 +1,27 @@ +#! /bin/bash +set -e + +# These tests spawn the counter app and server by execing the ABCI_APP command and run some simple client tests against it + +# Get the directory of where this script is. +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done +DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + +# Change into that dir because we expect that. +cd "$DIR" + +echo "RUN COUNTER OVER SOCKET" +# test golang counter +ABCI_APP="counter" go run ./*.go +echo "----------------------" + + +echo "RUN COUNTER OVER GRPC" +# test golang counter via grpc +ABCI_APP="counter --abci=grpc" ABCI="grpc" go run ./*.go +echo "----------------------" + +# test nodejs counter +# TODO: fix node app +#ABCI_APP="node $GOPATH/src/github.com/tendermint/js-abci/example/app.js" go test -test.run TestCounter diff --git a/abci/tests/test_cli/ex1.abci b/abci/tests/test_cli/ex1.abci new file mode 100644 index 000000000..e909266ec --- /dev/null +++ b/abci/tests/test_cli/ex1.abci @@ -0,0 +1,10 @@ +echo hello +info +commit +deliver_tx "abc" +info +commit +query "abc" +deliver_tx "def=xyz" +commit +query "def" diff --git a/abci/tests/test_cli/ex1.abci.out b/abci/tests/test_cli/ex1.abci.out new file mode 100644 index 000000000..5d4c196dc --- /dev/null +++ b/abci/tests/test_cli/ex1.abci.out @@ -0,0 +1,47 @@ +> echo hello +-> code: OK +-> data: hello +-> data.hex: 0x68656C6C6F + +> info +-> code: OK +-> data: {"size":0} +-> data.hex: 0x7B2273697A65223A307D + +> commit +-> code: OK +-> data.hex: 0x0000000000000000 + +> deliver_tx "abc" +-> code: OK + +> info +-> code: OK +-> data: {"size":1} +-> data.hex: 0x7B2273697A65223A317D + +> commit +-> code: OK +-> data.hex: 0x0200000000000000 + +> query "abc" +-> code: OK +-> log: exists +-> height: 0 +-> value: abc +-> value.hex: 616263 + +> deliver_tx "def=xyz" +-> code: OK + +> commit +-> code: OK +-> data.hex: 0x0400000000000000 + +> query "def" +-> code: OK +-> log: exists +-> height: 0 +-> value: xyz +-> value.hex: 78797A + diff --git a/abci/tests/test_cli/ex2.abci b/abci/tests/test_cli/ex2.abci new file mode 100644 index 000000000..3b435f22a --- /dev/null +++ b/abci/tests/test_cli/ex2.abci @@ -0,0 +1,8 @@ +set_option serial on +check_tx 0x00 +check_tx 0xff +deliver_tx 0x00 +check_tx 0x00 +deliver_tx 0x01 +deliver_tx 0x04 +info diff --git a/abci/tests/test_cli/ex2.abci.out b/abci/tests/test_cli/ex2.abci.out new file mode 100644 index 000000000..5bceb85d8 --- /dev/null +++ b/abci/tests/test_cli/ex2.abci.out @@ -0,0 +1,29 @@ +> set_option serial on +-> code: OK +-> log: OK (SetOption doesn't return anything.) + +> check_tx 0x00 +-> code: OK + +> check_tx 0xff +-> code: OK + +> deliver_tx 0x00 +-> code: OK + +> check_tx 0x00 +-> code: 2 +-> log: Invalid nonce. Expected >= 1, got 0 + +> deliver_tx 0x01 +-> code: OK + +> deliver_tx 0x04 +-> code: 2 +-> log: Invalid nonce. Expected 2, got 4 + +> info +-> code: OK +-> data: {"hashes":0,"txs":2} +-> data.hex: 0x7B22686173686573223A302C22747873223A327D + diff --git a/abci/tests/test_cli/test.sh b/abci/tests/test_cli/test.sh new file mode 100755 index 000000000..ce074f513 --- /dev/null +++ b/abci/tests/test_cli/test.sh @@ -0,0 +1,42 @@ +#! /bin/bash +set -e + +# Get the root directory. +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done +DIR="$( cd -P "$( dirname "$SOURCE" )/../.." && pwd )" + +# Change into that dir because we expect that. +cd "$DIR" || exit + +function testExample() { + N=$1 + INPUT=$2 + APP="$3 $4" + + echo "Example $N: $APP" + $APP &> /dev/null & + sleep 2 + abci-cli --log_level=error --verbose batch < "$INPUT" > "${INPUT}.out.new" + killall "$3" + + pre=$(shasum < "${INPUT}.out") + post=$(shasum < "${INPUT}.out.new") + + if [[ "$pre" != "$post" ]]; then + echo "You broke the tutorial" + echo "Got:" + cat "${INPUT}.out.new" + echo "Expected:" + cat "${INPUT}.out" + exit 1 + fi + + rm "${INPUT}".out.new +} + +testExample 1 tests/test_cli/ex1.abci abci-cli kvstore +testExample 2 tests/test_cli/ex2.abci abci-cli counter + +echo "" +echo "PASS" diff --git a/abci/tests/test_cover.sh b/abci/tests/test_cover.sh new file mode 100755 index 000000000..abbbbe563 --- /dev/null +++ b/abci/tests/test_cover.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -e +echo "" > coverage.txt + +echo "==> Running unit tests" +for d in $(go list ./... | grep -v vendor); do + go test -race -coverprofile=profile.out -covermode=atomic "$d" + if [ -f profile.out ]; then + cat profile.out >> coverage.txt + rm profile.out + fi +done diff --git a/abci/tests/tests.go b/abci/tests/tests.go new file mode 100644 index 000000000..ca8701d29 --- /dev/null +++ b/abci/tests/tests.go @@ -0,0 +1 @@ +package tests diff --git a/abci/types/application.go b/abci/types/application.go new file mode 100644 index 000000000..ef1bc92e5 --- /dev/null +++ b/abci/types/application.go @@ -0,0 +1,138 @@ +package types // nolint: goimports + +import ( + context "golang.org/x/net/context" +) + +// Application is an interface that enables any finite, deterministic state machine +// to be driven by a blockchain-based replication engine via the ABCI. +// All methods take a RequestXxx argument and return a ResponseXxx argument, +// except CheckTx/DeliverTx, which take `tx []byte`, and `Commit`, which takes nothing. +type Application interface { + // Info/Query Connection + Info(RequestInfo) ResponseInfo // Return application info + SetOption(RequestSetOption) ResponseSetOption // Set application option + Query(RequestQuery) ResponseQuery // Query for state + + // Mempool Connection + CheckTx(tx []byte) ResponseCheckTx // Validate a tx for the mempool + + // Consensus Connection + InitChain(RequestInitChain) ResponseInitChain // Initialize blockchain with validators and other info from TendermintCore + BeginBlock(RequestBeginBlock) ResponseBeginBlock // Signals the beginning of a block + DeliverTx(tx []byte) ResponseDeliverTx // Deliver a tx for full processing + EndBlock(RequestEndBlock) ResponseEndBlock // Signals the end of a block, returns changes to the validator set + Commit() ResponseCommit // Commit the state and return the application Merkle root hash +} + +//------------------------------------------------------- +// BaseApplication is a base form of Application + +var _ Application = (*BaseApplication)(nil) + +type BaseApplication struct { +} + +func NewBaseApplication() *BaseApplication { + return &BaseApplication{} +} + +func (BaseApplication) Info(req RequestInfo) ResponseInfo { + return ResponseInfo{} +} + +func (BaseApplication) SetOption(req RequestSetOption) ResponseSetOption { + return ResponseSetOption{} +} + +func (BaseApplication) DeliverTx(tx []byte) ResponseDeliverTx { + return ResponseDeliverTx{Code: CodeTypeOK} +} + +func (BaseApplication) CheckTx(tx []byte) ResponseCheckTx { + return ResponseCheckTx{Code: CodeTypeOK} +} + +func (BaseApplication) Commit() ResponseCommit { + return ResponseCommit{} +} + +func (BaseApplication) Query(req RequestQuery) ResponseQuery { + return ResponseQuery{Code: CodeTypeOK} +} + +func (BaseApplication) InitChain(req RequestInitChain) ResponseInitChain { + return ResponseInitChain{} +} + +func (BaseApplication) BeginBlock(req RequestBeginBlock) ResponseBeginBlock { + return ResponseBeginBlock{} +} + +func (BaseApplication) EndBlock(req RequestEndBlock) ResponseEndBlock { + return ResponseEndBlock{} +} + +//------------------------------------------------------- + +// GRPCApplication is a GRPC wrapper for Application +type GRPCApplication struct { + app Application +} + +func NewGRPCApplication(app Application) *GRPCApplication { + return &GRPCApplication{app} +} + +func (app *GRPCApplication) Echo(ctx context.Context, req *RequestEcho) (*ResponseEcho, error) { + return &ResponseEcho{req.Message}, nil +} + +func (app *GRPCApplication) Flush(ctx context.Context, req *RequestFlush) (*ResponseFlush, error) { + return &ResponseFlush{}, nil +} + +func (app *GRPCApplication) Info(ctx context.Context, req *RequestInfo) (*ResponseInfo, error) { + res := app.app.Info(*req) + return &res, nil +} + +func (app *GRPCApplication) SetOption(ctx context.Context, req *RequestSetOption) (*ResponseSetOption, error) { + res := app.app.SetOption(*req) + return &res, nil +} + +func (app *GRPCApplication) DeliverTx(ctx context.Context, req *RequestDeliverTx) (*ResponseDeliverTx, error) { + res := app.app.DeliverTx(req.Tx) + return &res, nil +} + +func (app *GRPCApplication) CheckTx(ctx context.Context, req *RequestCheckTx) (*ResponseCheckTx, error) { + res := app.app.CheckTx(req.Tx) + return &res, nil +} + +func (app *GRPCApplication) Query(ctx context.Context, req *RequestQuery) (*ResponseQuery, error) { + res := app.app.Query(*req) + return &res, nil +} + +func (app *GRPCApplication) Commit(ctx context.Context, req *RequestCommit) (*ResponseCommit, error) { + res := app.app.Commit() + return &res, nil +} + +func (app *GRPCApplication) InitChain(ctx context.Context, req *RequestInitChain) (*ResponseInitChain, error) { + res := app.app.InitChain(*req) + return &res, nil +} + +func (app *GRPCApplication) BeginBlock(ctx context.Context, req *RequestBeginBlock) (*ResponseBeginBlock, error) { + res := app.app.BeginBlock(*req) + return &res, nil +} + +func (app *GRPCApplication) EndBlock(ctx context.Context, req *RequestEndBlock) (*ResponseEndBlock, error) { + res := app.app.EndBlock(*req) + return &res, nil +} diff --git a/abci/types/messages.go b/abci/types/messages.go new file mode 100644 index 000000000..52e4b6758 --- /dev/null +++ b/abci/types/messages.go @@ -0,0 +1,210 @@ +package types + +import ( + "bufio" + "encoding/binary" + "io" + + "github.com/gogo/protobuf/proto" +) + +const ( + maxMsgSize = 104857600 // 100MB +) + +// WriteMessage writes a varint length-delimited protobuf message. +func WriteMessage(msg proto.Message, w io.Writer) error { + bz, err := proto.Marshal(msg) + if err != nil { + return err + } + return encodeByteSlice(w, bz) +} + +// ReadMessage reads a varint length-delimited protobuf message. +func ReadMessage(r io.Reader, msg proto.Message) error { + return readProtoMsg(r, msg, maxMsgSize) +} + +func readProtoMsg(r io.Reader, msg proto.Message, maxSize int) error { + // binary.ReadVarint takes an io.ByteReader, eg. a bufio.Reader + reader, ok := r.(*bufio.Reader) + if !ok { + reader = bufio.NewReader(r) + } + length64, err := binary.ReadVarint(reader) + if err != nil { + return err + } + length := int(length64) + if length < 0 || length > maxSize { + return io.ErrShortBuffer + } + buf := make([]byte, length) + if _, err := io.ReadFull(reader, buf); err != nil { + return err + } + return proto.Unmarshal(buf, msg) +} + +//----------------------------------------------------------------------- +// NOTE: we copied wire.EncodeByteSlice from go-wire rather than keep +// go-wire as a dep + +func encodeByteSlice(w io.Writer, bz []byte) (err error) { + err = encodeVarint(w, int64(len(bz))) + if err != nil { + return + } + _, err = w.Write(bz) + return +} + +func encodeVarint(w io.Writer, i int64) (err error) { + var buf [10]byte + n := binary.PutVarint(buf[:], i) + _, err = w.Write(buf[0:n]) + return +} + +//---------------------------------------- + +func ToRequestEcho(message string) *Request { + return &Request{ + Value: &Request_Echo{&RequestEcho{message}}, + } +} + +func ToRequestFlush() *Request { + return &Request{ + Value: &Request_Flush{&RequestFlush{}}, + } +} + +func ToRequestInfo(req RequestInfo) *Request { + return &Request{ + Value: &Request_Info{&req}, + } +} + +func ToRequestSetOption(req RequestSetOption) *Request { + return &Request{ + Value: &Request_SetOption{&req}, + } +} + +func ToRequestDeliverTx(tx []byte) *Request { + return &Request{ + Value: &Request_DeliverTx{&RequestDeliverTx{tx}}, + } +} + +func ToRequestCheckTx(tx []byte) *Request { + return &Request{ + Value: &Request_CheckTx{&RequestCheckTx{tx}}, + } +} + +func ToRequestCommit() *Request { + return &Request{ + Value: &Request_Commit{&RequestCommit{}}, + } +} + +func ToRequestQuery(req RequestQuery) *Request { + return &Request{ + Value: &Request_Query{&req}, + } +} + +func ToRequestInitChain(req RequestInitChain) *Request { + return &Request{ + Value: &Request_InitChain{&req}, + } +} + +func ToRequestBeginBlock(req RequestBeginBlock) *Request { + return &Request{ + Value: &Request_BeginBlock{&req}, + } +} + +func ToRequestEndBlock(req RequestEndBlock) *Request { + return &Request{ + Value: &Request_EndBlock{&req}, + } +} + +//---------------------------------------- + +func ToResponseException(errStr string) *Response { + return &Response{ + Value: &Response_Exception{&ResponseException{errStr}}, + } +} + +func ToResponseEcho(message string) *Response { + return &Response{ + Value: &Response_Echo{&ResponseEcho{message}}, + } +} + +func ToResponseFlush() *Response { + return &Response{ + Value: &Response_Flush{&ResponseFlush{}}, + } +} + +func ToResponseInfo(res ResponseInfo) *Response { + return &Response{ + Value: &Response_Info{&res}, + } +} + +func ToResponseSetOption(res ResponseSetOption) *Response { + return &Response{ + Value: &Response_SetOption{&res}, + } +} + +func ToResponseDeliverTx(res ResponseDeliverTx) *Response { + return &Response{ + Value: &Response_DeliverTx{&res}, + } +} + +func ToResponseCheckTx(res ResponseCheckTx) *Response { + return &Response{ + Value: &Response_CheckTx{&res}, + } +} + +func ToResponseCommit(res ResponseCommit) *Response { + return &Response{ + Value: &Response_Commit{&res}, + } +} + +func ToResponseQuery(res ResponseQuery) *Response { + return &Response{ + Value: &Response_Query{&res}, + } +} + +func ToResponseInitChain(res ResponseInitChain) *Response { + return &Response{ + Value: &Response_InitChain{&res}, + } +} + +func ToResponseBeginBlock(res ResponseBeginBlock) *Response { + return &Response{ + Value: &Response_BeginBlock{&res}, + } +} + +func ToResponseEndBlock(res ResponseEndBlock) *Response { + return &Response{ + Value: &Response_EndBlock{&res}, + } +} diff --git a/abci/types/messages_test.go b/abci/types/messages_test.go new file mode 100644 index 000000000..da6595a46 --- /dev/null +++ b/abci/types/messages_test.go @@ -0,0 +1,104 @@ +package types + +import ( + "bytes" + "encoding/json" + "strings" + "testing" + + "github.com/gogo/protobuf/proto" + "github.com/stretchr/testify/assert" + cmn "github.com/tendermint/tendermint/libs/common" +) + +func TestMarshalJSON(t *testing.T) { + b, err := json.Marshal(&ResponseDeliverTx{}) + assert.Nil(t, err) + // Do not include empty fields. + assert.False(t, strings.Contains(string(b), "code")) + + r1 := ResponseCheckTx{ + Code: 1, + Data: []byte("hello"), + GasWanted: 43, + Tags: []cmn.KVPair{ + {[]byte("pho"), []byte("bo")}, + }, + } + b, err = json.Marshal(&r1) + assert.Nil(t, err) + + var r2 ResponseCheckTx + err = json.Unmarshal(b, &r2) + assert.Nil(t, err) + assert.Equal(t, r1, r2) +} + +func TestWriteReadMessageSimple(t *testing.T) { + cases := []proto.Message{ + &RequestEcho{ + Message: "Hello", + }, + } + + for _, c := range cases { + buf := new(bytes.Buffer) + err := WriteMessage(c, buf) + assert.Nil(t, err) + + msg := new(RequestEcho) + err = ReadMessage(buf, msg) + assert.Nil(t, err) + + assert.Equal(t, c, msg) + } +} + +func TestWriteReadMessage(t *testing.T) { + cases := []proto.Message{ + &Header{ + NumTxs: 4, + }, + // TODO: add the rest + } + + for _, c := range cases { + buf := new(bytes.Buffer) + err := WriteMessage(c, buf) + assert.Nil(t, err) + + msg := new(Header) + err = ReadMessage(buf, msg) + assert.Nil(t, err) + + assert.Equal(t, c, msg) + } +} + +func TestWriteReadMessage2(t *testing.T) { + phrase := "hello-world" + cases := []proto.Message{ + &ResponseCheckTx{ + Data: []byte(phrase), + Log: phrase, + GasWanted: 10, + Tags: []cmn.KVPair{ + cmn.KVPair{[]byte("abc"), []byte("def")}, + }, + // Fee: cmn.KI64Pair{ + }, + // TODO: add the rest + } + + for _, c := range cases { + buf := new(bytes.Buffer) + err := WriteMessage(c, buf) + assert.Nil(t, err) + + msg := new(ResponseCheckTx) + err = ReadMessage(buf, msg) + assert.Nil(t, err) + + assert.Equal(t, c, msg) + } +} diff --git a/abci/types/protoreplace/protoreplace.go b/abci/types/protoreplace/protoreplace.go new file mode 100644 index 000000000..3ea0c73da --- /dev/null +++ b/abci/types/protoreplace/protoreplace.go @@ -0,0 +1,55 @@ +// +build ignore + +package main + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "os/exec" + "regexp" + "strings" +) + +// This script replaces most `[]byte` with `data.Bytes` in a `.pb.go` file. +// It was written before we realized we could use `gogo/protobuf` to achieve +// this more natively. So it's here for safe keeping in case we ever need to +// abandon `gogo/protobuf`. + +func main() { + bytePattern := regexp.MustCompile("[[][]]byte") + const oldPath = "types/types.pb.go" + const tmpPath = "types/types.pb.new" + content, err := ioutil.ReadFile(oldPath) + if err != nil { + panic("cannot read " + oldPath) + os.Exit(1) + } + lines := bytes.Split(content, []byte("\n")) + outFile, _ := os.Create(tmpPath) + wroteImport := false + for _, line_bytes := range lines { + line := string(line_bytes) + gotPackageLine := strings.HasPrefix(line, "package ") + writeImportTime := strings.HasPrefix(line, "import ") + containsDescriptor := strings.Contains(line, "Descriptor") + containsByteArray := strings.Contains(line, "[]byte") + if containsByteArray && !containsDescriptor { + line = string(bytePattern.ReplaceAll([]byte(line), []byte("data.Bytes"))) + } + if writeImportTime && !wroteImport { + wroteImport = true + fmt.Fprintf(outFile, "import \"github.com/tendermint/go-wire/data\"\n") + + } + if gotPackageLine { + fmt.Fprintf(outFile, "%s\n", "//nolint: gas") + } + fmt.Fprintf(outFile, "%s\n", line) + } + outFile.Close() + os.Remove(oldPath) + os.Rename(tmpPath, oldPath) + exec.Command("goimports", "-w", oldPath) +} diff --git a/abci/types/pubkey.go b/abci/types/pubkey.go new file mode 100644 index 000000000..e5cd5fbf3 --- /dev/null +++ b/abci/types/pubkey.go @@ -0,0 +1,16 @@ +package types + +const ( + PubKeyEd25519 = "ed25519" +) + +func Ed25519Validator(pubkey []byte, power int64) Validator { + return Validator{ + // Address: + PubKey: PubKey{ + Type: PubKeyEd25519, + Data: pubkey, + }, + Power: power, + } +} diff --git a/abci/types/result.go b/abci/types/result.go new file mode 100644 index 000000000..dbf409f4c --- /dev/null +++ b/abci/types/result.go @@ -0,0 +1,121 @@ +package types + +import ( + "bytes" + "encoding/json" + + "github.com/gogo/protobuf/jsonpb" +) + +const ( + CodeTypeOK uint32 = 0 +) + +// IsOK returns true if Code is OK. +func (r ResponseCheckTx) IsOK() bool { + return r.Code == CodeTypeOK +} + +// IsErr returns true if Code is something other than OK. +func (r ResponseCheckTx) IsErr() bool { + return r.Code != CodeTypeOK +} + +// IsOK returns true if Code is OK. +func (r ResponseDeliverTx) IsOK() bool { + return r.Code == CodeTypeOK +} + +// IsErr returns true if Code is something other than OK. +func (r ResponseDeliverTx) IsErr() bool { + return r.Code != CodeTypeOK +} + +// IsOK returns true if Code is OK. +func (r ResponseQuery) IsOK() bool { + return r.Code == CodeTypeOK +} + +// IsErr returns true if Code is something other than OK. +func (r ResponseQuery) IsErr() bool { + return r.Code != CodeTypeOK +} + +//--------------------------------------------------------------------------- +// override JSON marshalling so we dont emit defaults (ie. disable omitempty) +// note we need Unmarshal functions too because protobuf had the bright idea +// to marshal int64->string. cool. cool, cool, cool: https://developers.google.com/protocol-buffers/docs/proto3#json + +var ( + jsonpbMarshaller = jsonpb.Marshaler{ + EnumsAsInts: true, + EmitDefaults: false, + } + jsonpbUnmarshaller = jsonpb.Unmarshaler{} +) + +func (r *ResponseSetOption) MarshalJSON() ([]byte, error) { + s, err := jsonpbMarshaller.MarshalToString(r) + return []byte(s), err +} + +func (r *ResponseSetOption) UnmarshalJSON(b []byte) error { + reader := bytes.NewBuffer(b) + return jsonpbUnmarshaller.Unmarshal(reader, r) +} + +func (r *ResponseCheckTx) MarshalJSON() ([]byte, error) { + s, err := jsonpbMarshaller.MarshalToString(r) + return []byte(s), err +} + +func (r *ResponseCheckTx) UnmarshalJSON(b []byte) error { + reader := bytes.NewBuffer(b) + return jsonpbUnmarshaller.Unmarshal(reader, r) +} + +func (r *ResponseDeliverTx) MarshalJSON() ([]byte, error) { + s, err := jsonpbMarshaller.MarshalToString(r) + return []byte(s), err +} + +func (r *ResponseDeliverTx) UnmarshalJSON(b []byte) error { + reader := bytes.NewBuffer(b) + return jsonpbUnmarshaller.Unmarshal(reader, r) +} + +func (r *ResponseQuery) MarshalJSON() ([]byte, error) { + s, err := jsonpbMarshaller.MarshalToString(r) + return []byte(s), err +} + +func (r *ResponseQuery) UnmarshalJSON(b []byte) error { + reader := bytes.NewBuffer(b) + return jsonpbUnmarshaller.Unmarshal(reader, r) +} + +func (r *ResponseCommit) MarshalJSON() ([]byte, error) { + s, err := jsonpbMarshaller.MarshalToString(r) + return []byte(s), err +} + +func (r *ResponseCommit) UnmarshalJSON(b []byte) error { + reader := bytes.NewBuffer(b) + return jsonpbUnmarshaller.Unmarshal(reader, r) +} + +// Some compile time assertions to ensure we don't +// have accidental runtime surprises later on. + +// jsonEncodingRoundTripper ensures that asserted +// interfaces implement both MarshalJSON and UnmarshalJSON +type jsonRoundTripper interface { + json.Marshaler + json.Unmarshaler +} + +var _ jsonRoundTripper = (*ResponseCommit)(nil) +var _ jsonRoundTripper = (*ResponseQuery)(nil) +var _ jsonRoundTripper = (*ResponseDeliverTx)(nil) +var _ jsonRoundTripper = (*ResponseCheckTx)(nil) +var _ jsonRoundTripper = (*ResponseSetOption)(nil) diff --git a/abci/types/types.pb.go b/abci/types/types.pb.go new file mode 100644 index 000000000..8135db50f --- /dev/null +++ b/abci/types/types.pb.go @@ -0,0 +1,2455 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: types/types.proto + +/* +Package types is a generated protocol buffer package. + +It is generated from these files: + types/types.proto + +It has these top-level messages: + Request + RequestEcho + RequestFlush + RequestInfo + RequestSetOption + RequestInitChain + RequestQuery + RequestBeginBlock + RequestCheckTx + RequestDeliverTx + RequestEndBlock + RequestCommit + Response + ResponseException + ResponseEcho + ResponseFlush + ResponseInfo + ResponseSetOption + ResponseInitChain + ResponseQuery + ResponseBeginBlock + ResponseCheckTx + ResponseDeliverTx + ResponseEndBlock + ResponseCommit + ConsensusParams + BlockSize + TxSize + BlockGossip + Header + Validator + SigningValidator + PubKey + Evidence +*/ +//nolint: gas +package types + +import proto "github.com/gogo/protobuf/proto" +import fmt "fmt" +import math "math" +import _ "github.com/gogo/protobuf/gogoproto" +import common "github.com/tendermint/tendermint/libs/common" + +import context "golang.org/x/net/context" +import grpc "google.golang.org/grpc" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package + +type Request struct { + // Types that are valid to be assigned to Value: + // *Request_Echo + // *Request_Flush + // *Request_Info + // *Request_SetOption + // *Request_InitChain + // *Request_Query + // *Request_BeginBlock + // *Request_CheckTx + // *Request_DeliverTx + // *Request_EndBlock + // *Request_Commit + Value isRequest_Value `protobuf_oneof:"value"` +} + +func (m *Request) Reset() { *m = Request{} } +func (m *Request) String() string { return proto.CompactTextString(m) } +func (*Request) ProtoMessage() {} +func (*Request) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{0} } + +type isRequest_Value interface { + isRequest_Value() +} + +type Request_Echo struct { + Echo *RequestEcho `protobuf:"bytes,2,opt,name=echo,oneof"` +} +type Request_Flush struct { + Flush *RequestFlush `protobuf:"bytes,3,opt,name=flush,oneof"` +} +type Request_Info struct { + Info *RequestInfo `protobuf:"bytes,4,opt,name=info,oneof"` +} +type Request_SetOption struct { + SetOption *RequestSetOption `protobuf:"bytes,5,opt,name=set_option,json=setOption,oneof"` +} +type Request_InitChain struct { + InitChain *RequestInitChain `protobuf:"bytes,6,opt,name=init_chain,json=initChain,oneof"` +} +type Request_Query struct { + Query *RequestQuery `protobuf:"bytes,7,opt,name=query,oneof"` +} +type Request_BeginBlock struct { + BeginBlock *RequestBeginBlock `protobuf:"bytes,8,opt,name=begin_block,json=beginBlock,oneof"` +} +type Request_CheckTx struct { + CheckTx *RequestCheckTx `protobuf:"bytes,9,opt,name=check_tx,json=checkTx,oneof"` +} +type Request_DeliverTx struct { + DeliverTx *RequestDeliverTx `protobuf:"bytes,19,opt,name=deliver_tx,json=deliverTx,oneof"` +} +type Request_EndBlock struct { + EndBlock *RequestEndBlock `protobuf:"bytes,11,opt,name=end_block,json=endBlock,oneof"` +} +type Request_Commit struct { + Commit *RequestCommit `protobuf:"bytes,12,opt,name=commit,oneof"` +} + +func (*Request_Echo) isRequest_Value() {} +func (*Request_Flush) isRequest_Value() {} +func (*Request_Info) isRequest_Value() {} +func (*Request_SetOption) isRequest_Value() {} +func (*Request_InitChain) isRequest_Value() {} +func (*Request_Query) isRequest_Value() {} +func (*Request_BeginBlock) isRequest_Value() {} +func (*Request_CheckTx) isRequest_Value() {} +func (*Request_DeliverTx) isRequest_Value() {} +func (*Request_EndBlock) isRequest_Value() {} +func (*Request_Commit) isRequest_Value() {} + +func (m *Request) GetValue() isRequest_Value { + if m != nil { + return m.Value + } + return nil +} + +func (m *Request) GetEcho() *RequestEcho { + if x, ok := m.GetValue().(*Request_Echo); ok { + return x.Echo + } + return nil +} + +func (m *Request) GetFlush() *RequestFlush { + if x, ok := m.GetValue().(*Request_Flush); ok { + return x.Flush + } + return nil +} + +func (m *Request) GetInfo() *RequestInfo { + if x, ok := m.GetValue().(*Request_Info); ok { + return x.Info + } + return nil +} + +func (m *Request) GetSetOption() *RequestSetOption { + if x, ok := m.GetValue().(*Request_SetOption); ok { + return x.SetOption + } + return nil +} + +func (m *Request) GetInitChain() *RequestInitChain { + if x, ok := m.GetValue().(*Request_InitChain); ok { + return x.InitChain + } + return nil +} + +func (m *Request) GetQuery() *RequestQuery { + if x, ok := m.GetValue().(*Request_Query); ok { + return x.Query + } + return nil +} + +func (m *Request) GetBeginBlock() *RequestBeginBlock { + if x, ok := m.GetValue().(*Request_BeginBlock); ok { + return x.BeginBlock + } + return nil +} + +func (m *Request) GetCheckTx() *RequestCheckTx { + if x, ok := m.GetValue().(*Request_CheckTx); ok { + return x.CheckTx + } + return nil +} + +func (m *Request) GetDeliverTx() *RequestDeliverTx { + if x, ok := m.GetValue().(*Request_DeliverTx); ok { + return x.DeliverTx + } + return nil +} + +func (m *Request) GetEndBlock() *RequestEndBlock { + if x, ok := m.GetValue().(*Request_EndBlock); ok { + return x.EndBlock + } + return nil +} + +func (m *Request) GetCommit() *RequestCommit { + if x, ok := m.GetValue().(*Request_Commit); ok { + return x.Commit + } + return nil +} + +// XXX_OneofFuncs is for the internal use of the proto package. +func (*Request) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { + return _Request_OneofMarshaler, _Request_OneofUnmarshaler, _Request_OneofSizer, []interface{}{ + (*Request_Echo)(nil), + (*Request_Flush)(nil), + (*Request_Info)(nil), + (*Request_SetOption)(nil), + (*Request_InitChain)(nil), + (*Request_Query)(nil), + (*Request_BeginBlock)(nil), + (*Request_CheckTx)(nil), + (*Request_DeliverTx)(nil), + (*Request_EndBlock)(nil), + (*Request_Commit)(nil), + } +} + +func _Request_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { + m := msg.(*Request) + // value + switch x := m.Value.(type) { + case *Request_Echo: + _ = b.EncodeVarint(2<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Echo); err != nil { + return err + } + case *Request_Flush: + _ = b.EncodeVarint(3<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Flush); err != nil { + return err + } + case *Request_Info: + _ = b.EncodeVarint(4<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Info); err != nil { + return err + } + case *Request_SetOption: + _ = b.EncodeVarint(5<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.SetOption); err != nil { + return err + } + case *Request_InitChain: + _ = b.EncodeVarint(6<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.InitChain); err != nil { + return err + } + case *Request_Query: + _ = b.EncodeVarint(7<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Query); err != nil { + return err + } + case *Request_BeginBlock: + _ = b.EncodeVarint(8<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.BeginBlock); err != nil { + return err + } + case *Request_CheckTx: + _ = b.EncodeVarint(9<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.CheckTx); err != nil { + return err + } + case *Request_DeliverTx: + _ = b.EncodeVarint(19<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.DeliverTx); err != nil { + return err + } + case *Request_EndBlock: + _ = b.EncodeVarint(11<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.EndBlock); err != nil { + return err + } + case *Request_Commit: + _ = b.EncodeVarint(12<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Commit); err != nil { + return err + } + case nil: + default: + return fmt.Errorf("Request.Value has unexpected type %T", x) + } + return nil +} + +func _Request_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { + m := msg.(*Request) + switch tag { + case 2: // value.echo + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(RequestEcho) + err := b.DecodeMessage(msg) + m.Value = &Request_Echo{msg} + return true, err + case 3: // value.flush + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(RequestFlush) + err := b.DecodeMessage(msg) + m.Value = &Request_Flush{msg} + return true, err + case 4: // value.info + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(RequestInfo) + err := b.DecodeMessage(msg) + m.Value = &Request_Info{msg} + return true, err + case 5: // value.set_option + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(RequestSetOption) + err := b.DecodeMessage(msg) + m.Value = &Request_SetOption{msg} + return true, err + case 6: // value.init_chain + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(RequestInitChain) + err := b.DecodeMessage(msg) + m.Value = &Request_InitChain{msg} + return true, err + case 7: // value.query + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(RequestQuery) + err := b.DecodeMessage(msg) + m.Value = &Request_Query{msg} + return true, err + case 8: // value.begin_block + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(RequestBeginBlock) + err := b.DecodeMessage(msg) + m.Value = &Request_BeginBlock{msg} + return true, err + case 9: // value.check_tx + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(RequestCheckTx) + err := b.DecodeMessage(msg) + m.Value = &Request_CheckTx{msg} + return true, err + case 19: // value.deliver_tx + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(RequestDeliverTx) + err := b.DecodeMessage(msg) + m.Value = &Request_DeliverTx{msg} + return true, err + case 11: // value.end_block + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(RequestEndBlock) + err := b.DecodeMessage(msg) + m.Value = &Request_EndBlock{msg} + return true, err + case 12: // value.commit + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(RequestCommit) + err := b.DecodeMessage(msg) + m.Value = &Request_Commit{msg} + return true, err + default: + return false, nil + } +} + +func _Request_OneofSizer(msg proto.Message) (n int) { + m := msg.(*Request) + // value + switch x := m.Value.(type) { + case *Request_Echo: + s := proto.Size(x.Echo) + n += proto.SizeVarint(2<<3 | proto.WireBytes) + n += proto.SizeVarint(uint64(s)) + n += s + case *Request_Flush: + s := proto.Size(x.Flush) + n += proto.SizeVarint(3<<3 | proto.WireBytes) + n += proto.SizeVarint(uint64(s)) + n += s + case *Request_Info: + s := proto.Size(x.Info) + n += proto.SizeVarint(4<<3 | proto.WireBytes) + n += proto.SizeVarint(uint64(s)) + n += s + case *Request_SetOption: + s := proto.Size(x.SetOption) + n += proto.SizeVarint(5<<3 | proto.WireBytes) + n += proto.SizeVarint(uint64(s)) + n += s + case *Request_InitChain: + s := proto.Size(x.InitChain) + n += proto.SizeVarint(6<<3 | proto.WireBytes) + n += proto.SizeVarint(uint64(s)) + n += s + case *Request_Query: + s := proto.Size(x.Query) + n += proto.SizeVarint(7<<3 | proto.WireBytes) + n += proto.SizeVarint(uint64(s)) + n += s + case *Request_BeginBlock: + s := proto.Size(x.BeginBlock) + n += proto.SizeVarint(8<<3 | proto.WireBytes) + n += proto.SizeVarint(uint64(s)) + n += s + case *Request_CheckTx: + s := proto.Size(x.CheckTx) + n += proto.SizeVarint(9<<3 | proto.WireBytes) + n += proto.SizeVarint(uint64(s)) + n += s + case *Request_DeliverTx: + s := proto.Size(x.DeliverTx) + n += proto.SizeVarint(19<<3 | proto.WireBytes) + n += proto.SizeVarint(uint64(s)) + n += s + case *Request_EndBlock: + s := proto.Size(x.EndBlock) + n += proto.SizeVarint(11<<3 | proto.WireBytes) + n += proto.SizeVarint(uint64(s)) + n += s + case *Request_Commit: + s := proto.Size(x.Commit) + n += proto.SizeVarint(12<<3 | proto.WireBytes) + n += proto.SizeVarint(uint64(s)) + n += s + case nil: + default: + panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) + } + return n +} + +type RequestEcho struct { + Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"` +} + +func (m *RequestEcho) Reset() { *m = RequestEcho{} } +func (m *RequestEcho) String() string { return proto.CompactTextString(m) } +func (*RequestEcho) ProtoMessage() {} +func (*RequestEcho) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{1} } + +func (m *RequestEcho) GetMessage() string { + if m != nil { + return m.Message + } + return "" +} + +type RequestFlush struct { +} + +func (m *RequestFlush) Reset() { *m = RequestFlush{} } +func (m *RequestFlush) String() string { return proto.CompactTextString(m) } +func (*RequestFlush) ProtoMessage() {} +func (*RequestFlush) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{2} } + +type RequestInfo struct { + Version string `protobuf:"bytes,1,opt,name=version,proto3" json:"version,omitempty"` +} + +func (m *RequestInfo) Reset() { *m = RequestInfo{} } +func (m *RequestInfo) String() string { return proto.CompactTextString(m) } +func (*RequestInfo) ProtoMessage() {} +func (*RequestInfo) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{3} } + +func (m *RequestInfo) GetVersion() string { + if m != nil { + return m.Version + } + return "" +} + +// nondeterministic +type RequestSetOption struct { + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` +} + +func (m *RequestSetOption) Reset() { *m = RequestSetOption{} } +func (m *RequestSetOption) String() string { return proto.CompactTextString(m) } +func (*RequestSetOption) ProtoMessage() {} +func (*RequestSetOption) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{4} } + +func (m *RequestSetOption) GetKey() string { + if m != nil { + return m.Key + } + return "" +} + +func (m *RequestSetOption) GetValue() string { + if m != nil { + return m.Value + } + return "" +} + +type RequestInitChain struct { + Time int64 `protobuf:"varint,1,opt,name=time,proto3" json:"time,omitempty"` + ChainId string `protobuf:"bytes,2,opt,name=chain_id,json=chainId,proto3" json:"chain_id,omitempty"` + ConsensusParams *ConsensusParams `protobuf:"bytes,3,opt,name=consensus_params,json=consensusParams" json:"consensus_params,omitempty"` + Validators []Validator `protobuf:"bytes,4,rep,name=validators" json:"validators"` + AppStateBytes []byte `protobuf:"bytes,5,opt,name=app_state_bytes,json=appStateBytes,proto3" json:"app_state_bytes,omitempty"` +} + +func (m *RequestInitChain) Reset() { *m = RequestInitChain{} } +func (m *RequestInitChain) String() string { return proto.CompactTextString(m) } +func (*RequestInitChain) ProtoMessage() {} +func (*RequestInitChain) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{5} } + +func (m *RequestInitChain) GetTime() int64 { + if m != nil { + return m.Time + } + return 0 +} + +func (m *RequestInitChain) GetChainId() string { + if m != nil { + return m.ChainId + } + return "" +} + +func (m *RequestInitChain) GetConsensusParams() *ConsensusParams { + if m != nil { + return m.ConsensusParams + } + return nil +} + +func (m *RequestInitChain) GetValidators() []Validator { + if m != nil { + return m.Validators + } + return nil +} + +func (m *RequestInitChain) GetAppStateBytes() []byte { + if m != nil { + return m.AppStateBytes + } + return nil +} + +type RequestQuery struct { + Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` + Path string `protobuf:"bytes,2,opt,name=path,proto3" json:"path,omitempty"` + Height int64 `protobuf:"varint,3,opt,name=height,proto3" json:"height,omitempty"` + Prove bool `protobuf:"varint,4,opt,name=prove,proto3" json:"prove,omitempty"` +} + +func (m *RequestQuery) Reset() { *m = RequestQuery{} } +func (m *RequestQuery) String() string { return proto.CompactTextString(m) } +func (*RequestQuery) ProtoMessage() {} +func (*RequestQuery) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{6} } + +func (m *RequestQuery) GetData() []byte { + if m != nil { + return m.Data + } + return nil +} + +func (m *RequestQuery) GetPath() string { + if m != nil { + return m.Path + } + return "" +} + +func (m *RequestQuery) GetHeight() int64 { + if m != nil { + return m.Height + } + return 0 +} + +func (m *RequestQuery) GetProve() bool { + if m != nil { + return m.Prove + } + return false +} + +type RequestBeginBlock struct { + Hash []byte `protobuf:"bytes,1,opt,name=hash,proto3" json:"hash,omitempty"` + Header Header `protobuf:"bytes,2,opt,name=header" json:"header"` + Validators []SigningValidator `protobuf:"bytes,3,rep,name=validators" json:"validators"` + ByzantineValidators []Evidence `protobuf:"bytes,4,rep,name=byzantine_validators,json=byzantineValidators" json:"byzantine_validators"` +} + +func (m *RequestBeginBlock) Reset() { *m = RequestBeginBlock{} } +func (m *RequestBeginBlock) String() string { return proto.CompactTextString(m) } +func (*RequestBeginBlock) ProtoMessage() {} +func (*RequestBeginBlock) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{7} } + +func (m *RequestBeginBlock) GetHash() []byte { + if m != nil { + return m.Hash + } + return nil +} + +func (m *RequestBeginBlock) GetHeader() Header { + if m != nil { + return m.Header + } + return Header{} +} + +func (m *RequestBeginBlock) GetValidators() []SigningValidator { + if m != nil { + return m.Validators + } + return nil +} + +func (m *RequestBeginBlock) GetByzantineValidators() []Evidence { + if m != nil { + return m.ByzantineValidators + } + return nil +} + +type RequestCheckTx struct { + Tx []byte `protobuf:"bytes,1,opt,name=tx,proto3" json:"tx,omitempty"` +} + +func (m *RequestCheckTx) Reset() { *m = RequestCheckTx{} } +func (m *RequestCheckTx) String() string { return proto.CompactTextString(m) } +func (*RequestCheckTx) ProtoMessage() {} +func (*RequestCheckTx) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{8} } + +func (m *RequestCheckTx) GetTx() []byte { + if m != nil { + return m.Tx + } + return nil +} + +type RequestDeliverTx struct { + Tx []byte `protobuf:"bytes,1,opt,name=tx,proto3" json:"tx,omitempty"` +} + +func (m *RequestDeliverTx) Reset() { *m = RequestDeliverTx{} } +func (m *RequestDeliverTx) String() string { return proto.CompactTextString(m) } +func (*RequestDeliverTx) ProtoMessage() {} +func (*RequestDeliverTx) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{9} } + +func (m *RequestDeliverTx) GetTx() []byte { + if m != nil { + return m.Tx + } + return nil +} + +type RequestEndBlock struct { + Height int64 `protobuf:"varint,1,opt,name=height,proto3" json:"height,omitempty"` +} + +func (m *RequestEndBlock) Reset() { *m = RequestEndBlock{} } +func (m *RequestEndBlock) String() string { return proto.CompactTextString(m) } +func (*RequestEndBlock) ProtoMessage() {} +func (*RequestEndBlock) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{10} } + +func (m *RequestEndBlock) GetHeight() int64 { + if m != nil { + return m.Height + } + return 0 +} + +type RequestCommit struct { +} + +func (m *RequestCommit) Reset() { *m = RequestCommit{} } +func (m *RequestCommit) String() string { return proto.CompactTextString(m) } +func (*RequestCommit) ProtoMessage() {} +func (*RequestCommit) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{11} } + +type Response struct { + // Types that are valid to be assigned to Value: + // *Response_Exception + // *Response_Echo + // *Response_Flush + // *Response_Info + // *Response_SetOption + // *Response_InitChain + // *Response_Query + // *Response_BeginBlock + // *Response_CheckTx + // *Response_DeliverTx + // *Response_EndBlock + // *Response_Commit + Value isResponse_Value `protobuf_oneof:"value"` +} + +func (m *Response) Reset() { *m = Response{} } +func (m *Response) String() string { return proto.CompactTextString(m) } +func (*Response) ProtoMessage() {} +func (*Response) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{12} } + +type isResponse_Value interface { + isResponse_Value() +} + +type Response_Exception struct { + Exception *ResponseException `protobuf:"bytes,1,opt,name=exception,oneof"` +} +type Response_Echo struct { + Echo *ResponseEcho `protobuf:"bytes,2,opt,name=echo,oneof"` +} +type Response_Flush struct { + Flush *ResponseFlush `protobuf:"bytes,3,opt,name=flush,oneof"` +} +type Response_Info struct { + Info *ResponseInfo `protobuf:"bytes,4,opt,name=info,oneof"` +} +type Response_SetOption struct { + SetOption *ResponseSetOption `protobuf:"bytes,5,opt,name=set_option,json=setOption,oneof"` +} +type Response_InitChain struct { + InitChain *ResponseInitChain `protobuf:"bytes,6,opt,name=init_chain,json=initChain,oneof"` +} +type Response_Query struct { + Query *ResponseQuery `protobuf:"bytes,7,opt,name=query,oneof"` +} +type Response_BeginBlock struct { + BeginBlock *ResponseBeginBlock `protobuf:"bytes,8,opt,name=begin_block,json=beginBlock,oneof"` +} +type Response_CheckTx struct { + CheckTx *ResponseCheckTx `protobuf:"bytes,9,opt,name=check_tx,json=checkTx,oneof"` +} +type Response_DeliverTx struct { + DeliverTx *ResponseDeliverTx `protobuf:"bytes,10,opt,name=deliver_tx,json=deliverTx,oneof"` +} +type Response_EndBlock struct { + EndBlock *ResponseEndBlock `protobuf:"bytes,11,opt,name=end_block,json=endBlock,oneof"` +} +type Response_Commit struct { + Commit *ResponseCommit `protobuf:"bytes,12,opt,name=commit,oneof"` +} + +func (*Response_Exception) isResponse_Value() {} +func (*Response_Echo) isResponse_Value() {} +func (*Response_Flush) isResponse_Value() {} +func (*Response_Info) isResponse_Value() {} +func (*Response_SetOption) isResponse_Value() {} +func (*Response_InitChain) isResponse_Value() {} +func (*Response_Query) isResponse_Value() {} +func (*Response_BeginBlock) isResponse_Value() {} +func (*Response_CheckTx) isResponse_Value() {} +func (*Response_DeliverTx) isResponse_Value() {} +func (*Response_EndBlock) isResponse_Value() {} +func (*Response_Commit) isResponse_Value() {} + +func (m *Response) GetValue() isResponse_Value { + if m != nil { + return m.Value + } + return nil +} + +func (m *Response) GetException() *ResponseException { + if x, ok := m.GetValue().(*Response_Exception); ok { + return x.Exception + } + return nil +} + +func (m *Response) GetEcho() *ResponseEcho { + if x, ok := m.GetValue().(*Response_Echo); ok { + return x.Echo + } + return nil +} + +func (m *Response) GetFlush() *ResponseFlush { + if x, ok := m.GetValue().(*Response_Flush); ok { + return x.Flush + } + return nil +} + +func (m *Response) GetInfo() *ResponseInfo { + if x, ok := m.GetValue().(*Response_Info); ok { + return x.Info + } + return nil +} + +func (m *Response) GetSetOption() *ResponseSetOption { + if x, ok := m.GetValue().(*Response_SetOption); ok { + return x.SetOption + } + return nil +} + +func (m *Response) GetInitChain() *ResponseInitChain { + if x, ok := m.GetValue().(*Response_InitChain); ok { + return x.InitChain + } + return nil +} + +func (m *Response) GetQuery() *ResponseQuery { + if x, ok := m.GetValue().(*Response_Query); ok { + return x.Query + } + return nil +} + +func (m *Response) GetBeginBlock() *ResponseBeginBlock { + if x, ok := m.GetValue().(*Response_BeginBlock); ok { + return x.BeginBlock + } + return nil +} + +func (m *Response) GetCheckTx() *ResponseCheckTx { + if x, ok := m.GetValue().(*Response_CheckTx); ok { + return x.CheckTx + } + return nil +} + +func (m *Response) GetDeliverTx() *ResponseDeliverTx { + if x, ok := m.GetValue().(*Response_DeliverTx); ok { + return x.DeliverTx + } + return nil +} + +func (m *Response) GetEndBlock() *ResponseEndBlock { + if x, ok := m.GetValue().(*Response_EndBlock); ok { + return x.EndBlock + } + return nil +} + +func (m *Response) GetCommit() *ResponseCommit { + if x, ok := m.GetValue().(*Response_Commit); ok { + return x.Commit + } + return nil +} + +// XXX_OneofFuncs is for the internal use of the proto package. +func (*Response) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { + return _Response_OneofMarshaler, _Response_OneofUnmarshaler, _Response_OneofSizer, []interface{}{ + (*Response_Exception)(nil), + (*Response_Echo)(nil), + (*Response_Flush)(nil), + (*Response_Info)(nil), + (*Response_SetOption)(nil), + (*Response_InitChain)(nil), + (*Response_Query)(nil), + (*Response_BeginBlock)(nil), + (*Response_CheckTx)(nil), + (*Response_DeliverTx)(nil), + (*Response_EndBlock)(nil), + (*Response_Commit)(nil), + } +} + +func _Response_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { + m := msg.(*Response) + // value + switch x := m.Value.(type) { + case *Response_Exception: + _ = b.EncodeVarint(1<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Exception); err != nil { + return err + } + case *Response_Echo: + _ = b.EncodeVarint(2<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Echo); err != nil { + return err + } + case *Response_Flush: + _ = b.EncodeVarint(3<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Flush); err != nil { + return err + } + case *Response_Info: + _ = b.EncodeVarint(4<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Info); err != nil { + return err + } + case *Response_SetOption: + _ = b.EncodeVarint(5<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.SetOption); err != nil { + return err + } + case *Response_InitChain: + _ = b.EncodeVarint(6<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.InitChain); err != nil { + return err + } + case *Response_Query: + _ = b.EncodeVarint(7<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Query); err != nil { + return err + } + case *Response_BeginBlock: + _ = b.EncodeVarint(8<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.BeginBlock); err != nil { + return err + } + case *Response_CheckTx: + _ = b.EncodeVarint(9<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.CheckTx); err != nil { + return err + } + case *Response_DeliverTx: + _ = b.EncodeVarint(10<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.DeliverTx); err != nil { + return err + } + case *Response_EndBlock: + _ = b.EncodeVarint(11<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.EndBlock); err != nil { + return err + } + case *Response_Commit: + _ = b.EncodeVarint(12<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Commit); err != nil { + return err + } + case nil: + default: + return fmt.Errorf("Response.Value has unexpected type %T", x) + } + return nil +} + +func _Response_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { + m := msg.(*Response) + switch tag { + case 1: // value.exception + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(ResponseException) + err := b.DecodeMessage(msg) + m.Value = &Response_Exception{msg} + return true, err + case 2: // value.echo + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(ResponseEcho) + err := b.DecodeMessage(msg) + m.Value = &Response_Echo{msg} + return true, err + case 3: // value.flush + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(ResponseFlush) + err := b.DecodeMessage(msg) + m.Value = &Response_Flush{msg} + return true, err + case 4: // value.info + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(ResponseInfo) + err := b.DecodeMessage(msg) + m.Value = &Response_Info{msg} + return true, err + case 5: // value.set_option + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(ResponseSetOption) + err := b.DecodeMessage(msg) + m.Value = &Response_SetOption{msg} + return true, err + case 6: // value.init_chain + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(ResponseInitChain) + err := b.DecodeMessage(msg) + m.Value = &Response_InitChain{msg} + return true, err + case 7: // value.query + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(ResponseQuery) + err := b.DecodeMessage(msg) + m.Value = &Response_Query{msg} + return true, err + case 8: // value.begin_block + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(ResponseBeginBlock) + err := b.DecodeMessage(msg) + m.Value = &Response_BeginBlock{msg} + return true, err + case 9: // value.check_tx + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(ResponseCheckTx) + err := b.DecodeMessage(msg) + m.Value = &Response_CheckTx{msg} + return true, err + case 10: // value.deliver_tx + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(ResponseDeliverTx) + err := b.DecodeMessage(msg) + m.Value = &Response_DeliverTx{msg} + return true, err + case 11: // value.end_block + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(ResponseEndBlock) + err := b.DecodeMessage(msg) + m.Value = &Response_EndBlock{msg} + return true, err + case 12: // value.commit + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(ResponseCommit) + err := b.DecodeMessage(msg) + m.Value = &Response_Commit{msg} + return true, err + default: + return false, nil + } +} + +func _Response_OneofSizer(msg proto.Message) (n int) { + m := msg.(*Response) + // value + switch x := m.Value.(type) { + case *Response_Exception: + s := proto.Size(x.Exception) + n += proto.SizeVarint(1<<3 | proto.WireBytes) + n += proto.SizeVarint(uint64(s)) + n += s + case *Response_Echo: + s := proto.Size(x.Echo) + n += proto.SizeVarint(2<<3 | proto.WireBytes) + n += proto.SizeVarint(uint64(s)) + n += s + case *Response_Flush: + s := proto.Size(x.Flush) + n += proto.SizeVarint(3<<3 | proto.WireBytes) + n += proto.SizeVarint(uint64(s)) + n += s + case *Response_Info: + s := proto.Size(x.Info) + n += proto.SizeVarint(4<<3 | proto.WireBytes) + n += proto.SizeVarint(uint64(s)) + n += s + case *Response_SetOption: + s := proto.Size(x.SetOption) + n += proto.SizeVarint(5<<3 | proto.WireBytes) + n += proto.SizeVarint(uint64(s)) + n += s + case *Response_InitChain: + s := proto.Size(x.InitChain) + n += proto.SizeVarint(6<<3 | proto.WireBytes) + n += proto.SizeVarint(uint64(s)) + n += s + case *Response_Query: + s := proto.Size(x.Query) + n += proto.SizeVarint(7<<3 | proto.WireBytes) + n += proto.SizeVarint(uint64(s)) + n += s + case *Response_BeginBlock: + s := proto.Size(x.BeginBlock) + n += proto.SizeVarint(8<<3 | proto.WireBytes) + n += proto.SizeVarint(uint64(s)) + n += s + case *Response_CheckTx: + s := proto.Size(x.CheckTx) + n += proto.SizeVarint(9<<3 | proto.WireBytes) + n += proto.SizeVarint(uint64(s)) + n += s + case *Response_DeliverTx: + s := proto.Size(x.DeliverTx) + n += proto.SizeVarint(10<<3 | proto.WireBytes) + n += proto.SizeVarint(uint64(s)) + n += s + case *Response_EndBlock: + s := proto.Size(x.EndBlock) + n += proto.SizeVarint(11<<3 | proto.WireBytes) + n += proto.SizeVarint(uint64(s)) + n += s + case *Response_Commit: + s := proto.Size(x.Commit) + n += proto.SizeVarint(12<<3 | proto.WireBytes) + n += proto.SizeVarint(uint64(s)) + n += s + case nil: + default: + panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) + } + return n +} + +// nondeterministic +type ResponseException struct { + Error string `protobuf:"bytes,1,opt,name=error,proto3" json:"error,omitempty"` +} + +func (m *ResponseException) Reset() { *m = ResponseException{} } +func (m *ResponseException) String() string { return proto.CompactTextString(m) } +func (*ResponseException) ProtoMessage() {} +func (*ResponseException) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{13} } + +func (m *ResponseException) GetError() string { + if m != nil { + return m.Error + } + return "" +} + +type ResponseEcho struct { + Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"` +} + +func (m *ResponseEcho) Reset() { *m = ResponseEcho{} } +func (m *ResponseEcho) String() string { return proto.CompactTextString(m) } +func (*ResponseEcho) ProtoMessage() {} +func (*ResponseEcho) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{14} } + +func (m *ResponseEcho) GetMessage() string { + if m != nil { + return m.Message + } + return "" +} + +type ResponseFlush struct { +} + +func (m *ResponseFlush) Reset() { *m = ResponseFlush{} } +func (m *ResponseFlush) String() string { return proto.CompactTextString(m) } +func (*ResponseFlush) ProtoMessage() {} +func (*ResponseFlush) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{15} } + +type ResponseInfo struct { + Data string `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` + Version string `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` + LastBlockHeight int64 `protobuf:"varint,3,opt,name=last_block_height,json=lastBlockHeight,proto3" json:"last_block_height,omitempty"` + LastBlockAppHash []byte `protobuf:"bytes,4,opt,name=last_block_app_hash,json=lastBlockAppHash,proto3" json:"last_block_app_hash,omitempty"` +} + +func (m *ResponseInfo) Reset() { *m = ResponseInfo{} } +func (m *ResponseInfo) String() string { return proto.CompactTextString(m) } +func (*ResponseInfo) ProtoMessage() {} +func (*ResponseInfo) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{16} } + +func (m *ResponseInfo) GetData() string { + if m != nil { + return m.Data + } + return "" +} + +func (m *ResponseInfo) GetVersion() string { + if m != nil { + return m.Version + } + return "" +} + +func (m *ResponseInfo) GetLastBlockHeight() int64 { + if m != nil { + return m.LastBlockHeight + } + return 0 +} + +func (m *ResponseInfo) GetLastBlockAppHash() []byte { + if m != nil { + return m.LastBlockAppHash + } + return nil +} + +// nondeterministic +type ResponseSetOption struct { + Code uint32 `protobuf:"varint,1,opt,name=code,proto3" json:"code,omitempty"` + // bytes data = 2; + Log string `protobuf:"bytes,3,opt,name=log,proto3" json:"log,omitempty"` + Info string `protobuf:"bytes,4,opt,name=info,proto3" json:"info,omitempty"` +} + +func (m *ResponseSetOption) Reset() { *m = ResponseSetOption{} } +func (m *ResponseSetOption) String() string { return proto.CompactTextString(m) } +func (*ResponseSetOption) ProtoMessage() {} +func (*ResponseSetOption) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{17} } + +func (m *ResponseSetOption) GetCode() uint32 { + if m != nil { + return m.Code + } + return 0 +} + +func (m *ResponseSetOption) GetLog() string { + if m != nil { + return m.Log + } + return "" +} + +func (m *ResponseSetOption) GetInfo() string { + if m != nil { + return m.Info + } + return "" +} + +type ResponseInitChain struct { + ConsensusParams *ConsensusParams `protobuf:"bytes,1,opt,name=consensus_params,json=consensusParams" json:"consensus_params,omitempty"` + Validators []Validator `protobuf:"bytes,2,rep,name=validators" json:"validators"` +} + +func (m *ResponseInitChain) Reset() { *m = ResponseInitChain{} } +func (m *ResponseInitChain) String() string { return proto.CompactTextString(m) } +func (*ResponseInitChain) ProtoMessage() {} +func (*ResponseInitChain) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{18} } + +func (m *ResponseInitChain) GetConsensusParams() *ConsensusParams { + if m != nil { + return m.ConsensusParams + } + return nil +} + +func (m *ResponseInitChain) GetValidators() []Validator { + if m != nil { + return m.Validators + } + return nil +} + +type ResponseQuery struct { + Code uint32 `protobuf:"varint,1,opt,name=code,proto3" json:"code,omitempty"` + // bytes data = 2; // use "value" instead. + Log string `protobuf:"bytes,3,opt,name=log,proto3" json:"log,omitempty"` + Info string `protobuf:"bytes,4,opt,name=info,proto3" json:"info,omitempty"` + Index int64 `protobuf:"varint,5,opt,name=index,proto3" json:"index,omitempty"` + Key []byte `protobuf:"bytes,6,opt,name=key,proto3" json:"key,omitempty"` + Value []byte `protobuf:"bytes,7,opt,name=value,proto3" json:"value,omitempty"` + Proof []byte `protobuf:"bytes,8,opt,name=proof,proto3" json:"proof,omitempty"` + Height int64 `protobuf:"varint,9,opt,name=height,proto3" json:"height,omitempty"` +} + +func (m *ResponseQuery) Reset() { *m = ResponseQuery{} } +func (m *ResponseQuery) String() string { return proto.CompactTextString(m) } +func (*ResponseQuery) ProtoMessage() {} +func (*ResponseQuery) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{19} } + +func (m *ResponseQuery) GetCode() uint32 { + if m != nil { + return m.Code + } + return 0 +} + +func (m *ResponseQuery) GetLog() string { + if m != nil { + return m.Log + } + return "" +} + +func (m *ResponseQuery) GetInfo() string { + if m != nil { + return m.Info + } + return "" +} + +func (m *ResponseQuery) GetIndex() int64 { + if m != nil { + return m.Index + } + return 0 +} + +func (m *ResponseQuery) GetKey() []byte { + if m != nil { + return m.Key + } + return nil +} + +func (m *ResponseQuery) GetValue() []byte { + if m != nil { + return m.Value + } + return nil +} + +func (m *ResponseQuery) GetProof() []byte { + if m != nil { + return m.Proof + } + return nil +} + +func (m *ResponseQuery) GetHeight() int64 { + if m != nil { + return m.Height + } + return 0 +} + +type ResponseBeginBlock struct { + Tags []common.KVPair `protobuf:"bytes,1,rep,name=tags" json:"tags,omitempty"` +} + +func (m *ResponseBeginBlock) Reset() { *m = ResponseBeginBlock{} } +func (m *ResponseBeginBlock) String() string { return proto.CompactTextString(m) } +func (*ResponseBeginBlock) ProtoMessage() {} +func (*ResponseBeginBlock) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{20} } + +func (m *ResponseBeginBlock) GetTags() []common.KVPair { + if m != nil { + return m.Tags + } + return nil +} + +type ResponseCheckTx struct { + Code uint32 `protobuf:"varint,1,opt,name=code,proto3" json:"code,omitempty"` + Data []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` + Log string `protobuf:"bytes,3,opt,name=log,proto3" json:"log,omitempty"` + Info string `protobuf:"bytes,4,opt,name=info,proto3" json:"info,omitempty"` + GasWanted int64 `protobuf:"varint,5,opt,name=gas_wanted,json=gasWanted,proto3" json:"gas_wanted,omitempty"` + GasUsed int64 `protobuf:"varint,6,opt,name=gas_used,json=gasUsed,proto3" json:"gas_used,omitempty"` + Tags []common.KVPair `protobuf:"bytes,7,rep,name=tags" json:"tags,omitempty"` + Fee common.KI64Pair `protobuf:"bytes,8,opt,name=fee" json:"fee"` +} + +func (m *ResponseCheckTx) Reset() { *m = ResponseCheckTx{} } +func (m *ResponseCheckTx) String() string { return proto.CompactTextString(m) } +func (*ResponseCheckTx) ProtoMessage() {} +func (*ResponseCheckTx) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{21} } + +func (m *ResponseCheckTx) GetCode() uint32 { + if m != nil { + return m.Code + } + return 0 +} + +func (m *ResponseCheckTx) GetData() []byte { + if m != nil { + return m.Data + } + return nil +} + +func (m *ResponseCheckTx) GetLog() string { + if m != nil { + return m.Log + } + return "" +} + +func (m *ResponseCheckTx) GetInfo() string { + if m != nil { + return m.Info + } + return "" +} + +func (m *ResponseCheckTx) GetGasWanted() int64 { + if m != nil { + return m.GasWanted + } + return 0 +} + +func (m *ResponseCheckTx) GetGasUsed() int64 { + if m != nil { + return m.GasUsed + } + return 0 +} + +func (m *ResponseCheckTx) GetTags() []common.KVPair { + if m != nil { + return m.Tags + } + return nil +} + +func (m *ResponseCheckTx) GetFee() common.KI64Pair { + if m != nil { + return m.Fee + } + return common.KI64Pair{} +} + +type ResponseDeliverTx struct { + Code uint32 `protobuf:"varint,1,opt,name=code,proto3" json:"code,omitempty"` + Data []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` + Log string `protobuf:"bytes,3,opt,name=log,proto3" json:"log,omitempty"` + Info string `protobuf:"bytes,4,opt,name=info,proto3" json:"info,omitempty"` + GasWanted int64 `protobuf:"varint,5,opt,name=gas_wanted,json=gasWanted,proto3" json:"gas_wanted,omitempty"` + GasUsed int64 `protobuf:"varint,6,opt,name=gas_used,json=gasUsed,proto3" json:"gas_used,omitempty"` + Tags []common.KVPair `protobuf:"bytes,7,rep,name=tags" json:"tags,omitempty"` + Fee common.KI64Pair `protobuf:"bytes,8,opt,name=fee" json:"fee"` +} + +func (m *ResponseDeliverTx) Reset() { *m = ResponseDeliverTx{} } +func (m *ResponseDeliverTx) String() string { return proto.CompactTextString(m) } +func (*ResponseDeliverTx) ProtoMessage() {} +func (*ResponseDeliverTx) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{22} } + +func (m *ResponseDeliverTx) GetCode() uint32 { + if m != nil { + return m.Code + } + return 0 +} + +func (m *ResponseDeliverTx) GetData() []byte { + if m != nil { + return m.Data + } + return nil +} + +func (m *ResponseDeliverTx) GetLog() string { + if m != nil { + return m.Log + } + return "" +} + +func (m *ResponseDeliverTx) GetInfo() string { + if m != nil { + return m.Info + } + return "" +} + +func (m *ResponseDeliverTx) GetGasWanted() int64 { + if m != nil { + return m.GasWanted + } + return 0 +} + +func (m *ResponseDeliverTx) GetGasUsed() int64 { + if m != nil { + return m.GasUsed + } + return 0 +} + +func (m *ResponseDeliverTx) GetTags() []common.KVPair { + if m != nil { + return m.Tags + } + return nil +} + +func (m *ResponseDeliverTx) GetFee() common.KI64Pair { + if m != nil { + return m.Fee + } + return common.KI64Pair{} +} + +type ResponseEndBlock struct { + ValidatorUpdates []Validator `protobuf:"bytes,1,rep,name=validator_updates,json=validatorUpdates" json:"validator_updates"` + ConsensusParamUpdates *ConsensusParams `protobuf:"bytes,2,opt,name=consensus_param_updates,json=consensusParamUpdates" json:"consensus_param_updates,omitempty"` + Tags []common.KVPair `protobuf:"bytes,3,rep,name=tags" json:"tags,omitempty"` +} + +func (m *ResponseEndBlock) Reset() { *m = ResponseEndBlock{} } +func (m *ResponseEndBlock) String() string { return proto.CompactTextString(m) } +func (*ResponseEndBlock) ProtoMessage() {} +func (*ResponseEndBlock) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{23} } + +func (m *ResponseEndBlock) GetValidatorUpdates() []Validator { + if m != nil { + return m.ValidatorUpdates + } + return nil +} + +func (m *ResponseEndBlock) GetConsensusParamUpdates() *ConsensusParams { + if m != nil { + return m.ConsensusParamUpdates + } + return nil +} + +func (m *ResponseEndBlock) GetTags() []common.KVPair { + if m != nil { + return m.Tags + } + return nil +} + +type ResponseCommit struct { + // reserve 1 + Data []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` +} + +func (m *ResponseCommit) Reset() { *m = ResponseCommit{} } +func (m *ResponseCommit) String() string { return proto.CompactTextString(m) } +func (*ResponseCommit) ProtoMessage() {} +func (*ResponseCommit) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{24} } + +func (m *ResponseCommit) GetData() []byte { + if m != nil { + return m.Data + } + return nil +} + +// ConsensusParams contains all consensus-relevant parameters +// that can be adjusted by the abci app +type ConsensusParams struct { + BlockSize *BlockSize `protobuf:"bytes,1,opt,name=block_size,json=blockSize" json:"block_size,omitempty"` + TxSize *TxSize `protobuf:"bytes,2,opt,name=tx_size,json=txSize" json:"tx_size,omitempty"` + BlockGossip *BlockGossip `protobuf:"bytes,3,opt,name=block_gossip,json=blockGossip" json:"block_gossip,omitempty"` +} + +func (m *ConsensusParams) Reset() { *m = ConsensusParams{} } +func (m *ConsensusParams) String() string { return proto.CompactTextString(m) } +func (*ConsensusParams) ProtoMessage() {} +func (*ConsensusParams) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{25} } + +func (m *ConsensusParams) GetBlockSize() *BlockSize { + if m != nil { + return m.BlockSize + } + return nil +} + +func (m *ConsensusParams) GetTxSize() *TxSize { + if m != nil { + return m.TxSize + } + return nil +} + +func (m *ConsensusParams) GetBlockGossip() *BlockGossip { + if m != nil { + return m.BlockGossip + } + return nil +} + +// BlockSize contain limits on the block size. +type BlockSize struct { + MaxBytes int32 `protobuf:"varint,1,opt,name=max_bytes,json=maxBytes,proto3" json:"max_bytes,omitempty"` + MaxTxs int32 `protobuf:"varint,2,opt,name=max_txs,json=maxTxs,proto3" json:"max_txs,omitempty"` + MaxGas int64 `protobuf:"varint,3,opt,name=max_gas,json=maxGas,proto3" json:"max_gas,omitempty"` +} + +func (m *BlockSize) Reset() { *m = BlockSize{} } +func (m *BlockSize) String() string { return proto.CompactTextString(m) } +func (*BlockSize) ProtoMessage() {} +func (*BlockSize) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{26} } + +func (m *BlockSize) GetMaxBytes() int32 { + if m != nil { + return m.MaxBytes + } + return 0 +} + +func (m *BlockSize) GetMaxTxs() int32 { + if m != nil { + return m.MaxTxs + } + return 0 +} + +func (m *BlockSize) GetMaxGas() int64 { + if m != nil { + return m.MaxGas + } + return 0 +} + +// TxSize contain limits on the tx size. +type TxSize struct { + MaxBytes int32 `protobuf:"varint,1,opt,name=max_bytes,json=maxBytes,proto3" json:"max_bytes,omitempty"` + MaxGas int64 `protobuf:"varint,2,opt,name=max_gas,json=maxGas,proto3" json:"max_gas,omitempty"` +} + +func (m *TxSize) Reset() { *m = TxSize{} } +func (m *TxSize) String() string { return proto.CompactTextString(m) } +func (*TxSize) ProtoMessage() {} +func (*TxSize) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{27} } + +func (m *TxSize) GetMaxBytes() int32 { + if m != nil { + return m.MaxBytes + } + return 0 +} + +func (m *TxSize) GetMaxGas() int64 { + if m != nil { + return m.MaxGas + } + return 0 +} + +// BlockGossip determine consensus critical +// elements of how blocks are gossiped +type BlockGossip struct { + // Note: must not be 0 + BlockPartSizeBytes int32 `protobuf:"varint,1,opt,name=block_part_size_bytes,json=blockPartSizeBytes,proto3" json:"block_part_size_bytes,omitempty"` +} + +func (m *BlockGossip) Reset() { *m = BlockGossip{} } +func (m *BlockGossip) String() string { return proto.CompactTextString(m) } +func (*BlockGossip) ProtoMessage() {} +func (*BlockGossip) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{28} } + +func (m *BlockGossip) GetBlockPartSizeBytes() int32 { + if m != nil { + return m.BlockPartSizeBytes + } + return 0 +} + +// just the minimum the app might need +type Header struct { + // basics + ChainID string `protobuf:"bytes,1,opt,name=chain_id,json=chainId,proto3" json:"chain_id,omitempty"` + Height int64 `protobuf:"varint,2,opt,name=height,proto3" json:"height,omitempty"` + Time int64 `protobuf:"varint,3,opt,name=time,proto3" json:"time,omitempty"` + // txs + NumTxs int32 `protobuf:"varint,4,opt,name=num_txs,json=numTxs,proto3" json:"num_txs,omitempty"` + TotalTxs int64 `protobuf:"varint,5,opt,name=total_txs,json=totalTxs,proto3" json:"total_txs,omitempty"` + // hashes + LastBlockHash []byte `protobuf:"bytes,6,opt,name=last_block_hash,json=lastBlockHash,proto3" json:"last_block_hash,omitempty"` + ValidatorsHash []byte `protobuf:"bytes,7,opt,name=validators_hash,json=validatorsHash,proto3" json:"validators_hash,omitempty"` + AppHash []byte `protobuf:"bytes,8,opt,name=app_hash,json=appHash,proto3" json:"app_hash,omitempty"` + // consensus + Proposer Validator `protobuf:"bytes,9,opt,name=proposer" json:"proposer"` +} + +func (m *Header) Reset() { *m = Header{} } +func (m *Header) String() string { return proto.CompactTextString(m) } +func (*Header) ProtoMessage() {} +func (*Header) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{29} } + +func (m *Header) GetChainID() string { + if m != nil { + return m.ChainID + } + return "" +} + +func (m *Header) GetHeight() int64 { + if m != nil { + return m.Height + } + return 0 +} + +func (m *Header) GetTime() int64 { + if m != nil { + return m.Time + } + return 0 +} + +func (m *Header) GetNumTxs() int32 { + if m != nil { + return m.NumTxs + } + return 0 +} + +func (m *Header) GetTotalTxs() int64 { + if m != nil { + return m.TotalTxs + } + return 0 +} + +func (m *Header) GetLastBlockHash() []byte { + if m != nil { + return m.LastBlockHash + } + return nil +} + +func (m *Header) GetValidatorsHash() []byte { + if m != nil { + return m.ValidatorsHash + } + return nil +} + +func (m *Header) GetAppHash() []byte { + if m != nil { + return m.AppHash + } + return nil +} + +func (m *Header) GetProposer() Validator { + if m != nil { + return m.Proposer + } + return Validator{} +} + +// Validator +type Validator struct { + Address []byte `protobuf:"bytes,1,opt,name=address,proto3" json:"address,omitempty"` + PubKey PubKey `protobuf:"bytes,2,opt,name=pub_key,json=pubKey" json:"pub_key"` + Power int64 `protobuf:"varint,3,opt,name=power,proto3" json:"power,omitempty"` +} + +func (m *Validator) Reset() { *m = Validator{} } +func (m *Validator) String() string { return proto.CompactTextString(m) } +func (*Validator) ProtoMessage() {} +func (*Validator) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{30} } + +func (m *Validator) GetAddress() []byte { + if m != nil { + return m.Address + } + return nil +} + +func (m *Validator) GetPubKey() PubKey { + if m != nil { + return m.PubKey + } + return PubKey{} +} + +func (m *Validator) GetPower() int64 { + if m != nil { + return m.Power + } + return 0 +} + +// Validator with an extra bool +type SigningValidator struct { + Validator Validator `protobuf:"bytes,1,opt,name=validator" json:"validator"` + SignedLastBlock bool `protobuf:"varint,2,opt,name=signed_last_block,json=signedLastBlock,proto3" json:"signed_last_block,omitempty"` +} + +func (m *SigningValidator) Reset() { *m = SigningValidator{} } +func (m *SigningValidator) String() string { return proto.CompactTextString(m) } +func (*SigningValidator) ProtoMessage() {} +func (*SigningValidator) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{31} } + +func (m *SigningValidator) GetValidator() Validator { + if m != nil { + return m.Validator + } + return Validator{} +} + +func (m *SigningValidator) GetSignedLastBlock() bool { + if m != nil { + return m.SignedLastBlock + } + return false +} + +type PubKey struct { + Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` + Data []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` +} + +func (m *PubKey) Reset() { *m = PubKey{} } +func (m *PubKey) String() string { return proto.CompactTextString(m) } +func (*PubKey) ProtoMessage() {} +func (*PubKey) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{32} } + +func (m *PubKey) GetType() string { + if m != nil { + return m.Type + } + return "" +} + +func (m *PubKey) GetData() []byte { + if m != nil { + return m.Data + } + return nil +} + +type Evidence struct { + Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` + Validator Validator `protobuf:"bytes,2,opt,name=validator" json:"validator"` + Height int64 `protobuf:"varint,3,opt,name=height,proto3" json:"height,omitempty"` + Time int64 `protobuf:"varint,4,opt,name=time,proto3" json:"time,omitempty"` + TotalVotingPower int64 `protobuf:"varint,5,opt,name=total_voting_power,json=totalVotingPower,proto3" json:"total_voting_power,omitempty"` +} + +func (m *Evidence) Reset() { *m = Evidence{} } +func (m *Evidence) String() string { return proto.CompactTextString(m) } +func (*Evidence) ProtoMessage() {} +func (*Evidence) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{33} } + +func (m *Evidence) GetType() string { + if m != nil { + return m.Type + } + return "" +} + +func (m *Evidence) GetValidator() Validator { + if m != nil { + return m.Validator + } + return Validator{} +} + +func (m *Evidence) GetHeight() int64 { + if m != nil { + return m.Height + } + return 0 +} + +func (m *Evidence) GetTime() int64 { + if m != nil { + return m.Time + } + return 0 +} + +func (m *Evidence) GetTotalVotingPower() int64 { + if m != nil { + return m.TotalVotingPower + } + return 0 +} + +func init() { + proto.RegisterType((*Request)(nil), "types.Request") + proto.RegisterType((*RequestEcho)(nil), "types.RequestEcho") + proto.RegisterType((*RequestFlush)(nil), "types.RequestFlush") + proto.RegisterType((*RequestInfo)(nil), "types.RequestInfo") + proto.RegisterType((*RequestSetOption)(nil), "types.RequestSetOption") + proto.RegisterType((*RequestInitChain)(nil), "types.RequestInitChain") + proto.RegisterType((*RequestQuery)(nil), "types.RequestQuery") + proto.RegisterType((*RequestBeginBlock)(nil), "types.RequestBeginBlock") + proto.RegisterType((*RequestCheckTx)(nil), "types.RequestCheckTx") + proto.RegisterType((*RequestDeliverTx)(nil), "types.RequestDeliverTx") + proto.RegisterType((*RequestEndBlock)(nil), "types.RequestEndBlock") + proto.RegisterType((*RequestCommit)(nil), "types.RequestCommit") + proto.RegisterType((*Response)(nil), "types.Response") + proto.RegisterType((*ResponseException)(nil), "types.ResponseException") + proto.RegisterType((*ResponseEcho)(nil), "types.ResponseEcho") + proto.RegisterType((*ResponseFlush)(nil), "types.ResponseFlush") + proto.RegisterType((*ResponseInfo)(nil), "types.ResponseInfo") + proto.RegisterType((*ResponseSetOption)(nil), "types.ResponseSetOption") + proto.RegisterType((*ResponseInitChain)(nil), "types.ResponseInitChain") + proto.RegisterType((*ResponseQuery)(nil), "types.ResponseQuery") + proto.RegisterType((*ResponseBeginBlock)(nil), "types.ResponseBeginBlock") + proto.RegisterType((*ResponseCheckTx)(nil), "types.ResponseCheckTx") + proto.RegisterType((*ResponseDeliverTx)(nil), "types.ResponseDeliverTx") + proto.RegisterType((*ResponseEndBlock)(nil), "types.ResponseEndBlock") + proto.RegisterType((*ResponseCommit)(nil), "types.ResponseCommit") + proto.RegisterType((*ConsensusParams)(nil), "types.ConsensusParams") + proto.RegisterType((*BlockSize)(nil), "types.BlockSize") + proto.RegisterType((*TxSize)(nil), "types.TxSize") + proto.RegisterType((*BlockGossip)(nil), "types.BlockGossip") + proto.RegisterType((*Header)(nil), "types.Header") + proto.RegisterType((*Validator)(nil), "types.Validator") + proto.RegisterType((*SigningValidator)(nil), "types.SigningValidator") + proto.RegisterType((*PubKey)(nil), "types.PubKey") + proto.RegisterType((*Evidence)(nil), "types.Evidence") +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion4 + +// Client API for ABCIApplication service + +type ABCIApplicationClient interface { + Echo(ctx context.Context, in *RequestEcho, opts ...grpc.CallOption) (*ResponseEcho, error) + Flush(ctx context.Context, in *RequestFlush, opts ...grpc.CallOption) (*ResponseFlush, error) + Info(ctx context.Context, in *RequestInfo, opts ...grpc.CallOption) (*ResponseInfo, error) + SetOption(ctx context.Context, in *RequestSetOption, opts ...grpc.CallOption) (*ResponseSetOption, error) + DeliverTx(ctx context.Context, in *RequestDeliverTx, opts ...grpc.CallOption) (*ResponseDeliverTx, error) + CheckTx(ctx context.Context, in *RequestCheckTx, opts ...grpc.CallOption) (*ResponseCheckTx, error) + Query(ctx context.Context, in *RequestQuery, opts ...grpc.CallOption) (*ResponseQuery, error) + Commit(ctx context.Context, in *RequestCommit, opts ...grpc.CallOption) (*ResponseCommit, error) + InitChain(ctx context.Context, in *RequestInitChain, opts ...grpc.CallOption) (*ResponseInitChain, error) + BeginBlock(ctx context.Context, in *RequestBeginBlock, opts ...grpc.CallOption) (*ResponseBeginBlock, error) + EndBlock(ctx context.Context, in *RequestEndBlock, opts ...grpc.CallOption) (*ResponseEndBlock, error) +} + +type aBCIApplicationClient struct { + cc *grpc.ClientConn +} + +func NewABCIApplicationClient(cc *grpc.ClientConn) ABCIApplicationClient { + return &aBCIApplicationClient{cc} +} + +func (c *aBCIApplicationClient) Echo(ctx context.Context, in *RequestEcho, opts ...grpc.CallOption) (*ResponseEcho, error) { + out := new(ResponseEcho) + err := grpc.Invoke(ctx, "/types.ABCIApplication/Echo", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *aBCIApplicationClient) Flush(ctx context.Context, in *RequestFlush, opts ...grpc.CallOption) (*ResponseFlush, error) { + out := new(ResponseFlush) + err := grpc.Invoke(ctx, "/types.ABCIApplication/Flush", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *aBCIApplicationClient) Info(ctx context.Context, in *RequestInfo, opts ...grpc.CallOption) (*ResponseInfo, error) { + out := new(ResponseInfo) + err := grpc.Invoke(ctx, "/types.ABCIApplication/Info", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *aBCIApplicationClient) SetOption(ctx context.Context, in *RequestSetOption, opts ...grpc.CallOption) (*ResponseSetOption, error) { + out := new(ResponseSetOption) + err := grpc.Invoke(ctx, "/types.ABCIApplication/SetOption", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *aBCIApplicationClient) DeliverTx(ctx context.Context, in *RequestDeliverTx, opts ...grpc.CallOption) (*ResponseDeliverTx, error) { + out := new(ResponseDeliverTx) + err := grpc.Invoke(ctx, "/types.ABCIApplication/DeliverTx", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *aBCIApplicationClient) CheckTx(ctx context.Context, in *RequestCheckTx, opts ...grpc.CallOption) (*ResponseCheckTx, error) { + out := new(ResponseCheckTx) + err := grpc.Invoke(ctx, "/types.ABCIApplication/CheckTx", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *aBCIApplicationClient) Query(ctx context.Context, in *RequestQuery, opts ...grpc.CallOption) (*ResponseQuery, error) { + out := new(ResponseQuery) + err := grpc.Invoke(ctx, "/types.ABCIApplication/Query", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *aBCIApplicationClient) Commit(ctx context.Context, in *RequestCommit, opts ...grpc.CallOption) (*ResponseCommit, error) { + out := new(ResponseCommit) + err := grpc.Invoke(ctx, "/types.ABCIApplication/Commit", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *aBCIApplicationClient) InitChain(ctx context.Context, in *RequestInitChain, opts ...grpc.CallOption) (*ResponseInitChain, error) { + out := new(ResponseInitChain) + err := grpc.Invoke(ctx, "/types.ABCIApplication/InitChain", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *aBCIApplicationClient) BeginBlock(ctx context.Context, in *RequestBeginBlock, opts ...grpc.CallOption) (*ResponseBeginBlock, error) { + out := new(ResponseBeginBlock) + err := grpc.Invoke(ctx, "/types.ABCIApplication/BeginBlock", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *aBCIApplicationClient) EndBlock(ctx context.Context, in *RequestEndBlock, opts ...grpc.CallOption) (*ResponseEndBlock, error) { + out := new(ResponseEndBlock) + err := grpc.Invoke(ctx, "/types.ABCIApplication/EndBlock", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// Server API for ABCIApplication service + +type ABCIApplicationServer interface { + Echo(context.Context, *RequestEcho) (*ResponseEcho, error) + Flush(context.Context, *RequestFlush) (*ResponseFlush, error) + Info(context.Context, *RequestInfo) (*ResponseInfo, error) + SetOption(context.Context, *RequestSetOption) (*ResponseSetOption, error) + DeliverTx(context.Context, *RequestDeliverTx) (*ResponseDeliverTx, error) + CheckTx(context.Context, *RequestCheckTx) (*ResponseCheckTx, error) + Query(context.Context, *RequestQuery) (*ResponseQuery, error) + Commit(context.Context, *RequestCommit) (*ResponseCommit, error) + InitChain(context.Context, *RequestInitChain) (*ResponseInitChain, error) + BeginBlock(context.Context, *RequestBeginBlock) (*ResponseBeginBlock, error) + EndBlock(context.Context, *RequestEndBlock) (*ResponseEndBlock, error) +} + +func RegisterABCIApplicationServer(s *grpc.Server, srv ABCIApplicationServer) { + s.RegisterService(&_ABCIApplication_serviceDesc, srv) +} + +func _ABCIApplication_Echo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RequestEcho) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ABCIApplicationServer).Echo(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/types.ABCIApplication/Echo", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ABCIApplicationServer).Echo(ctx, req.(*RequestEcho)) + } + return interceptor(ctx, in, info, handler) +} + +func _ABCIApplication_Flush_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RequestFlush) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ABCIApplicationServer).Flush(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/types.ABCIApplication/Flush", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ABCIApplicationServer).Flush(ctx, req.(*RequestFlush)) + } + return interceptor(ctx, in, info, handler) +} + +func _ABCIApplication_Info_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RequestInfo) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ABCIApplicationServer).Info(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/types.ABCIApplication/Info", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ABCIApplicationServer).Info(ctx, req.(*RequestInfo)) + } + return interceptor(ctx, in, info, handler) +} + +func _ABCIApplication_SetOption_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RequestSetOption) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ABCIApplicationServer).SetOption(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/types.ABCIApplication/SetOption", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ABCIApplicationServer).SetOption(ctx, req.(*RequestSetOption)) + } + return interceptor(ctx, in, info, handler) +} + +func _ABCIApplication_DeliverTx_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RequestDeliverTx) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ABCIApplicationServer).DeliverTx(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/types.ABCIApplication/DeliverTx", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ABCIApplicationServer).DeliverTx(ctx, req.(*RequestDeliverTx)) + } + return interceptor(ctx, in, info, handler) +} + +func _ABCIApplication_CheckTx_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RequestCheckTx) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ABCIApplicationServer).CheckTx(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/types.ABCIApplication/CheckTx", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ABCIApplicationServer).CheckTx(ctx, req.(*RequestCheckTx)) + } + return interceptor(ctx, in, info, handler) +} + +func _ABCIApplication_Query_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RequestQuery) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ABCIApplicationServer).Query(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/types.ABCIApplication/Query", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ABCIApplicationServer).Query(ctx, req.(*RequestQuery)) + } + return interceptor(ctx, in, info, handler) +} + +func _ABCIApplication_Commit_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RequestCommit) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ABCIApplicationServer).Commit(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/types.ABCIApplication/Commit", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ABCIApplicationServer).Commit(ctx, req.(*RequestCommit)) + } + return interceptor(ctx, in, info, handler) +} + +func _ABCIApplication_InitChain_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RequestInitChain) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ABCIApplicationServer).InitChain(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/types.ABCIApplication/InitChain", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ABCIApplicationServer).InitChain(ctx, req.(*RequestInitChain)) + } + return interceptor(ctx, in, info, handler) +} + +func _ABCIApplication_BeginBlock_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RequestBeginBlock) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ABCIApplicationServer).BeginBlock(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/types.ABCIApplication/BeginBlock", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ABCIApplicationServer).BeginBlock(ctx, req.(*RequestBeginBlock)) + } + return interceptor(ctx, in, info, handler) +} + +func _ABCIApplication_EndBlock_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RequestEndBlock) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ABCIApplicationServer).EndBlock(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/types.ABCIApplication/EndBlock", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ABCIApplicationServer).EndBlock(ctx, req.(*RequestEndBlock)) + } + return interceptor(ctx, in, info, handler) +} + +var _ABCIApplication_serviceDesc = grpc.ServiceDesc{ + ServiceName: "types.ABCIApplication", + HandlerType: (*ABCIApplicationServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Echo", + Handler: _ABCIApplication_Echo_Handler, + }, + { + MethodName: "Flush", + Handler: _ABCIApplication_Flush_Handler, + }, + { + MethodName: "Info", + Handler: _ABCIApplication_Info_Handler, + }, + { + MethodName: "SetOption", + Handler: _ABCIApplication_SetOption_Handler, + }, + { + MethodName: "DeliverTx", + Handler: _ABCIApplication_DeliverTx_Handler, + }, + { + MethodName: "CheckTx", + Handler: _ABCIApplication_CheckTx_Handler, + }, + { + MethodName: "Query", + Handler: _ABCIApplication_Query_Handler, + }, + { + MethodName: "Commit", + Handler: _ABCIApplication_Commit_Handler, + }, + { + MethodName: "InitChain", + Handler: _ABCIApplication_InitChain_Handler, + }, + { + MethodName: "BeginBlock", + Handler: _ABCIApplication_BeginBlock_Handler, + }, + { + MethodName: "EndBlock", + Handler: _ABCIApplication_EndBlock_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "types/types.proto", +} + +func init() { proto.RegisterFile("types/types.proto", fileDescriptorTypes) } + +var fileDescriptorTypes = []byte{ + // 1846 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe4, 0x58, 0xcd, 0x6e, 0x1b, 0xc9, + 0x11, 0x16, 0xff, 0x39, 0xa5, 0x1f, 0xd2, 0x2d, 0xdb, 0xa2, 0xb9, 0x08, 0x6c, 0x0c, 0x02, 0xaf, + 0x9c, 0xd5, 0x8a, 0x89, 0x76, 0x6d, 0xd8, 0xbb, 0xc9, 0x22, 0x92, 0xd6, 0x59, 0x0a, 0x9b, 0x1f, + 0x65, 0xec, 0x75, 0x80, 0x5c, 0x88, 0x26, 0xa7, 0x45, 0x0e, 0x4c, 0xce, 0xcc, 0x4e, 0x37, 0xb5, + 0x94, 0x6f, 0xb9, 0x2f, 0x72, 0xcd, 0x39, 0x2f, 0x90, 0x43, 0x80, 0xbc, 0x42, 0x90, 0x97, 0x88, + 0x0f, 0x49, 0x4e, 0x79, 0x89, 0x04, 0x55, 0xdd, 0xf3, 0xab, 0xa1, 0xe1, 0x38, 0xc7, 0xbd, 0x48, + 0x5d, 0x5d, 0x55, 0x3d, 0x5d, 0xc5, 0xaa, 0xaf, 0xaa, 0x1a, 0x6e, 0xa8, 0xab, 0x50, 0xc8, 0x01, + 0xfd, 0x3d, 0x0c, 0xa3, 0x40, 0x05, 0xac, 0x41, 0x44, 0xff, 0xc3, 0xa9, 0xa7, 0x66, 0xcb, 0xf1, + 0xe1, 0x24, 0x58, 0x0c, 0xa6, 0xc1, 0x34, 0x18, 0x10, 0x77, 0xbc, 0xbc, 0x20, 0x8a, 0x08, 0x5a, + 0x69, 0xad, 0xfe, 0x20, 0x23, 0xae, 0x84, 0xef, 0x8a, 0x68, 0xe1, 0xf9, 0x6a, 0xa0, 0x16, 0x73, + 0x6f, 0x2c, 0x07, 0x93, 0x60, 0xb1, 0x08, 0xfc, 0xec, 0x67, 0xec, 0xbf, 0xd6, 0xa1, 0xe5, 0x88, + 0xaf, 0x97, 0x42, 0x2a, 0xb6, 0x0f, 0x75, 0x31, 0x99, 0x05, 0xbd, 0xea, 0xbd, 0xca, 0xfe, 0xe6, + 0x11, 0x3b, 0xd4, 0x72, 0x86, 0xfb, 0x74, 0x32, 0x0b, 0x86, 0x1b, 0x0e, 0x49, 0xb0, 0x0f, 0xa0, + 0x71, 0x31, 0x5f, 0xca, 0x59, 0xaf, 0x46, 0xa2, 0xbb, 0x79, 0xd1, 0x9f, 0x21, 0x6b, 0xb8, 0xe1, + 0x68, 0x19, 0x3c, 0xd6, 0xf3, 0x2f, 0x82, 0x5e, 0xbd, 0xec, 0xd8, 0x33, 0xff, 0x82, 0x8e, 0x45, + 0x09, 0xf6, 0x18, 0x40, 0x0a, 0x35, 0x0a, 0x42, 0xe5, 0x05, 0x7e, 0xaf, 0x41, 0xf2, 0x7b, 0x79, + 0xf9, 0x67, 0x42, 0xfd, 0x8a, 0xd8, 0xc3, 0x0d, 0xc7, 0x92, 0x31, 0x81, 0x9a, 0x9e, 0xef, 0xa9, + 0xd1, 0x64, 0xc6, 0x3d, 0xbf, 0xd7, 0x2c, 0xd3, 0x3c, 0xf3, 0x3d, 0x75, 0x8a, 0x6c, 0xd4, 0xf4, + 0x62, 0x02, 0x4d, 0xf9, 0x7a, 0x29, 0xa2, 0xab, 0x5e, 0xab, 0xcc, 0x94, 0x5f, 0x23, 0x0b, 0x4d, + 0x21, 0x19, 0xf6, 0x29, 0x6c, 0x8e, 0xc5, 0xd4, 0xf3, 0x47, 0xe3, 0x79, 0x30, 0x79, 0xd9, 0x6b, + 0x93, 0x4a, 0x2f, 0xaf, 0x72, 0x82, 0x02, 0x27, 0xc8, 0x1f, 0x6e, 0x38, 0x30, 0x4e, 0x28, 0x76, + 0x04, 0xed, 0xc9, 0x4c, 0x4c, 0x5e, 0x8e, 0xd4, 0xaa, 0x67, 0x91, 0xe6, 0xad, 0xbc, 0xe6, 0x29, + 0x72, 0x9f, 0xaf, 0x86, 0x1b, 0x4e, 0x6b, 0xa2, 0x97, 0x68, 0x97, 0x2b, 0xe6, 0xde, 0xa5, 0x88, + 0x50, 0x6b, 0xb7, 0xcc, 0xae, 0xcf, 0x35, 0x9f, 0xf4, 0x2c, 0x37, 0x26, 0xd8, 0x43, 0xb0, 0x84, + 0xef, 0x9a, 0x8b, 0x6e, 0x92, 0xe2, 0xed, 0xc2, 0x2f, 0xea, 0xbb, 0xf1, 0x35, 0xdb, 0xc2, 0xac, + 0xd9, 0x21, 0x34, 0x31, 0x4a, 0x3c, 0xd5, 0xdb, 0x22, 0x9d, 0x9b, 0x85, 0x2b, 0x12, 0x6f, 0xb8, + 0xe1, 0x18, 0xa9, 0x93, 0x16, 0x34, 0x2e, 0xf9, 0x7c, 0x29, 0xec, 0xf7, 0x61, 0x33, 0x13, 0x29, + 0xac, 0x07, 0xad, 0x85, 0x90, 0x92, 0x4f, 0x45, 0xaf, 0x72, 0xaf, 0xb2, 0x6f, 0x39, 0x31, 0x69, + 0xef, 0xc0, 0x56, 0x36, 0x4e, 0x32, 0x8a, 0x18, 0x0b, 0xa8, 0x78, 0x29, 0x22, 0x89, 0x01, 0x60, + 0x14, 0x0d, 0x69, 0x7f, 0x02, 0xdd, 0x62, 0x10, 0xb0, 0x2e, 0xd4, 0x5e, 0x8a, 0x2b, 0x23, 0x89, + 0x4b, 0x76, 0xd3, 0x5c, 0x88, 0xa2, 0xd8, 0x72, 0xcc, 0xed, 0xfe, 0x55, 0x49, 0x94, 0x93, 0x38, + 0x60, 0x0c, 0xea, 0xca, 0x5b, 0xe8, 0x0b, 0xd6, 0x1c, 0x5a, 0xb3, 0x3b, 0xf8, 0x23, 0x71, 0xcf, + 0x1f, 0x79, 0xae, 0x39, 0xa1, 0x45, 0xf4, 0x99, 0xcb, 0x8e, 0xa1, 0x3b, 0x09, 0x7c, 0x29, 0x7c, + 0xb9, 0x94, 0xa3, 0x90, 0x47, 0x7c, 0x21, 0x4d, 0xfc, 0xc7, 0x8e, 0x3d, 0x8d, 0xd9, 0xe7, 0xc4, + 0x75, 0x3a, 0x93, 0xfc, 0x06, 0x7b, 0x04, 0x70, 0xc9, 0xe7, 0x9e, 0xcb, 0x55, 0x10, 0xc9, 0x5e, + 0xfd, 0x5e, 0x6d, 0x7f, 0xf3, 0xa8, 0x6b, 0x94, 0x5f, 0xc4, 0x8c, 0x93, 0xfa, 0xdf, 0x5e, 0xdf, + 0xdd, 0x70, 0x32, 0x92, 0xec, 0x3e, 0x74, 0x78, 0x18, 0x8e, 0xa4, 0xe2, 0x4a, 0x8c, 0xc6, 0x57, + 0x4a, 0x48, 0xca, 0x8e, 0x2d, 0x67, 0x9b, 0x87, 0xe1, 0x33, 0xdc, 0x3d, 0xc1, 0x4d, 0xdb, 0x4d, + 0x7c, 0x4b, 0x81, 0x8b, 0x16, 0xba, 0x5c, 0x71, 0xb2, 0x70, 0xcb, 0xa1, 0x35, 0xee, 0x85, 0x5c, + 0xcd, 0x8c, 0x75, 0xb4, 0x66, 0xb7, 0xa1, 0x39, 0x13, 0xde, 0x74, 0xa6, 0xc8, 0xa0, 0x9a, 0x63, + 0x28, 0x74, 0x66, 0x18, 0x05, 0x97, 0x82, 0x72, 0xb7, 0xed, 0x68, 0xc2, 0xfe, 0x7b, 0x05, 0x6e, + 0x5c, 0x0b, 0x76, 0x3c, 0x77, 0xc6, 0xe5, 0x2c, 0xfe, 0x16, 0xae, 0xd9, 0x07, 0x78, 0x2e, 0x77, + 0x45, 0x64, 0x30, 0x65, 0xdb, 0xd8, 0x3a, 0xa4, 0x4d, 0x63, 0xa8, 0x11, 0x61, 0x3f, 0xc9, 0x39, + 0xa7, 0x46, 0xce, 0x89, 0x63, 0xfd, 0x99, 0x37, 0xf5, 0x3d, 0x7f, 0xfa, 0x26, 0x1f, 0x0d, 0xe1, + 0xe6, 0xf8, 0xea, 0x15, 0xf7, 0x95, 0xe7, 0x8b, 0xd1, 0x35, 0x2f, 0x77, 0xcc, 0x41, 0x4f, 0x2f, + 0x3d, 0x57, 0xf8, 0x13, 0x61, 0x0e, 0xd8, 0x4d, 0x54, 0x92, 0xa3, 0xa5, 0x7d, 0x0f, 0x76, 0xf2, + 0x19, 0xc9, 0x76, 0xa0, 0xaa, 0x56, 0xc6, 0xb2, 0xaa, 0x5a, 0xd9, 0x76, 0x12, 0x4d, 0x49, 0xf6, + 0x5d, 0x93, 0x79, 0x00, 0x9d, 0x42, 0xa2, 0x65, 0xdc, 0x5c, 0xc9, 0xba, 0xd9, 0xee, 0xc0, 0x76, + 0x2e, 0xbf, 0xec, 0x6f, 0x1b, 0xd0, 0x76, 0x84, 0x0c, 0x31, 0x7c, 0xd8, 0x63, 0xb0, 0xc4, 0x6a, + 0x22, 0x34, 0x28, 0x56, 0x0a, 0x90, 0xa3, 0x65, 0x9e, 0xc6, 0x7c, 0xc4, 0x80, 0x44, 0x98, 0x3d, + 0xc8, 0x01, 0xfa, 0x6e, 0x51, 0x29, 0x8b, 0xe8, 0x07, 0x79, 0x44, 0xbf, 0x59, 0x90, 0x2d, 0x40, + 0xfa, 0x83, 0x1c, 0xa4, 0x17, 0x0f, 0xce, 0x61, 0xfa, 0x93, 0x12, 0x4c, 0x2f, 0x5e, 0x7f, 0x0d, + 0xa8, 0x3f, 0x29, 0x01, 0xf5, 0xde, 0xb5, 0x6f, 0x95, 0xa2, 0xfa, 0x41, 0x1e, 0xd5, 0x8b, 0xe6, + 0x14, 0x60, 0xfd, 0xc7, 0x65, 0xb0, 0x7e, 0xa7, 0xa0, 0xb3, 0x16, 0xd7, 0x3f, 0xba, 0x86, 0xeb, + 0xb7, 0x0b, 0xaa, 0x25, 0xc0, 0xfe, 0x24, 0x07, 0xec, 0x50, 0x6a, 0xdb, 0x1a, 0x64, 0x7f, 0x74, + 0x1d, 0xd9, 0xf7, 0x8a, 0x3f, 0x6d, 0x19, 0xb4, 0x0f, 0x0a, 0xd0, 0x7e, 0xab, 0x78, 0xcb, 0xb5, + 0xd8, 0xfe, 0x00, 0xf3, 0xbd, 0x10, 0x69, 0x88, 0x0d, 0x22, 0x8a, 0x82, 0xc8, 0x80, 0xaf, 0x26, + 0xec, 0x7d, 0x44, 0xa0, 0x34, 0xbe, 0xde, 0x50, 0x07, 0x28, 0xe8, 0x33, 0xd1, 0x65, 0xff, 0xa1, + 0x92, 0xea, 0x52, 0x29, 0xc8, 0xa2, 0x97, 0x65, 0xd0, 0x2b, 0x53, 0x1e, 0xaa, 0xb9, 0xf2, 0xc0, + 0x7e, 0x00, 0x37, 0xe6, 0x5c, 0x2a, 0xed, 0x97, 0x51, 0x0e, 0xce, 0x3a, 0xc8, 0xd0, 0x0e, 0xd1, + 0xb8, 0xf6, 0x21, 0xec, 0x66, 0x64, 0x11, 0x5a, 0x09, 0xba, 0xea, 0x94, 0xbc, 0xdd, 0x44, 0xfa, + 0x38, 0x0c, 0x87, 0x5c, 0xce, 0xec, 0x5f, 0xa4, 0xf6, 0xa7, 0xa5, 0x87, 0x41, 0x7d, 0x12, 0xb8, + 0xda, 0xac, 0x6d, 0x87, 0xd6, 0x58, 0x8e, 0xe6, 0xc1, 0x94, 0xbe, 0x6a, 0x39, 0xb8, 0x44, 0xa9, + 0x24, 0x53, 0x2c, 0x9d, 0x12, 0xf6, 0xef, 0x2b, 0xe9, 0x79, 0x69, 0x35, 0x2a, 0x2b, 0x2f, 0x95, + 0xff, 0xa7, 0xbc, 0x54, 0xdf, 0xb6, 0xbc, 0xd8, 0x7f, 0xa9, 0xa4, 0xbf, 0x45, 0x52, 0x38, 0xde, + 0xcd, 0x38, 0x0c, 0x0b, 0xcf, 0x77, 0xc5, 0x8a, 0x52, 0xbd, 0xe6, 0x68, 0x22, 0xae, 0xd3, 0x4d, + 0x72, 0x70, 0xbe, 0x4e, 0xb7, 0x68, 0x4f, 0x13, 0xa6, 0xe0, 0x04, 0x17, 0x94, 0x83, 0x5b, 0x8e, + 0x26, 0x32, 0xb8, 0x69, 0xe5, 0x70, 0xf3, 0x1c, 0xd8, 0xf5, 0xec, 0x64, 0x9f, 0x40, 0x5d, 0xf1, + 0x29, 0x3a, 0x0f, 0xed, 0xdf, 0x39, 0xd4, 0x5d, 0xef, 0xe1, 0x97, 0x2f, 0xce, 0xb9, 0x17, 0x9d, + 0xdc, 0x46, 0xeb, 0xff, 0xfd, 0xfa, 0xee, 0x0e, 0xca, 0x1c, 0x04, 0x0b, 0x4f, 0x89, 0x45, 0xa8, + 0xae, 0x1c, 0xd2, 0xb1, 0xff, 0x53, 0x41, 0xd4, 0xce, 0x65, 0x6d, 0xa9, 0x2f, 0xe2, 0xd0, 0xac, + 0x66, 0x0a, 0xeb, 0xdb, 0xf9, 0xe7, 0x7b, 0x00, 0x53, 0x2e, 0x47, 0xdf, 0x70, 0x5f, 0x09, 0xd7, + 0x38, 0xc9, 0x9a, 0x72, 0xf9, 0x1b, 0xda, 0xc0, 0xfe, 0x03, 0xd9, 0x4b, 0x29, 0x5c, 0xf2, 0x56, + 0xcd, 0x69, 0x4d, 0xb9, 0xfc, 0x4a, 0x0a, 0x37, 0xb1, 0xab, 0xf5, 0xbf, 0xdb, 0xc5, 0xf6, 0xa1, + 0x76, 0x21, 0x84, 0x41, 0xb6, 0x6e, 0xa2, 0x7a, 0xf6, 0xe8, 0x63, 0x52, 0xd6, 0x21, 0x81, 0x22, + 0xf6, 0xef, 0xaa, 0x69, 0x70, 0xa6, 0xc5, 0xed, 0xbb, 0xe5, 0x83, 0x7f, 0x52, 0xb7, 0x98, 0x87, + 0x52, 0x76, 0x0a, 0x37, 0x92, 0x94, 0x19, 0x2d, 0x43, 0x97, 0x63, 0x17, 0x56, 0x79, 0x63, 0x8e, + 0x75, 0x13, 0x85, 0xaf, 0xb4, 0x3c, 0xfb, 0x25, 0xec, 0x15, 0x92, 0x3c, 0x39, 0xaa, 0xfa, 0xc6, + 0x5c, 0xbf, 0x95, 0xcf, 0xf5, 0xf8, 0xbc, 0xd8, 0x1f, 0xb5, 0x77, 0x88, 0xf5, 0xef, 0x63, 0x9b, + 0x93, 0x85, 0xfe, 0xb2, 0x5f, 0xd4, 0xfe, 0x63, 0x05, 0x3a, 0x85, 0xcb, 0xb0, 0x01, 0x80, 0x46, + 0x4e, 0xe9, 0xbd, 0x12, 0x06, 0xa4, 0x62, 0x1f, 0x90, 0xb3, 0x9e, 0x79, 0xaf, 0x84, 0x63, 0x8d, + 0xe3, 0x25, 0xbb, 0x0f, 0x2d, 0xb5, 0xd2, 0xd2, 0xf9, 0x46, 0xf0, 0xf9, 0x8a, 0x44, 0x9b, 0x8a, + 0xfe, 0xb3, 0x87, 0xb0, 0xa5, 0x0f, 0x9e, 0x06, 0x52, 0x7a, 0xa1, 0x69, 0x46, 0x58, 0xf6, 0xe8, + 0x2f, 0x88, 0xe3, 0x6c, 0x8e, 0x53, 0xc2, 0xfe, 0x2d, 0x58, 0xc9, 0x67, 0xd9, 0x7b, 0x60, 0x2d, + 0xf8, 0xca, 0x74, 0xc9, 0x78, 0xb7, 0x86, 0xd3, 0x5e, 0xf0, 0x15, 0x35, 0xc8, 0x6c, 0x0f, 0x5a, + 0xc8, 0x54, 0x2b, 0xed, 0xef, 0x86, 0xd3, 0x5c, 0xf0, 0xd5, 0xf3, 0x55, 0xc2, 0x98, 0x72, 0x19, + 0xb7, 0xc0, 0x0b, 0xbe, 0xfa, 0x82, 0x4b, 0xfb, 0x33, 0x68, 0xea, 0x4b, 0xbe, 0xd5, 0xc1, 0xa8, + 0x5f, 0xcd, 0xe9, 0xff, 0x14, 0x36, 0x33, 0xf7, 0x66, 0x3f, 0x82, 0x5b, 0xda, 0xc2, 0x90, 0x47, + 0x8a, 0x3c, 0x92, 0x3b, 0x90, 0x11, 0xf3, 0x9c, 0x47, 0x0a, 0x3f, 0xa9, 0x9b, 0xfa, 0x3f, 0x57, + 0xa1, 0xa9, 0x1b, 0x66, 0x76, 0x3f, 0x33, 0x9d, 0x50, 0x55, 0x3c, 0xd9, 0xfc, 0xc7, 0xeb, 0xbb, + 0x2d, 0x2a, 0x20, 0x67, 0x9f, 0xa7, 0xa3, 0x4a, 0x0a, 0x98, 0xd5, 0x5c, 0x3f, 0x1f, 0x4f, 0x3c, + 0xb5, 0xcc, 0xc4, 0xb3, 0x07, 0x2d, 0x7f, 0xb9, 0x20, 0x97, 0xd4, 0xb5, 0x4b, 0xfc, 0xe5, 0x02, + 0x5d, 0xf2, 0x1e, 0x58, 0x2a, 0x50, 0x7c, 0x4e, 0x2c, 0x9d, 0xa4, 0x6d, 0xda, 0x40, 0xe6, 0x7d, + 0xe8, 0x64, 0xab, 0x2d, 0x56, 0x4f, 0x0d, 0xee, 0xdb, 0x69, 0xad, 0xc5, 0x09, 0xe0, 0x7d, 0xe8, + 0xa4, 0x85, 0x46, 0xcb, 0x69, 0xc0, 0xdf, 0x49, 0xb7, 0x49, 0xf0, 0x0e, 0xb4, 0x93, 0x3a, 0xac, + 0xc1, 0xbf, 0xc5, 0x75, 0xf9, 0xc5, 0xc1, 0x39, 0x8c, 0x82, 0x30, 0x90, 0x22, 0x32, 0x0d, 0xd6, + 0xba, 0x84, 0x4b, 0xe4, 0x6c, 0x0f, 0xac, 0x84, 0x89, 0x4d, 0x03, 0x77, 0xdd, 0x48, 0x48, 0x69, + 0xfa, 0xf3, 0x98, 0x64, 0x07, 0xd0, 0x0a, 0x97, 0xe3, 0x11, 0xd6, 0xa6, 0x7c, 0x60, 0x9e, 0x2f, + 0xc7, 0x5f, 0x8a, 0xab, 0x78, 0x42, 0x09, 0x89, 0xa2, 0xea, 0x14, 0x7c, 0x23, 0x22, 0xe3, 0x3f, + 0x4d, 0xd8, 0x0a, 0xba, 0xc5, 0xf1, 0x84, 0x7d, 0x0c, 0x56, 0x62, 0x5f, 0x21, 0x41, 0x8a, 0x77, + 0x4e, 0x05, 0xb1, 0x85, 0x91, 0xde, 0xd4, 0x17, 0xee, 0x28, 0xf5, 0x2d, 0xdd, 0xab, 0xed, 0x74, + 0x34, 0xe3, 0xe7, 0xb1, 0x73, 0xed, 0x1f, 0x42, 0x53, 0xdf, 0x91, 0x7e, 0xd4, 0xab, 0x30, 0xee, + 0xaf, 0x68, 0x5d, 0x9a, 0xc9, 0x7f, 0xaa, 0x40, 0x3b, 0x1e, 0x7f, 0x4a, 0x95, 0x72, 0x97, 0xae, + 0xbe, 0xed, 0xa5, 0xd7, 0xcd, 0x8e, 0x71, 0xac, 0xd5, 0x33, 0xb1, 0x76, 0x00, 0x4c, 0x87, 0xd4, + 0x65, 0xa0, 0x3c, 0x7f, 0x3a, 0xd2, 0xde, 0xd4, 0xb1, 0xd5, 0x25, 0xce, 0x0b, 0x62, 0x9c, 0xe3, + 0xfe, 0xd1, 0xb7, 0x0d, 0xe8, 0x1c, 0x9f, 0x9c, 0x9e, 0x1d, 0x87, 0xe1, 0xdc, 0x9b, 0x70, 0xea, + 0xba, 0x06, 0x50, 0xa7, 0xbe, 0xb2, 0xe4, 0x75, 0xaa, 0x5f, 0x36, 0xe0, 0xb0, 0x23, 0x68, 0x50, + 0x7b, 0xc9, 0xca, 0x1e, 0xa9, 0xfa, 0xa5, 0x73, 0x0e, 0x7e, 0x44, 0x37, 0xa0, 0xd7, 0xdf, 0xaa, + 0xfa, 0x65, 0xc3, 0x0e, 0xfb, 0x0c, 0xac, 0xb4, 0x31, 0x5c, 0xf7, 0x62, 0xd5, 0x5f, 0x3b, 0xf6, + 0xa0, 0x7e, 0x5a, 0x6b, 0xd7, 0xbd, 0xef, 0xf4, 0xd7, 0xce, 0x07, 0xec, 0x31, 0xb4, 0xe2, 0x6e, + 0xa5, 0xfc, 0x4d, 0xa9, 0xbf, 0x66, 0x24, 0x41, 0xf7, 0xe8, 0x8e, 0xaf, 0xec, 0xe1, 0xab, 0x5f, + 0x3a, 0x37, 0xb1, 0x87, 0xd0, 0x34, 0x05, 0xa3, 0xf4, 0x75, 0xa8, 0x5f, 0x3e, 0x58, 0xa0, 0x91, + 0x69, 0xb7, 0xbb, 0xee, 0x71, 0xae, 0xbf, 0x76, 0xc0, 0x63, 0xc7, 0x00, 0x99, 0x2e, 0x6f, 0xed, + 0xab, 0x5b, 0x7f, 0xfd, 0xe0, 0xc6, 0x3e, 0x85, 0x76, 0x3a, 0x8c, 0x97, 0xbf, 0x86, 0xf5, 0xd7, + 0xcd, 0x52, 0xe3, 0x26, 0xbd, 0x98, 0x7e, 0xf4, 0xdf, 0x00, 0x00, 0x00, 0xff, 0xff, 0xe5, 0xf3, + 0xb2, 0x34, 0xad, 0x15, 0x00, 0x00, +} diff --git a/abci/types/types.proto b/abci/types/types.proto new file mode 100644 index 000000000..b4f4b2aa6 --- /dev/null +++ b/abci/types/types.proto @@ -0,0 +1,282 @@ +syntax = "proto3"; +package types; + +// For more information on gogo.proto, see: +// https://github.com/gogo/protobuf/blob/master/extensions.md +import "github.com/gogo/protobuf/gogoproto/gogo.proto"; +import "github.com/tendermint/tmlibs/common/types.proto"; + +// This file is copied from http://github.com/tendermint/abci +// NOTE: When using custom types, mind the warnings. +// https://github.com/gogo/protobuf/blob/master/custom_types.md#warnings-and-issues + +//---------------------------------------- +// Request types + +message Request { + oneof value { + RequestEcho echo = 2; + RequestFlush flush = 3; + RequestInfo info = 4; + RequestSetOption set_option = 5; + RequestInitChain init_chain = 6; + RequestQuery query = 7; + RequestBeginBlock begin_block = 8; + RequestCheckTx check_tx = 9; + RequestDeliverTx deliver_tx = 19; + RequestEndBlock end_block = 11; + RequestCommit commit = 12; + } +} + +message RequestEcho { + string message = 1; +} + +message RequestFlush { +} + +message RequestInfo { + string version = 1; +} + +// nondeterministic +message RequestSetOption { + string key = 1; + string value = 2; +} + +message RequestInitChain { + int64 time = 1; + string chain_id = 2; + ConsensusParams consensus_params = 3; + repeated Validator validators = 4 [(gogoproto.nullable)=false]; + bytes app_state_bytes = 5; +} + +message RequestQuery { + bytes data = 1; + string path = 2; + int64 height = 3; + bool prove = 4; +} + +message RequestBeginBlock { + bytes hash = 1; + Header header = 2 [(gogoproto.nullable)=false]; + repeated SigningValidator validators = 3 [(gogoproto.nullable)=false]; + repeated Evidence byzantine_validators = 4 [(gogoproto.nullable)=false]; +} + +message RequestCheckTx { + bytes tx = 1; +} + +message RequestDeliverTx { + bytes tx = 1; +} + +message RequestEndBlock { + int64 height = 1; +} + +message RequestCommit { +} + +//---------------------------------------- +// Response types + +message Response { + oneof value { + ResponseException exception = 1; + ResponseEcho echo = 2; + ResponseFlush flush = 3; + ResponseInfo info = 4; + ResponseSetOption set_option = 5; + ResponseInitChain init_chain = 6; + ResponseQuery query = 7; + ResponseBeginBlock begin_block = 8; + ResponseCheckTx check_tx = 9; + ResponseDeliverTx deliver_tx = 10; + ResponseEndBlock end_block = 11; + ResponseCommit commit = 12; + } +} + +// nondeterministic +message ResponseException { + string error = 1; +} + +message ResponseEcho { + string message = 1; +} + +message ResponseFlush { +} + +message ResponseInfo { + string data = 1; + string version = 2; + int64 last_block_height = 3; + bytes last_block_app_hash = 4; +} + +// nondeterministic +message ResponseSetOption { + uint32 code = 1; + // bytes data = 2; + string log = 3; + string info = 4; +} + +message ResponseInitChain { + ConsensusParams consensus_params = 1; + repeated Validator validators = 2 [(gogoproto.nullable)=false]; +} + +message ResponseQuery { + uint32 code = 1; + // bytes data = 2; // use "value" instead. + string log = 3; // nondeterministic + string info = 4; // nondeterministic + int64 index = 5; + bytes key = 6; + bytes value = 7; + bytes proof = 8; + int64 height = 9; +} + +message ResponseBeginBlock { + repeated common.KVPair tags = 1 [(gogoproto.nullable)=false, (gogoproto.jsontag)="tags,omitempty"]; +} + +message ResponseCheckTx { + uint32 code = 1; + bytes data = 2; + string log = 3; // nondeterministic + string info = 4; // nondeterministic + int64 gas_wanted = 5; + int64 gas_used = 6; + repeated common.KVPair tags = 7 [(gogoproto.nullable)=false, (gogoproto.jsontag)="tags,omitempty"]; + common.KI64Pair fee = 8 [(gogoproto.nullable)=false]; +} + +message ResponseDeliverTx { + uint32 code = 1; + bytes data = 2; + string log = 3; // nondeterministic + string info = 4; // nondeterministic + int64 gas_wanted = 5; + int64 gas_used = 6; + repeated common.KVPair tags = 7 [(gogoproto.nullable)=false, (gogoproto.jsontag)="tags,omitempty"]; + common.KI64Pair fee = 8 [(gogoproto.nullable)=false]; +} + +message ResponseEndBlock { + repeated Validator validator_updates = 1 [(gogoproto.nullable)=false]; + ConsensusParams consensus_param_updates = 2; + repeated common.KVPair tags = 3 [(gogoproto.nullable)=false, (gogoproto.jsontag)="tags,omitempty"]; +} + +message ResponseCommit { + // reserve 1 + bytes data = 2; +} + +//---------------------------------------- +// Misc. + +// ConsensusParams contains all consensus-relevant parameters +// that can be adjusted by the abci app +message ConsensusParams { + BlockSize block_size = 1; + TxSize tx_size = 2; + BlockGossip block_gossip = 3; +} + +// BlockSize contain limits on the block size. +message BlockSize { + int32 max_bytes = 1; + int32 max_txs = 2; + int64 max_gas = 3; +} + +// TxSize contain limits on the tx size. +message TxSize { + int32 max_bytes = 1; + int64 max_gas = 2; +} + +// BlockGossip determine consensus critical +// elements of how blocks are gossiped +message BlockGossip { + // Note: must not be 0 + int32 block_part_size_bytes = 1; +} + +//---------------------------------------- +// Blockchain Types + +// just the minimum the app might need +message Header { + // basics + string chain_id = 1 [(gogoproto.customname)="ChainID"]; + int64 height = 2; + int64 time = 3; + + // txs + int32 num_txs = 4; + int64 total_txs = 5; + + // hashes + bytes last_block_hash = 6; + bytes validators_hash = 7; + bytes app_hash = 8; + + // consensus + Validator proposer = 9 [(gogoproto.nullable)=false]; +} + +// Validator +message Validator { + bytes address = 1; + PubKey pub_key = 2 [(gogoproto.nullable)=false]; + int64 power = 3; +} + +// Validator with an extra bool +message SigningValidator { + Validator validator = 1 [(gogoproto.nullable)=false]; + bool signed_last_block = 2; +} + +message PubKey { + string type = 1; + bytes data = 2; +} + +message Evidence { + string type = 1; + Validator validator = 2 [(gogoproto.nullable)=false]; + int64 height = 3; + int64 time = 4; + int64 total_voting_power = 5; +} + +//---------------------------------------- +// Service Definition + +service ABCIApplication { + rpc Echo(RequestEcho) returns (ResponseEcho) ; + rpc Flush(RequestFlush) returns (ResponseFlush); + rpc Info(RequestInfo) returns (ResponseInfo); + rpc SetOption(RequestSetOption) returns (ResponseSetOption); + rpc DeliverTx(RequestDeliverTx) returns (ResponseDeliverTx); + rpc CheckTx(RequestCheckTx) returns (ResponseCheckTx); + rpc Query(RequestQuery) returns (ResponseQuery); + rpc Commit(RequestCommit) returns (ResponseCommit); + rpc InitChain(RequestInitChain) returns (ResponseInitChain); + rpc BeginBlock(RequestBeginBlock) returns (ResponseBeginBlock); + rpc EndBlock(RequestEndBlock) returns (ResponseEndBlock); +} diff --git a/abci/types/types_test.go b/abci/types/types_test.go new file mode 100644 index 000000000..baa8155cd --- /dev/null +++ b/abci/types/types_test.go @@ -0,0 +1,31 @@ +package types + +import ( + "testing" + + asrt "github.com/stretchr/testify/assert" +) + +func TestConsensusParams(t *testing.T) { + assert := asrt.New(t) + + params := &ConsensusParams{ + BlockSize: &BlockSize{MaxGas: 12345}, + BlockGossip: &BlockGossip{BlockPartSizeBytes: 54321}, + } + var noParams *ConsensusParams // nil + + // no error with nil fields + assert.Nil(noParams.GetBlockSize()) + assert.EqualValues(noParams.GetBlockSize().GetMaxGas(), 0) + + // get values with real fields + assert.NotNil(params.GetBlockSize()) + assert.EqualValues(params.GetBlockSize().GetMaxTxs(), 0) + assert.EqualValues(params.GetBlockSize().GetMaxGas(), 12345) + assert.NotNil(params.GetBlockGossip()) + assert.EqualValues(params.GetBlockGossip().GetBlockPartSizeBytes(), 54321) + assert.Nil(params.GetTxSize()) + assert.EqualValues(params.GetTxSize().GetMaxBytes(), 0) + +} diff --git a/abci/types/util.go b/abci/types/util.go new file mode 100644 index 000000000..458024c58 --- /dev/null +++ b/abci/types/util.go @@ -0,0 +1,59 @@ +package types + +import ( + "bytes" + "encoding/json" + "sort" + + cmn "github.com/tendermint/tendermint/libs/common" +) + +//------------------------------------------------------------------------------ + +// Validators is a list of validators that implements the Sort interface +type Validators []Validator + +var _ sort.Interface = (Validators)(nil) + +// All these methods for Validators: +// Len, Less and Swap +// are for Validators to implement sort.Interface +// which will be used by the sort package. +// See Issue https://github.com/tendermint/abci/issues/212 + +func (v Validators) Len() int { + return len(v) +} + +// XXX: doesn't distinguish same validator with different power +func (v Validators) Less(i, j int) bool { + return bytes.Compare(v[i].PubKey.Data, v[j].PubKey.Data) <= 0 +} + +func (v Validators) Swap(i, j int) { + v1 := v[i] + v[i] = v[j] + v[j] = v1 +} + +func ValidatorsString(vs Validators) string { + s := make([]validatorPretty, len(vs)) + for i, v := range vs { + s[i] = validatorPretty{ + Address: v.Address, + PubKey: v.PubKey.Data, + Power: v.Power, + } + } + b, err := json.Marshal(s) + if err != nil { + panic(err.Error()) + } + return string(b) +} + +type validatorPretty struct { + Address cmn.HexBytes `json:"address"` + PubKey []byte `json:"pub_key"` + Power int64 `json:"power"` +} diff --git a/abci/version/version.go b/abci/version/version.go new file mode 100644 index 000000000..7223a86ad --- /dev/null +++ b/abci/version/version.go @@ -0,0 +1,9 @@ +package version + +// NOTE: we should probably be versioning the ABCI and the abci-cli separately + +const Maj = "0" +const Min = "12" +const Fix = "0" + +const Version = "0.12.0" diff --git a/benchmarks/codec_test.go b/benchmarks/codec_test.go index 9acafce71..53cbf632c 100644 --- a/benchmarks/codec_test.go +++ b/benchmarks/codec_test.go @@ -5,9 +5,9 @@ import ( "time" "github.com/tendermint/go-amino" - "github.com/tendermint/go-crypto" proto "github.com/tendermint/tendermint/benchmarks/proto" + "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/p2p" ctypes "github.com/tendermint/tendermint/rpc/core/types" ) diff --git a/benchmarks/map_test.go b/benchmarks/map_test.go index 2d9789026..d13a19edf 100644 --- a/benchmarks/map_test.go +++ b/benchmarks/map_test.go @@ -3,7 +3,7 @@ package benchmarks import ( "testing" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) func BenchmarkSomething(b *testing.B) { diff --git a/benchmarks/os_test.go b/benchmarks/os_test.go index dfadc3128..406038b9d 100644 --- a/benchmarks/os_test.go +++ b/benchmarks/os_test.go @@ -4,7 +4,7 @@ import ( "os" "testing" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) func BenchmarkFileWrite(b *testing.B) { diff --git a/benchmarks/simu/counter.go b/benchmarks/simu/counter.go index dd00408cb..b7d2c4d63 100644 --- a/benchmarks/simu/counter.go +++ b/benchmarks/simu/counter.go @@ -7,7 +7,7 @@ import ( "time" rpcclient "github.com/tendermint/tendermint/rpc/lib/client" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) func main() { diff --git a/blockchain/pool.go b/blockchain/pool.go index 8b964e81a..e379d846a 100644 --- a/blockchain/pool.go +++ b/blockchain/pool.go @@ -8,9 +8,9 @@ import ( "sync/atomic" "time" - cmn "github.com/tendermint/tmlibs/common" - flow "github.com/tendermint/tmlibs/flowrate" - "github.com/tendermint/tmlibs/log" + cmn "github.com/tendermint/tendermint/libs/common" + flow "github.com/tendermint/tendermint/libs/flowrate" + "github.com/tendermint/tendermint/libs/log" "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/types" diff --git a/blockchain/pool_test.go b/blockchain/pool_test.go index 82120eaef..c2f615f94 100644 --- a/blockchain/pool_test.go +++ b/blockchain/pool_test.go @@ -5,8 +5,8 @@ import ( "testing" "time" - cmn "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tmlibs/log" + cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/libs/log" "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/types" diff --git a/blockchain/reactor.go b/blockchain/reactor.go index 33dfdd288..70a599bab 100644 --- a/blockchain/reactor.go +++ b/blockchain/reactor.go @@ -9,8 +9,8 @@ import ( "github.com/tendermint/tendermint/p2p" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" - cmn "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tmlibs/log" + cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/libs/log" ) const ( diff --git a/blockchain/reactor_test.go b/blockchain/reactor_test.go index c7f7e9afd..2bb6e9762 100644 --- a/blockchain/reactor_test.go +++ b/blockchain/reactor_test.go @@ -4,9 +4,9 @@ import ( "net" "testing" - cmn "github.com/tendermint/tmlibs/common" - dbm "github.com/tendermint/tmlibs/db" - "github.com/tendermint/tmlibs/log" + cmn "github.com/tendermint/tendermint/libs/common" + dbm "github.com/tendermint/tendermint/libs/db" + "github.com/tendermint/tendermint/libs/log" cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/p2p" diff --git a/blockchain/store.go b/blockchain/store.go index e7608b2cc..f02d4facb 100644 --- a/blockchain/store.go +++ b/blockchain/store.go @@ -4,8 +4,8 @@ import ( "fmt" "sync" - cmn "github.com/tendermint/tmlibs/common" - dbm "github.com/tendermint/tmlibs/db" + cmn "github.com/tendermint/tendermint/libs/common" + dbm "github.com/tendermint/tendermint/libs/db" "github.com/tendermint/tendermint/types" ) diff --git a/blockchain/store_test.go b/blockchain/store_test.go index a1bd0fd5d..888040bdf 100644 --- a/blockchain/store_test.go +++ b/blockchain/store_test.go @@ -10,8 +10,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/tendermint/tmlibs/db" - "github.com/tendermint/tmlibs/log" + "github.com/tendermint/tendermint/libs/db" + "github.com/tendermint/tendermint/libs/log" "github.com/tendermint/tendermint/types" ) @@ -29,7 +29,7 @@ func TestLoadBlockStoreStateJSON(t *testing.T) { func TestNewBlockStore(t *testing.T) { db := db.NewMemDB() - db.Set(blockStoreKey, []byte(`{"height": 10000}`)) + db.Set(blockStoreKey, []byte(`{"height": "10000"}`)) bs := NewBlockStore(db) require.Equal(t, int64(10000), bs.Height(), "failed to properly parse blockstore") @@ -153,14 +153,14 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) { parts: validPartSet, seenCommit: seenCommit1, corruptCommitInDB: true, // Corrupt the DB's commit entry - wantPanic: "Error reading block commit", + wantPanic: "unmarshal to types.Commit failed", }, { block: newBlock(&header1, commitAtH10), parts: validPartSet, seenCommit: seenCommit1, - wantPanic: "Error reading block", + wantPanic: "unmarshal to types.BlockMeta failed", corruptBlockInDB: true, // Corrupt the DB's block entry }, @@ -179,7 +179,7 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) { seenCommit: seenCommit1, corruptSeenCommitInDB: true, - wantPanic: "Error reading block seen commit", + wantPanic: "unmarshal to types.Commit failed", }, { @@ -287,7 +287,7 @@ func TestLoadBlockPart(t *testing.T) { db.Set(calcBlockPartKey(height, index), []byte("Tendermint")) res, _, panicErr = doFn(loadPart) require.NotNil(t, panicErr, "expecting a non-nil panic") - require.Contains(t, panicErr.Error(), "Error reading block part") + require.Contains(t, panicErr.Error(), "unmarshal to types.Part failed") // 3. A good block serialized and saved to the DB should be retrievable db.Set(calcBlockPartKey(height, index), cdc.MustMarshalBinaryBare(part1)) @@ -316,7 +316,7 @@ func TestLoadBlockMeta(t *testing.T) { db.Set(calcBlockMetaKey(height), []byte("Tendermint-Meta")) res, _, panicErr = doFn(loadMeta) require.NotNil(t, panicErr, "expecting a non-nil panic") - require.Contains(t, panicErr.Error(), "Error reading block meta") + require.Contains(t, panicErr.Error(), "unmarshal to types.BlockMeta") // 3. A good blockMeta serialized and saved to the DB should be retrievable meta := &types.BlockMeta{} diff --git a/blockchain/wire.go b/blockchain/wire.go index 55b4e60ae..70b50565d 100644 --- a/blockchain/wire.go +++ b/blockchain/wire.go @@ -2,7 +2,7 @@ package blockchain import ( "github.com/tendermint/go-amino" - "github.com/tendermint/go-crypto" + "github.com/tendermint/tendermint/crypto" ) var cdc = amino.NewCodec() diff --git a/cmd/priv_val_server/main.go b/cmd/priv_val_server/main.go index c78adeb82..20c23f4c4 100644 --- a/cmd/priv_val_server/main.go +++ b/cmd/priv_val_server/main.go @@ -4,9 +4,9 @@ import ( "flag" "os" - crypto "github.com/tendermint/go-crypto" - cmn "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tmlibs/log" + crypto "github.com/tendermint/tendermint/crypto" + cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/libs/log" "github.com/tendermint/tendermint/privval" ) diff --git a/cmd/tendermint/commands/gen_node_key.go b/cmd/tendermint/commands/gen_node_key.go index 4990be477..7aedcd0dc 100644 --- a/cmd/tendermint/commands/gen_node_key.go +++ b/cmd/tendermint/commands/gen_node_key.go @@ -6,7 +6,7 @@ import ( "github.com/spf13/cobra" "github.com/tendermint/tendermint/p2p" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) // GenNodeKeyCmd allows the generation of a node key. It prints node's ID to diff --git a/cmd/tendermint/commands/init.go b/cmd/tendermint/commands/init.go index 45812b9ed..a44c73ebf 100644 --- a/cmd/tendermint/commands/init.go +++ b/cmd/tendermint/commands/init.go @@ -9,7 +9,7 @@ import ( "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/privval" "github.com/tendermint/tendermint/types" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) // InitFilesCmd initialises a fresh Tendermint Core instance. @@ -52,8 +52,9 @@ func initFilesWithConfig(config *cfg.Config) error { logger.Info("Found genesis file", "path", genFile) } else { genDoc := types.GenesisDoc{ - ChainID: cmn.Fmt("test-chain-%v", cmn.RandStr(6)), - GenesisTime: time.Now(), + ChainID: cmn.Fmt("test-chain-%v", cmn.RandStr(6)), + GenesisTime: time.Now(), + ConsensusParams: types.DefaultConsensusParams(), } genDoc.Validators = []types.GenesisValidator{{ PubKey: pv.GetPubKey(), diff --git a/cmd/tendermint/commands/lite.go b/cmd/tendermint/commands/lite.go index 6987b7f19..d57598816 100644 --- a/cmd/tendermint/commands/lite.go +++ b/cmd/tendermint/commands/lite.go @@ -6,7 +6,7 @@ import ( "github.com/spf13/cobra" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" "github.com/tendermint/tendermint/lite/proxy" rpcclient "github.com/tendermint/tendermint/rpc/client" diff --git a/cmd/tendermint/commands/reset_priv_validator.go b/cmd/tendermint/commands/reset_priv_validator.go index 32d7b1433..ef0ba3019 100644 --- a/cmd/tendermint/commands/reset_priv_validator.go +++ b/cmd/tendermint/commands/reset_priv_validator.go @@ -6,7 +6,7 @@ import ( "github.com/spf13/cobra" "github.com/tendermint/tendermint/privval" - "github.com/tendermint/tmlibs/log" + "github.com/tendermint/tendermint/libs/log" ) // ResetAllCmd removes the database of this Tendermint core diff --git a/cmd/tendermint/commands/root.go b/cmd/tendermint/commands/root.go index f229a7889..3c67ddc14 100644 --- a/cmd/tendermint/commands/root.go +++ b/cmd/tendermint/commands/root.go @@ -7,9 +7,9 @@ import ( "github.com/spf13/viper" cfg "github.com/tendermint/tendermint/config" - "github.com/tendermint/tmlibs/cli" - tmflags "github.com/tendermint/tmlibs/cli/flags" - "github.com/tendermint/tmlibs/log" + "github.com/tendermint/tendermint/libs/cli" + tmflags "github.com/tendermint/tendermint/libs/cli/flags" + "github.com/tendermint/tendermint/libs/log" ) var ( diff --git a/cmd/tendermint/commands/root_test.go b/cmd/tendermint/commands/root_test.go index 59d258af7..e8095b387 100644 --- a/cmd/tendermint/commands/root_test.go +++ b/cmd/tendermint/commands/root_test.go @@ -14,8 +14,8 @@ import ( "github.com/stretchr/testify/require" cfg "github.com/tendermint/tendermint/config" - "github.com/tendermint/tmlibs/cli" - cmn "github.com/tendermint/tmlibs/common" + "github.com/tendermint/tendermint/libs/cli" + cmn "github.com/tendermint/tendermint/libs/common" ) var ( diff --git a/cmd/tendermint/commands/run_node.go b/cmd/tendermint/commands/run_node.go index 0d50f9e4b..542e5c991 100644 --- a/cmd/tendermint/commands/run_node.go +++ b/cmd/tendermint/commands/run_node.go @@ -33,7 +33,7 @@ func AddNodeFlags(cmd *cobra.Command) { cmd.Flags().String("p2p.laddr", config.P2P.ListenAddress, "Node listen address. (0.0.0.0:0 means any interface, any port)") cmd.Flags().String("p2p.seeds", config.P2P.Seeds, "Comma-delimited ID@host:port seed nodes") cmd.Flags().String("p2p.persistent_peers", config.P2P.PersistentPeers, "Comma-delimited ID@host:port persistent peers") - cmd.Flags().Bool("p2p.skip_upnp", config.P2P.SkipUPNP, "Skip UPNP configuration") + cmd.Flags().Bool("p2p.upnp", config.P2P.UPNP, "Enable/disable UPNP port forwarding") cmd.Flags().Bool("p2p.pex", config.P2P.PexReactor, "Enable/disable Peer-Exchange") cmd.Flags().Bool("p2p.seed_mode", config.P2P.SeedMode, "Enable/disable seed mode") cmd.Flags().String("p2p.private_peer_ids", config.P2P.PrivatePeerIDs, "Comma-delimited private peer IDs") diff --git a/cmd/tendermint/commands/testnet.go b/cmd/tendermint/commands/testnet.go index 29d29502e..f7639fb27 100644 --- a/cmd/tendermint/commands/testnet.go +++ b/cmd/tendermint/commands/testnet.go @@ -14,7 +14,7 @@ import ( "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/privval" "github.com/tendermint/tendermint/types" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) var ( diff --git a/cmd/tendermint/commands/wire.go b/cmd/tendermint/commands/wire.go index 4c133a8c5..a09019133 100644 --- a/cmd/tendermint/commands/wire.go +++ b/cmd/tendermint/commands/wire.go @@ -2,7 +2,7 @@ package commands import ( "github.com/tendermint/go-amino" - "github.com/tendermint/go-crypto" + "github.com/tendermint/tendermint/crypto" ) var cdc = amino.NewCodec() diff --git a/cmd/tendermint/main.go b/cmd/tendermint/main.go index 8c7f0cd17..a5a8d2d80 100644 --- a/cmd/tendermint/main.go +++ b/cmd/tendermint/main.go @@ -4,7 +4,7 @@ import ( "os" "path/filepath" - "github.com/tendermint/tmlibs/cli" + "github.com/tendermint/tendermint/libs/cli" cmd "github.com/tendermint/tendermint/cmd/tendermint/commands" cfg "github.com/tendermint/tendermint/config" diff --git a/config/config.go b/config/config.go index 5ba568f26..22cecf989 100644 --- a/config/config.go +++ b/config/config.go @@ -45,34 +45,37 @@ type Config struct { BaseConfig `mapstructure:",squash"` // Options for services - RPC *RPCConfig `mapstructure:"rpc"` - P2P *P2PConfig `mapstructure:"p2p"` - Mempool *MempoolConfig `mapstructure:"mempool"` - Consensus *ConsensusConfig `mapstructure:"consensus"` - TxIndex *TxIndexConfig `mapstructure:"tx_index"` + RPC *RPCConfig `mapstructure:"rpc"` + P2P *P2PConfig `mapstructure:"p2p"` + Mempool *MempoolConfig `mapstructure:"mempool"` + Consensus *ConsensusConfig `mapstructure:"consensus"` + TxIndex *TxIndexConfig `mapstructure:"tx_index"` + Instrumentation *InstrumentationConfig `mapstructure:"instrumentation"` } // DefaultConfig returns a default configuration for a Tendermint node func DefaultConfig() *Config { return &Config{ - BaseConfig: DefaultBaseConfig(), - RPC: DefaultRPCConfig(), - P2P: DefaultP2PConfig(), - Mempool: DefaultMempoolConfig(), - Consensus: DefaultConsensusConfig(), - TxIndex: DefaultTxIndexConfig(), + BaseConfig: DefaultBaseConfig(), + RPC: DefaultRPCConfig(), + P2P: DefaultP2PConfig(), + Mempool: DefaultMempoolConfig(), + Consensus: DefaultConsensusConfig(), + TxIndex: DefaultTxIndexConfig(), + Instrumentation: DefaultInstrumentationConfig(), } } // TestConfig returns a configuration that can be used for testing func TestConfig() *Config { return &Config{ - BaseConfig: TestBaseConfig(), - RPC: TestRPCConfig(), - P2P: TestP2PConfig(), - Mempool: TestMempoolConfig(), - Consensus: TestConsensusConfig(), - TxIndex: TestTxIndexConfig(), + BaseConfig: TestBaseConfig(), + RPC: TestRPCConfig(), + P2P: TestP2PConfig(), + Mempool: TestMempoolConfig(), + Consensus: TestConsensusConfig(), + TxIndex: TestTxIndexConfig(), + Instrumentation: TestInstrumentationConfig(), } } @@ -221,16 +224,36 @@ type RPCConfig struct { // NOTE: This server only supports /broadcast_tx_commit GRPCListenAddress string `mapstructure:"grpc_laddr"` + // Maximum number of simultaneous connections. + // Does not include RPC (HTTP&WebSocket) connections. See max_open_connections + // If you want to accept more significant number than the default, make sure + // you increase your OS limits. + // 0 - unlimited. + GRPCMaxOpenConnections int `mapstructure:"grpc_max_open_connections"` + // Activate unsafe RPC commands like /dial_persistent_peers and /unsafe_flush_mempool Unsafe bool `mapstructure:"unsafe"` + + // Maximum number of simultaneous connections (including WebSocket). + // Does not include gRPC connections. See grpc_max_open_connections + // If you want to accept more significant number than the default, make sure + // you increase your OS limits. + // 0 - unlimited. + MaxOpenConnections int `mapstructure:"max_open_connections"` } // DefaultRPCConfig returns a default configuration for the RPC server func DefaultRPCConfig() *RPCConfig { return &RPCConfig{ - ListenAddress: "tcp://0.0.0.0:26657", - GRPCListenAddress: "", - Unsafe: false, + ListenAddress: "tcp://0.0.0.0:26657", + + GRPCListenAddress: "", + GRPCMaxOpenConnections: 900, // no ipv4 + + Unsafe: false, + // should be < {ulimit -Sn} - {MaxNumPeers} - {N of wal, db and other open files} + // 1024 - 50 - 50 = 924 = ~900 + MaxOpenConnections: 900, } } @@ -253,6 +276,9 @@ type P2PConfig struct { // Address to listen for incoming connections ListenAddress string `mapstructure:"laddr"` + // Address to advertise to peers for them to dial + ExternalAddress string `mapstructure:"external_address"` + // Comma separated list of seed nodes to connect to // We only use these if we can’t connect to peers in the addrbook Seeds string `mapstructure:"seeds"` @@ -261,8 +287,8 @@ type P2PConfig struct { // Do not add private peers to this list if you don't want them advertised PersistentPeers string `mapstructure:"persistent_peers"` - // Skip UPNP port forwarding - SkipUPNP bool `mapstructure:"skip_upnp"` + // UPNP port forwarding + UPNP bool `mapstructure:"upnp"` // Path to address book AddrBook string `mapstructure:"addr_book_file"` @@ -317,6 +343,8 @@ type P2PConfig struct { func DefaultP2PConfig() *P2PConfig { return &P2PConfig{ ListenAddress: "tcp://0.0.0.0:26656", + ExternalAddress: "", + UPNP: false, AddrBook: defaultAddrBookPath, AddrBookStrict: true, MaxNumPeers: 50, @@ -339,7 +367,6 @@ func DefaultP2PConfig() *P2PConfig { func TestP2PConfig() *P2PConfig { cfg := DefaultP2PConfig() cfg.ListenAddress = "tcp://0.0.0.0:36656" - cfg.SkipUPNP = true cfg.FlushThrottleTimeout = 10 cfg.AllowDuplicateIP = true return cfg @@ -411,7 +438,7 @@ func (cfg *MempoolConfig) WalDir() string { //----------------------------------------------------------------------------- // ConsensusConfig -// ConsensusConfig defines the confuguration for the Tendermint consensus service, +// ConsensusConfig defines the configuration for the Tendermint consensus service, // including timeouts and details about the WAL and the block structure. type ConsensusConfig struct { RootDir string `mapstructure:"home"` @@ -430,10 +457,6 @@ type ConsensusConfig struct { // Make progress as soon as we have all the precommits (as if TimeoutCommit = 0) SkipTimeoutCommit bool `mapstructure:"skip_timeout_commit"` - // BlockSize - MaxBlockSizeTxs int `mapstructure:"max_block_size_txs"` - MaxBlockSizeBytes int `mapstructure:"max_block_size_bytes"` - // EmptyBlocks mode and possible interval between empty blocks in seconds CreateEmptyBlocks bool `mapstructure:"create_empty_blocks"` CreateEmptyBlocksInterval int `mapstructure:"create_empty_blocks_interval"` @@ -455,8 +478,6 @@ func DefaultConsensusConfig() *ConsensusConfig { TimeoutPrecommitDelta: 500, TimeoutCommit: 1000, SkipTimeoutCommit: false, - MaxBlockSizeTxs: 10000, - MaxBlockSizeBytes: 1, // TODO CreateEmptyBlocks: true, CreateEmptyBlocksInterval: 0, PeerGossipSleepDuration: 100, @@ -536,14 +557,14 @@ func (cfg *ConsensusConfig) SetWalFile(walFile string) { //----------------------------------------------------------------------------- // TxIndexConfig -// TxIndexConfig defines the confuguration for the transaction +// TxIndexConfig defines the configuration for the transaction // indexer, including tags to index. type TxIndexConfig struct { // What indexer to use for transactions // // Options: - // 1) "null" (default) - // 2) "kv" - the simplest possible indexer, backed by key-value storage (defaults to levelDB; see DBBackend). + // 1) "null" + // 2) "kv" (default) - the simplest possible indexer, backed by key-value storage (defaults to levelDB; see DBBackend). Indexer string `mapstructure:"indexer"` // Comma-separated list of tags to index (by default the only tag is tx hash) @@ -573,6 +594,35 @@ func TestTxIndexConfig() *TxIndexConfig { return DefaultTxIndexConfig() } +//----------------------------------------------------------------------------- +// InstrumentationConfig + +// InstrumentationConfig defines the configuration for metrics reporting. +type InstrumentationConfig struct { + // When true, Prometheus metrics are served under /metrics on + // PrometheusListenAddr. + // Check out the documentation for the list of available metrics. + Prometheus bool `mapstructure:"prometheus"` + + // Address to listen for Prometheus collector(s) connections. + PrometheusListenAddr string `mapstructure:"prometheus_listen_addr"` +} + +// DefaultInstrumentationConfig returns a default configuration for metrics +// reporting. +func DefaultInstrumentationConfig() *InstrumentationConfig { + return &InstrumentationConfig{ + Prometheus: false, + PrometheusListenAddr: ":26660", + } +} + +// TestInstrumentationConfig returns a default configuration for metrics +// reporting. +func TestInstrumentationConfig() *InstrumentationConfig { + return DefaultInstrumentationConfig() +} + //----------------------------------------------------------------------------- // Utils diff --git a/config/toml.go b/config/toml.go index 7ed3e971d..084325baa 100644 --- a/config/toml.go +++ b/config/toml.go @@ -6,7 +6,7 @@ import ( "path/filepath" "text/template" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) var configTemplate *template.Template @@ -119,15 +119,35 @@ laddr = "{{ .RPC.ListenAddress }}" # NOTE: This server only supports /broadcast_tx_commit grpc_laddr = "{{ .RPC.GRPCListenAddress }}" +# Maximum number of simultaneous connections. +# Does not include RPC (HTTP&WebSocket) connections. See max_open_connections +# If you want to accept more significant number than the default, make sure +# you increase your OS limits. +# 0 - unlimited. +grpc_max_open_connections = {{ .RPC.GRPCMaxOpenConnections }} + # Activate unsafe RPC commands like /dial_seeds and /unsafe_flush_mempool unsafe = {{ .RPC.Unsafe }} +# Maximum number of simultaneous connections (including WebSocket). +# Does not include gRPC connections. See grpc_max_open_connections +# If you want to accept more significant number than the default, make sure +# you increase your OS limits. +# 0 - unlimited. +max_open_connections = {{ .RPC.MaxOpenConnections }} + ##### peer to peer configuration options ##### [p2p] # Address to listen for incoming connections laddr = "{{ .P2P.ListenAddress }}" +# Address to advertise to peers for them to dial +# If empty, will use the same port as the laddr, +# and will introspect on the listener or use UPnP +# to figure out the address. +external_address = "{{ .P2P.ExternalAddress }}" + # Comma separated list of seed nodes to connect to seeds = "{{ .P2P.Seeds }}" @@ -135,6 +155,9 @@ seeds = "{{ .P2P.Seeds }}" # Do not add private peers to this list if you don't want them advertised persistent_peers = "{{ .P2P.PersistentPeers }}" +# UPNP port forwarding +upnp = {{ .P2P.UPNP }} + # Path to address book addr_book_file = "{{ js .P2P.AddrBook }}" @@ -199,10 +222,6 @@ timeout_commit = {{ .Consensus.TimeoutCommit }} # Make progress as soon as we have all the precommits (as if TimeoutCommit = 0) skip_timeout_commit = {{ .Consensus.SkipTimeoutCommit }} -# BlockSize -max_block_size_txs = {{ .Consensus.MaxBlockSizeTxs }} -max_block_size_bytes = {{ .Consensus.MaxBlockSizeBytes }} - # EmptyBlocks mode and possible interval between empty blocks in seconds create_empty_blocks = {{ .Consensus.CreateEmptyBlocks }} create_empty_blocks_interval = {{ .Consensus.CreateEmptyBlocksInterval }} @@ -232,6 +251,17 @@ index_tags = "{{ .TxIndex.IndexTags }}" # desirable (see the comment above). IndexTags has a precedence over # IndexAllTags (i.e. when given both, IndexTags will be indexed). index_all_tags = {{ .TxIndex.IndexAllTags }} + +##### instrumentation configuration options ##### +[instrumentation] + +# When true, Prometheus metrics are served under /metrics on +# PrometheusListenAddr. +# Check out the documentation for the list of available metrics. +prometheus = {{ .Instrumentation.Prometheus }} + +# Address to listen for Prometheus collector(s) connections +prometheus_listen_addr = "{{ .Instrumentation.PrometheusListenAddr }}" ` /****** these are for test settings ***********/ @@ -287,10 +317,10 @@ var testGenesis = `{ "validators": [ { "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value":"AT/+aaL1eB0477Mud9JMm8Sh8BIvOYlPGC9KkIUmFaE=" }, - "power": 10, + "power": "10", "name": "" } ], @@ -298,16 +328,16 @@ var testGenesis = `{ }` var testPrivValidator = `{ - "address": "849CB2C877F87A20925F35D00AE6688342D25B47", + "address": "A3258DCBF45DCA0DF052981870F2D1441A36D145", "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "AT/+aaL1eB0477Mud9JMm8Sh8BIvOYlPGC9KkIUmFaE=" }, "priv_key": { - "type": "954568A3288910", + "type": "tendermint/PrivKeyEd25519", "value": "EVkqJO/jIXp3rkASXfh9YnyToYXRXhBr6g9cQVxPFnQBP/5povV4HTjvsy530kybxKHwEi85iU8YL0qQhSYVoQ==" }, - "last_height": 0, - "last_round": 0, + "last_height": "0", + "last_round": "0", "last_step": 0 }` diff --git a/consensus/byzantine_test.go b/consensus/byzantine_test.go index d3be8c358..5360a92c9 100644 --- a/consensus/byzantine_test.go +++ b/consensus/byzantine_test.go @@ -9,7 +9,7 @@ import ( "github.com/stretchr/testify/require" "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/types" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) func init() { diff --git a/consensus/common_test.go b/consensus/common_test.go index f50e57699..2df226ba1 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -12,8 +12,8 @@ import ( "testing" "time" - abcicli "github.com/tendermint/abci/client" - abci "github.com/tendermint/abci/types" + abcicli "github.com/tendermint/tendermint/abci/client" + abci "github.com/tendermint/tendermint/abci/types" bc "github.com/tendermint/tendermint/blockchain" cfg "github.com/tendermint/tendermint/config" cstypes "github.com/tendermint/tendermint/consensus/types" @@ -22,12 +22,12 @@ import ( "github.com/tendermint/tendermint/privval" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" - cmn "github.com/tendermint/tmlibs/common" - dbm "github.com/tendermint/tmlibs/db" - "github.com/tendermint/tmlibs/log" + cmn "github.com/tendermint/tendermint/libs/common" + dbm "github.com/tendermint/tendermint/libs/db" + "github.com/tendermint/tendermint/libs/log" - "github.com/tendermint/abci/example/counter" - "github.com/tendermint/abci/example/kvstore" + "github.com/tendermint/tendermint/abci/example/counter" + "github.com/tendermint/tendermint/abci/example/kvstore" "github.com/go-kit/kit/log/term" ) diff --git a/consensus/mempool_test.go b/consensus/mempool_test.go index 3c8180060..a811de731 100644 --- a/consensus/mempool_test.go +++ b/consensus/mempool_test.go @@ -8,9 +8,9 @@ import ( "github.com/stretchr/testify/assert" - "github.com/tendermint/abci/example/code" - abci "github.com/tendermint/abci/types" - cmn "github.com/tendermint/tmlibs/common" + "github.com/tendermint/tendermint/abci/example/code" + abci "github.com/tendermint/tendermint/abci/types" + cmn "github.com/tendermint/tendermint/libs/common" "github.com/tendermint/tendermint/types" ) diff --git a/consensus/metrics.go b/consensus/metrics.go new file mode 100644 index 000000000..253880e84 --- /dev/null +++ b/consensus/metrics.go @@ -0,0 +1,133 @@ +package consensus + +import ( + "github.com/go-kit/kit/metrics" + "github.com/go-kit/kit/metrics/discard" + + prometheus "github.com/go-kit/kit/metrics/prometheus" + stdprometheus "github.com/prometheus/client_golang/prometheus" +) + +// Metrics contains metrics exposed by this package. +type Metrics struct { + // Height of the chain. + Height metrics.Gauge + + // Number of rounds. + Rounds metrics.Gauge + + // Number of validators. + Validators metrics.Gauge + // Total power of all validators. + ValidatorsPower metrics.Gauge + // Number of validators who did not sign. + MissingValidators metrics.Gauge + // Total power of the missing validators. + MissingValidatorsPower metrics.Gauge + // Number of validators who tried to double sign. + ByzantineValidators metrics.Gauge + // Total power of the byzantine validators. + ByzantineValidatorsPower metrics.Gauge + + // Time between this and the last block. + BlockIntervalSeconds metrics.Histogram + + // Number of transactions. + NumTxs metrics.Gauge + // Size of the block. + BlockSizeBytes metrics.Gauge + // Total number of transactions. + TotalTxs metrics.Gauge +} + +// PrometheusMetrics returns Metrics build using Prometheus client library. +func PrometheusMetrics() *Metrics { + return &Metrics{ + Height: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Subsystem: "consensus", + Name: "height", + Help: "Height of the chain.", + }, []string{}), + Rounds: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Subsystem: "consensus", + Name: "rounds", + Help: "Number of rounds.", + }, []string{}), + + Validators: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Subsystem: "consensus", + Name: "validators", + Help: "Number of validators.", + }, []string{}), + ValidatorsPower: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Subsystem: "consensus", + Name: "validators_power", + Help: "Total power of all validators.", + }, []string{}), + MissingValidators: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Subsystem: "consensus", + Name: "missing_validators", + Help: "Number of validators who did not sign.", + }, []string{}), + MissingValidatorsPower: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Subsystem: "consensus", + Name: "missing_validators_power", + Help: "Total power of the missing validators.", + }, []string{}), + ByzantineValidators: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Subsystem: "consensus", + Name: "byzantine_validators", + Help: "Number of validators who tried to double sign.", + }, []string{}), + ByzantineValidatorsPower: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Subsystem: "consensus", + Name: "byzantine_validators_power", + Help: "Total power of the byzantine validators.", + }, []string{}), + + BlockIntervalSeconds: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{ + Subsystem: "consensus", + Name: "block_interval_seconds", + Help: "Time between this and the last block.", + Buckets: []float64{1, 2.5, 5, 10, 60}, + }, []string{}), + + NumTxs: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Subsystem: "consensus", + Name: "num_txs", + Help: "Number of transactions.", + }, []string{}), + BlockSizeBytes: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Subsystem: "consensus", + Name: "block_size_bytes", + Help: "Size of the block.", + }, []string{}), + TotalTxs: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Subsystem: "consensus", + Name: "total_txs", + Help: "Total number of transactions.", + }, []string{}), + } +} + +// NopMetrics returns no-op Metrics. +func NopMetrics() *Metrics { + return &Metrics{ + Height: discard.NewGauge(), + + Rounds: discard.NewGauge(), + + Validators: discard.NewGauge(), + ValidatorsPower: discard.NewGauge(), + MissingValidators: discard.NewGauge(), + MissingValidatorsPower: discard.NewGauge(), + ByzantineValidators: discard.NewGauge(), + ByzantineValidatorsPower: discard.NewGauge(), + + BlockIntervalSeconds: discard.NewHistogram(), + + NumTxs: discard.NewGauge(), + BlockSizeBytes: discard.NewGauge(), + TotalTxs: discard.NewGauge(), + } +} diff --git a/consensus/reactor.go b/consensus/reactor.go index 2034ad344..54407ae1e 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -9,8 +9,8 @@ import ( "github.com/pkg/errors" amino "github.com/tendermint/go-amino" - cmn "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tmlibs/log" + cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/libs/log" cstypes "github.com/tendermint/tendermint/consensus/types" tmevents "github.com/tendermint/tendermint/libs/events" diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index 0d9971192..9e2aa0a0b 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -10,9 +10,9 @@ import ( "testing" "time" - "github.com/tendermint/abci/example/kvstore" - cmn "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tmlibs/log" + "github.com/tendermint/tendermint/abci/example/kvstore" + cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/libs/log" cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/p2p" diff --git a/consensus/replay.go b/consensus/replay.go index 13ec9e403..3035f75d8 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -10,11 +10,11 @@ import ( //"strings" "time" - abci "github.com/tendermint/abci/types" - //auto "github.com/tendermint/tmlibs/autofile" - cmn "github.com/tendermint/tmlibs/common" - dbm "github.com/tendermint/tmlibs/db" - "github.com/tendermint/tmlibs/log" + abci "github.com/tendermint/tendermint/abci/types" + //auto "github.com/tendermint/tendermint/libs/autofile" + cmn "github.com/tendermint/tendermint/libs/common" + dbm "github.com/tendermint/tendermint/libs/db" + "github.com/tendermint/tendermint/libs/log" "github.com/tendermint/tendermint/proxy" sm "github.com/tendermint/tendermint/state" diff --git a/consensus/replay_file.go b/consensus/replay_file.go index 57204b01a..0c0b0dcb1 100644 --- a/consensus/replay_file.go +++ b/consensus/replay_file.go @@ -16,9 +16,9 @@ import ( "github.com/tendermint/tendermint/proxy" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" - cmn "github.com/tendermint/tmlibs/common" - dbm "github.com/tendermint/tmlibs/db" - "github.com/tendermint/tmlibs/log" + cmn "github.com/tendermint/tendermint/libs/common" + dbm "github.com/tendermint/tendermint/libs/db" + "github.com/tendermint/tendermint/libs/log" ) const ( diff --git a/consensus/replay_test.go b/consensus/replay_test.go index 725568ed5..da526d249 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -16,19 +16,19 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/tendermint/abci/example/kvstore" - abci "github.com/tendermint/abci/types" - crypto "github.com/tendermint/go-crypto" - auto "github.com/tendermint/tmlibs/autofile" - cmn "github.com/tendermint/tmlibs/common" - dbm "github.com/tendermint/tmlibs/db" + "github.com/tendermint/tendermint/abci/example/kvstore" + abci "github.com/tendermint/tendermint/abci/types" + crypto "github.com/tendermint/tendermint/crypto" + auto "github.com/tendermint/tendermint/libs/autofile" + cmn "github.com/tendermint/tendermint/libs/common" + dbm "github.com/tendermint/tendermint/libs/db" cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/privval" "github.com/tendermint/tendermint/proxy" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" - "github.com/tendermint/tmlibs/log" + "github.com/tendermint/tendermint/libs/log" ) var consensusReplayConfig *cfg.Config diff --git a/consensus/state.go b/consensus/state.go index 3834b1515..d013f4231 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -10,8 +10,8 @@ import ( "time" fail "github.com/ebuchman/fail-test" - cmn "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tmlibs/log" + cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/libs/log" cfg "github.com/tendermint/tendermint/config" cstypes "github.com/tendermint/tendermint/consensus/types" @@ -115,10 +115,24 @@ type ConsensusState struct { // synchronous pubsub between consensus state and reactor. // state only emits EventNewRoundStep, EventVote and EventProposalHeartbeat evsw tmevents.EventSwitch + + // for reporting metrics + metrics *Metrics } +// CSOption sets an optional parameter on the ConsensusState. +type CSOption func(*ConsensusState) + // NewConsensusState returns a new ConsensusState. -func NewConsensusState(config *cfg.ConsensusConfig, state sm.State, blockExec *sm.BlockExecutor, blockStore sm.BlockStore, mempool sm.Mempool, evpool sm.EvidencePool) *ConsensusState { +func NewConsensusState( + config *cfg.ConsensusConfig, + state sm.State, + blockExec *sm.BlockExecutor, + blockStore sm.BlockStore, + mempool sm.Mempool, + evpool sm.EvidencePool, + options ...CSOption, +) *ConsensusState { cs := &ConsensusState{ config: config, blockExec: blockExec, @@ -132,6 +146,7 @@ func NewConsensusState(config *cfg.ConsensusConfig, state sm.State, blockExec *s wal: nilWAL{}, evpool: evpool, evsw: tmevents.NewEventSwitch(), + metrics: NopMetrics(), } // set function defaults (may be overwritten before calling Start) cs.decideProposal = cs.defaultDecideProposal @@ -143,6 +158,9 @@ func NewConsensusState(config *cfg.ConsensusConfig, state sm.State, blockExec *s // We do that upon Start(). cs.reconstructLastCommit(state) cs.BaseService = *cmn.NewBaseService(nil, "ConsensusState", cs) + for _, option := range options { + option(cs) + } return cs } @@ -161,6 +179,11 @@ func (cs *ConsensusState) SetEventBus(b *types.EventBus) { cs.blockExec.SetEventBus(b) } +// WithMetrics sets the metrics. +func WithMetrics(metrics *Metrics) CSOption { + return func(cs *ConsensusState) { cs.metrics = metrics } +} + // String returns a string. func (cs *ConsensusState) String() string { // better not to access shared variables @@ -297,10 +320,7 @@ func (cs *ConsensusState) OnStop() { cs.timeoutTicker.Stop() - // Make BaseService.Wait() wait until cs.wal.Wait() - if cs.IsRunning() { - cs.wal.Wait() - } + cs.wal.Stop() } // Wait waits for the the main routine to return. @@ -387,6 +407,7 @@ func (cs *ConsensusState) SetProposalAndBlock(proposal *types.Proposal, block *t // internal functions for managing the state func (cs *ConsensusState) updateHeight(height int64) { + cs.metrics.Height.Set(float64(height)) cs.Height = height } @@ -600,7 +621,7 @@ func (cs *ConsensusState) handleMsg(mi msgInfo) { err = cs.setProposal(msg.Proposal) case *BlockPartMessage: // if the proposal is complete, we'll enterPrevote or tryFinalizeCommit - _, err = cs.addProposalBlockPart(msg.Height, msg.Part) + _, err = cs.addProposalBlockPart(msg, peerID) if err != nil && msg.Round != cs.Round { cs.Logger.Debug("Received block part from wrong round", "height", cs.Height, "csRound", cs.Round, "blockRound", msg.Round) err = nil @@ -722,6 +743,7 @@ func (cs *ConsensusState) enterNewRound(height int64, round int) { cs.Votes.SetRound(round + 1) // also track next round (round+1) to allow round-skipping cs.eventBus.PublishEventNewRound(cs.RoundStateEvent()) + cs.metrics.Rounds.Set(float64(round)) // Wait for txs to be available in the mempool // before we enterPropose in round 0. If the last block changed the app hash, @@ -907,7 +929,7 @@ func (cs *ConsensusState) createProposalBlock() (block *types.Block, blockParts } // Mempool validated transactions - txs := cs.mempool.Reap(cs.config.MaxBlockSizeTxs) + txs := cs.mempool.Reap(cs.state.ConsensusParams.BlockSize.MaxTxs) block, parts := cs.state.MakeBlock(cs.Height, txs, commit) evidence := cs.evpool.PendingEvidence() block.AddEvidence(evidence) @@ -1280,6 +1302,9 @@ func (cs *ConsensusState) finalizeCommit(height int64) { fail.Fail() // XXX + // must be called before we update state + cs.recordMetrics(height, block) + // NewHeightStep! cs.updateToState(stateCopy) @@ -1295,6 +1320,44 @@ func (cs *ConsensusState) finalizeCommit(height int64) { // * cs.StartTime is set to when we will start round0. } +func (cs *ConsensusState) recordMetrics(height int64, block *types.Block) { + cs.metrics.Validators.Set(float64(cs.Validators.Size())) + cs.metrics.ValidatorsPower.Set(float64(cs.Validators.TotalVotingPower())) + missingValidators := 0 + missingValidatorsPower := int64(0) + for i, val := range cs.Validators.Validators { + var vote *types.Vote + if i < len(block.LastCommit.Precommits) { + vote = block.LastCommit.Precommits[i] + } + if vote == nil { + missingValidators++ + missingValidatorsPower += val.VotingPower + } + } + cs.metrics.MissingValidators.Set(float64(missingValidators)) + cs.metrics.MissingValidatorsPower.Set(float64(missingValidatorsPower)) + cs.metrics.ByzantineValidators.Set(float64(len(block.Evidence.Evidence))) + byzantineValidatorsPower := int64(0) + for _, ev := range block.Evidence.Evidence { + if _, val := cs.Validators.GetByAddress(ev.Address()); val != nil { + byzantineValidatorsPower += val.VotingPower + } + } + cs.metrics.ByzantineValidatorsPower.Set(float64(byzantineValidatorsPower)) + + if height > 1 { + lastBlockMeta := cs.blockStore.LoadBlockMeta(height - 1) + cs.metrics.BlockIntervalSeconds.Observe( + block.Time.Sub(lastBlockMeta.Header.Time).Seconds(), + ) + } + + cs.metrics.NumTxs.Set(float64(block.NumTxs)) + cs.metrics.BlockSizeBytes.Set(float64(block.Size())) + cs.metrics.TotalTxs.Set(float64(block.TotalTxs)) +} + //----------------------------------------------------------------------------- func (cs *ConsensusState) defaultSetProposal(proposal *types.Proposal) error { @@ -1333,17 +1396,22 @@ func (cs *ConsensusState) defaultSetProposal(proposal *types.Proposal) error { // NOTE: block is not necessarily valid. // Asynchronously triggers either enterPrevote (before we timeout of propose) or tryFinalizeCommit, once we have the full block. -func (cs *ConsensusState) addProposalBlockPart(height int64, part *types.Part) (added bool, err error) { +func (cs *ConsensusState) addProposalBlockPart(msg *BlockPartMessage, peerID p2p.ID) (added bool, err error) { + height, round, part := msg.Height, msg.Round, msg.Part + // Blocks might be reused, so round mismatch is OK if cs.Height != height { - cs.Logger.Debug("Received block part from wrong height", "height", height) + cs.Logger.Debug("Received block part from wrong height", "height", height, "round", round) return false, nil } // We're not expecting a block part. if cs.ProposalBlockParts == nil { - cs.Logger.Info("Received a block part when we're not expecting any", "height", height) - return false, nil // TODO: bad peer? Return error? + // NOTE: this can happen when we've gone to a higher round and + // then receive parts from the previous round - not necessarily a bad peer. + cs.Logger.Info("Received a block part when we're not expecting any", + "height", height, "round", round, "index", part.Index, "peer", peerID) + return false, nil } added, err = cs.ProposalBlockParts.AddPart(part) @@ -1377,7 +1445,7 @@ func (cs *ConsensusState) addProposalBlockPart(height int64, part *types.Part) ( // procedure at this point. } - if cs.Step == cstypes.RoundStepPropose && cs.isProposalComplete() { + if cs.Step <= cstypes.RoundStepPropose && cs.isProposalComplete() { // Move onto the next step cs.enterPrevote(height, cs.Round) } else if cs.Step == cstypes.RoundStepCommit { diff --git a/consensus/state_test.go b/consensus/state_test.go index d0def6309..6a14e17b5 100644 --- a/consensus/state_test.go +++ b/consensus/state_test.go @@ -10,8 +10,8 @@ import ( cstypes "github.com/tendermint/tendermint/consensus/types" tmpubsub "github.com/tendermint/tendermint/libs/pubsub" "github.com/tendermint/tendermint/types" - cmn "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tmlibs/log" + cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/libs/log" ) func init() { diff --git a/consensus/test_data/many_blocks.cswal b/consensus/test_data/many_blocks.cswal deleted file mode 100644 index d443fff78..000000000 Binary files a/consensus/test_data/many_blocks.cswal and /dev/null differ diff --git a/consensus/ticker.go b/consensus/ticker.go index b37b7c495..a1e2174c3 100644 --- a/consensus/ticker.go +++ b/consensus/ticker.go @@ -3,8 +3,8 @@ package consensus import ( "time" - cmn "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tmlibs/log" + cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/libs/log" ) var ( diff --git a/consensus/types/height_vote_set.go b/consensus/types/height_vote_set.go index 3c9867940..70a38668f 100644 --- a/consensus/types/height_vote_set.go +++ b/consensus/types/height_vote_set.go @@ -8,7 +8,7 @@ import ( "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/types" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) type RoundVoteSet struct { diff --git a/consensus/types/height_vote_set_test.go b/consensus/types/height_vote_set_test.go index 678d34759..0de656000 100644 --- a/consensus/types/height_vote_set_test.go +++ b/consensus/types/height_vote_set_test.go @@ -6,7 +6,7 @@ import ( cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/types" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) var config *cfg.Config // NOTE: must be reset for each _test.go file diff --git a/consensus/types/peer_round_state.go b/consensus/types/peer_round_state.go index dcb6c8e02..7a5d69b8e 100644 --- a/consensus/types/peer_round_state.go +++ b/consensus/types/peer_round_state.go @@ -5,7 +5,7 @@ import ( "time" "github.com/tendermint/tendermint/types" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) //----------------------------------------------------------------------------- diff --git a/consensus/types/round_state.go b/consensus/types/round_state.go index 14da1f149..cca560ccf 100644 --- a/consensus/types/round_state.go +++ b/consensus/types/round_state.go @@ -6,7 +6,7 @@ import ( "time" "github.com/tendermint/tendermint/types" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) //----------------------------------------------------------------------------- diff --git a/consensus/types/round_state_test.go b/consensus/types/round_state_test.go index dc88c3a13..080178f24 100644 --- a/consensus/types/round_state_test.go +++ b/consensus/types/round_state_test.go @@ -5,9 +5,9 @@ import ( "time" "github.com/tendermint/go-amino" - "github.com/tendermint/go-crypto" + "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/types" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) func BenchmarkRoundStateDeepCopy(b *testing.B) { diff --git a/consensus/types/wire.go b/consensus/types/wire.go index bd5c4497d..6342d7eba 100644 --- a/consensus/types/wire.go +++ b/consensus/types/wire.go @@ -2,7 +2,7 @@ package types import ( "github.com/tendermint/go-amino" - "github.com/tendermint/go-crypto" + "github.com/tendermint/tendermint/crypto" ) var cdc = amino.NewCodec() diff --git a/consensus/version.go b/consensus/version.go index 2c137bf7f..5c74a16db 100644 --- a/consensus/version.go +++ b/consensus/version.go @@ -1,7 +1,7 @@ package consensus import ( - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) // kind of arbitrary diff --git a/consensus/wal.go b/consensus/wal.go index 3d9bf8afc..8c4c10bc7 100644 --- a/consensus/wal.go +++ b/consensus/wal.go @@ -12,8 +12,8 @@ import ( amino "github.com/tendermint/go-amino" "github.com/tendermint/tendermint/types" - auto "github.com/tendermint/tmlibs/autofile" - cmn "github.com/tendermint/tmlibs/common" + auto "github.com/tendermint/tendermint/libs/autofile" + cmn "github.com/tendermint/tendermint/libs/common" ) const ( diff --git a/consensus/wal_generator.go b/consensus/wal_generator.go index f61af15f5..f3a365809 100644 --- a/consensus/wal_generator.go +++ b/consensus/wal_generator.go @@ -10,24 +10,24 @@ import ( "time" "github.com/pkg/errors" - "github.com/tendermint/abci/example/kvstore" + "github.com/tendermint/tendermint/abci/example/kvstore" bc "github.com/tendermint/tendermint/blockchain" cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/privval" "github.com/tendermint/tendermint/proxy" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" - auto "github.com/tendermint/tmlibs/autofile" - cmn "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tmlibs/db" - "github.com/tendermint/tmlibs/log" + auto "github.com/tendermint/tendermint/libs/autofile" + cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/libs/db" + "github.com/tendermint/tendermint/libs/log" ) // WALWithNBlocks generates a consensus WAL. It does this by spining up a // stripped down version of node (proxy app, event bus, consensus state) with a // persistent kvstore application and special consensus wal instance // (byteBufferWAL) and waits until numBlocks are created. Then it returns a WAL -// content. +// content. If the node fails to produce given numBlocks, it returns an error. func WALWithNBlocks(numBlocks int) (data []byte, err error) { config := getConfig() @@ -89,15 +89,15 @@ func WALWithNBlocks(numBlocks int) (data []byte, err error) { if err := consensusState.Start(); err != nil { return nil, errors.Wrap(err, "failed to start consensus state") } - defer consensusState.Stop() select { case <-numBlocksWritten: + consensusState.Stop() wr.Flush() return b.Bytes(), nil case <-time.After(1 * time.Minute): - wr.Flush() - return b.Bytes(), fmt.Errorf("waited too long for tendermint to produce %d blocks (grep logs for `wal_generator`)", numBlocks) + consensusState.Stop() + return []byte{}, fmt.Errorf("waited too long for tendermint to produce %d blocks (grep logs for `wal_generator`)", numBlocks) } } diff --git a/consensus/wal_test.go b/consensus/wal_test.go index eebbc85a2..3ecb4fe8f 100644 --- a/consensus/wal_test.go +++ b/consensus/wal_test.go @@ -9,7 +9,7 @@ import ( "github.com/tendermint/tendermint/consensus/types" tmtypes "github.com/tendermint/tendermint/types" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/consensus/wire.go b/consensus/wire.go index 81223c689..5f231c0c7 100644 --- a/consensus/wire.go +++ b/consensus/wire.go @@ -2,7 +2,7 @@ package consensus import ( "github.com/tendermint/go-amino" - "github.com/tendermint/go-crypto" + "github.com/tendermint/tendermint/crypto" ) var cdc = amino.NewCodec() diff --git a/crypto/CHANGELOG.md b/crypto/CHANGELOG.md new file mode 100644 index 000000000..dd7c1039f --- /dev/null +++ b/crypto/CHANGELOG.md @@ -0,0 +1,154 @@ +# Changelog + +## 0.9.0 + +BREAKING CHANGES + +- `priv.PubKey()` no longer returns an error. Any applicable errors (such as when fetching the public key from a hardware wallet) should be checked and returned when constructing the private key. + +## 0.8.0 + +**TBD** + +## 0.7.0 + +**May 30th, 2018** + +BREAKING CHANGES + +No breaking changes compared to 0.6.2, but making up for the version bump that +should have happened in 0.6.1. + +We also bring in the `tmlibs/merkle` package with breaking changes: + +- change the hash function from RIPEMD160 to tmhash (first 20-bytes of SHA256) +- remove unused funcs and unexport SimpleMap + +FEATURES + +- [xchacha20poly1305] New authenticated encryption module +- [merkle] Moved in from tmlibs +- [merkle/tmhash] New hash function: the first 20-bytes of SHA256 + +IMPROVEMENTS + +- Remove some dead code +- Use constant-time compare for signatures + +BUG FIXES + +- Fix MixEntropy weakness +- Fix PrivKeyEd25519.Generate() + +## 0.6.2 (April 9, 2018) + +IMPROVEMENTS + +- Update for latest go-amino + +## 0.6.1 (March 26, 2018) + +BREAKING CHANGES + +- Encoding uses MarshalBinaryBare rather than MarshalBinary (which auto-length-prefixes) for pub/priv/sig. + +## 0.6.0 (March 2, 2018) + +BREAKING CHANGES + +- Update Amino names from "com.tendermint/..." to "tendermint/" + +## 0.5.0 (March 2, 2018) + +BREAKING CHANGES + +- nano: moved to `_nano` now while we're having build issues +- bcrypt: moved to `keys/bcrypt` +- hd: moved to `keys/hd`; `BTC` added to some function names; other function cleanup +- keys/cryptostore: moved to `keys`, renamed to `keybase`, and completely refactored +- keys: moved BIP39 related code to `keys/words` + +FEATURE + +- `Address` is a type alias for `cmn.HexBytes` + +BUG FIX + +- PrivKey comparisons done in constant time + +## 0.4.1 (October 27, 2017) + +This release removes support for bcrypt as it was merged too soon without an upgrade plan +for existing keys. + +REVERTS THE FOLLOWING COMMITS: + +- Parameterize and lower bcrypt cost - dfc4cdd2d71513e4a9922d679c74f36357c4c862 +- Upgrade keys to use bcrypt with salts (#38) - 8e7f0e7701f92206679ad093d013b9b162427631 + +## 0.4.0 (October 27, 2017) + +BREAKING CHANGES: + +- `keys`: use bcrypt plus salt + +FEATURES: + +- add support for signing via Ledger Nano + +IMPROVEMENTS: + +- linting and comments + +## 0.3.0 (September 22, 2017) + +BREAKING CHANGES: + +- Remove `cmd` and `keys/tx` packages altogether: move it to the cosmos-sdk +- `cryptostore.Generator` takes a secret +- Remove `String()` from `Signature` interface + +FEATURES: + +- `keys`: add CRC16 error correcting code + +IMPROVEMENTS: + +- Allow no passwords on keys for development convenience + + +## 0.2.1 (June 21, 2017) + +- Improve keys command + - No password prompts in non-interactive mode (echo 'foobar' | keys new foo) + - Added support for seed phrases + - Seed phrase now returned on `keys new` + - Add `keys restore` to restore private key from key phrase + - Checksum to verify typos in the seed phrase (rather than just a useless key) + - Add `keys delete` to remove a key if needed + +## 0.2.0 (May 18, 2017) + +BREAKING CHANGES: + +- [hd] The following functions no longer take a `coin string` as argument: `ComputeAddress`, `AddrFromPubKeyBytes`, `ComputeAddressForPrivKey`, `ComputeWIF`, `WIFFromPrivKeyBytes` +- Changes to `PrivKey`, `PubKey`, and `Signature` (denoted `Xxx` below): + - interfaces are renamed `XxxInner`, and are not for use outside the package, though they must be exposed for sake of serialization. + - `Xxx` is now a struct that wraps the corresponding `XxxInner` interface + +FEATURES: + +- `github.com/tendermint/go-keys -> github.com/tendermint/go-crypto/keys` - command and lib for generating and managing encrypted keys +- [hd] New function `WIFFromPrivKeyBytes(privKeyBytes []byte, compress bool) string` +- Changes to `PrivKey`, `PubKey`, and `Signature` (denoted `Xxx` below): + - Expose a new method `Unwrap() XxxInner` on the `Xxx` struct which returns the corresponding `XxxInner` interface + - Expose a new method `Wrap() Xxx` on the `XxxInner` interface which returns the corresponding `Xxx` struct + +IMPROVEMENTS: + +- Update to use new `tmlibs` repository + +## 0.1.0 (April 14, 2017) + +Initial release + diff --git a/crypto/Gopkg.lock b/crypto/Gopkg.lock new file mode 100644 index 000000000..7dbe6ae72 --- /dev/null +++ b/crypto/Gopkg.lock @@ -0,0 +1,137 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + branch = "master" + name = "github.com/btcsuite/btcd" + packages = ["btcec"] + revision = "86fed781132ac890ee03e906e4ecd5d6fa180c64" + +[[projects]] + branch = "master" + name = "github.com/btcsuite/btcutil" + packages = ["base58"] + revision = "d4cc87b860166d00d6b5b9e0d3b3d71d6088d4d4" + +[[projects]] + name = "github.com/davecgh/go-spew" + packages = ["spew"] + revision = "346938d642f2ec3594ed81d874461961cd0faa76" + version = "v1.1.0" + +[[projects]] + name = "github.com/go-kit/kit" + packages = [ + "log", + "log/level", + "log/term" + ] + revision = "4dc7be5d2d12881735283bcab7352178e190fc71" + version = "v0.6.0" + +[[projects]] + name = "github.com/go-logfmt/logfmt" + packages = ["."] + revision = "390ab7935ee28ec6b286364bba9b4dd6410cb3d5" + version = "v0.3.0" + +[[projects]] + name = "github.com/go-stack/stack" + packages = ["."] + revision = "259ab82a6cad3992b4e21ff5cac294ccb06474bc" + version = "v1.7.0" + +[[projects]] + name = "github.com/gogo/protobuf" + packages = [ + "gogoproto", + "proto", + "protoc-gen-gogo/descriptor" + ] + revision = "1adfc126b41513cc696b209667c8656ea7aac67c" + version = "v1.0.0" + +[[projects]] + branch = "master" + name = "github.com/kr/logfmt" + packages = ["."] + revision = "b84e30acd515aadc4b783ad4ff83aff3299bdfe0" + +[[projects]] + name = "github.com/pkg/errors" + packages = ["."] + revision = "645ef00459ed84a119197bfb8d8205042c6df63d" + version = "v0.8.0" + +[[projects]] + name = "github.com/pmezard/go-difflib" + packages = ["difflib"] + revision = "792786c7400a136282c1664665ae0a8db921c6c2" + version = "v1.0.0" + +[[projects]] + name = "github.com/stretchr/testify" + packages = [ + "assert", + "require" + ] + revision = "f35b8ab0b5a2cef36673838d662e249dd9c94686" + version = "v1.2.2" + +[[projects]] + branch = "master" + name = "github.com/tendermint/ed25519" + packages = [ + ".", + "edwards25519", + "extra25519" + ] + revision = "d8387025d2b9d158cf4efb07e7ebf814bcce2057" + +[[projects]] + name = "github.com/tendermint/go-amino" + packages = ["."] + revision = "2106ca61d91029c931fd54968c2bb02dc96b1412" + version = "0.10.1" + +[[projects]] + name = "github.com/tendermint/tmlibs" + packages = [ + "common", + "log", + "test" + ] + revision = "692f1d86a6e2c0efa698fd1e4541b68c74ffaf38" + version = "v0.8.4" + +[[projects]] + branch = "master" + name = "golang.org/x/crypto" + packages = [ + "bcrypt", + "blowfish", + "chacha20poly1305", + "hkdf", + "internal/chacha20", + "internal/subtle", + "nacl/secretbox", + "openpgp/armor", + "openpgp/errors", + "poly1305", + "ripemd160", + "salsa20/salsa" + ] + revision = "7f39a6fea4fe9364fb61e1def6a268a51b4f3a06" + +[[projects]] + branch = "master" + name = "golang.org/x/sys" + packages = ["cpu"] + revision = "ad87a3a340fa7f3bed189293fbfa7a9b7e021ae1" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + inputs-digest = "027b22b86396a971d5d5c1d298947f531f39743975d65a22e98601140aa1b1a1" + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/crypto/Gopkg.toml b/crypto/Gopkg.toml new file mode 100644 index 000000000..a614df866 --- /dev/null +++ b/crypto/Gopkg.toml @@ -0,0 +1,49 @@ +# Gopkg.toml example +# +# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md +# for detailed Gopkg.toml documentation. +# +# required = ["github.com/user/thing/cmd/thing"] +# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] +# +# [[constraint]] +# name = "github.com/user/project" +# version = "1.0.0" +# +# [[constraint]] +# name = "github.com/user/project2" +# branch = "dev" +# source = "github.com/myfork/project2" +# +# [[override]] +# name = "github.com/x/y" +# version = "2.4.0" +# +# [prune] +# non-go = false +# go-tests = true +# unused-packages = true + +[[constraint]] + name = "github.com/btcsuite/btcutil" + branch = "master" + +[[constraint]] + name = "github.com/stretchr/testify" + version = "1.2.1" + +[[constraint]] + name = "github.com/tendermint/ed25519" + branch = "master" + +[[constraint]] + name = "github.com/tendermint/go-amino" + version = "0.10.0-rc2" + +[[constraint]] + name = "github.com/tendermint/tmlibs" + version = "0.8.1" + +[prune] + go-tests = true + unused-packages = true diff --git a/crypto/LICENSE b/crypto/LICENSE new file mode 100644 index 000000000..3beb77b13 --- /dev/null +++ b/crypto/LICENSE @@ -0,0 +1,193 @@ +Tendermint Go-Crypto +Copyright (C) 2015 Tendermint + + + + Apache License + Version 2.0, January 2004 + https://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/crypto/Makefile b/crypto/Makefile new file mode 100644 index 000000000..a4fd3c37f --- /dev/null +++ b/crypto/Makefile @@ -0,0 +1,99 @@ +GOTOOLS = \ + github.com/golang/dep/cmd/dep \ + # gopkg.in/alecthomas/gometalinter.v2 \ + +GOTOOLS_CHECK = dep #gometalinter.v2 + +all: check get_vendor_deps build test install + +check: check_tools + + +######################################## +### Build + +# Command to generate the workd list (kept here for documentation purposes only): +wordlist: + # To re-generate wordlist.go run: + # go-bindata -ignore ".*\.go" -o keys/words/bip39/wordlist.go -pkg "wordlist" keys/bip39/wordlist/... + +build: wordlist + # Nothing else to build! + +install: + # Nothing to install! + + +######################################## +### Tools & dependencies + +check_tools: + @# https://stackoverflow.com/a/25668869 + @echo "Found tools: $(foreach tool,$(GOTOOLS_CHECK),\ + $(if $(shell which $(tool)),$(tool),$(error "No $(tool) in PATH")))" + +get_tools: + @echo "--> Installing tools" + go get -u -v $(GOTOOLS) + #@gometalinter.v2 --install + +update_tools: + @echo "--> Updating tools" + @go get -u $(GOTOOLS) + +get_vendor_deps: + @rm -rf vendor/ + @echo "--> Running dep ensure" + @dep ensure + + +######################################## +### Testing + +test: + CGO_ENABLED=0 go test -p 1 $(shell go list ./... | grep -v vendor) + +######################################## +### Formatting, linting, and vetting + +fmt: + @go fmt ./... + +metalinter: + @echo "==> Running linter" + gometalinter.v2 --vendor --deadline=600s --disable-all \ + --enable=maligned \ + --enable=deadcode \ + --enable=goconst \ + --enable=goimports \ + --enable=gosimple \ + --enable=ineffassign \ + --enable=megacheck \ + --enable=misspell \ + --enable=staticcheck \ + --enable=safesql \ + --enable=structcheck \ + --enable=unconvert \ + --enable=unused \ + --enable=varcheck \ + --enable=vetshadow \ + ./... + #--enable=gas \ + #--enable=dupl \ + #--enable=errcheck \ + #--enable=gocyclo \ + #--enable=golint \ <== comments on anything exported + #--enable=gotype \ + #--enable=interfacer \ + #--enable=unparam \ + #--enable=vet \ + +metalinter_all: + protoc $(INCLUDE) --lint_out=. types/*.proto + gometalinter.v2 --vendor --deadline=600s --enable-all --disable=lll ./... + + +# To avoid unintended conflicts with file names, always add to .PHONY +# unless there is a reason not to. +# https://www.gnu.org/software/make/manual/html_node/Phony-Targets.html +.PHONEY: check build install check_tools get_tools update_tools get_vendor_deps test fmt metalinter metalinter_all diff --git a/crypto/README.md b/crypto/README.md new file mode 100644 index 000000000..32afde699 --- /dev/null +++ b/crypto/README.md @@ -0,0 +1,25 @@ +# crypto + +crypto is the cryptographic package adapted for Tendermint's uses + +## Importing it +`import "github.com/tendermint/tendermint/crypto"` + +## Binary encoding + +For Binary encoding, please refer to the [Tendermint encoding spec](https://github.com/tendermint/tendermint/blob/master/docs/spec/blockchain/encoding.md). + +## JSON Encoding + +crypto `.Bytes()` uses Amino:binary encoding, but Amino:JSON is also supported. + +```go +Example Amino:JSON encodings: + +crypto.PrivKeyEd25519 - {"type":"954568A3288910","value":"EVkqJO/jIXp3rkASXfh9YnyToYXRXhBr6g9cQVxPFnQBP/5povV4HTjvsy530kybxKHwEi85iU8YL0qQhSYVoQ=="} +crypto.SignatureEd25519 - {"type":"6BF5903DA1DB28","value":"77sQNZOrf7ltExpf7AV1WaYPCHbyRLgjBsoWVzcduuLk+jIGmYk+s5R6Emm29p12HeiNAuhUJgdFGmwkpeGJCA=="} +crypto.PubKeyEd25519 - {"type":"AC26791624DE60","value":"AT/+aaL1eB0477Mud9JMm8Sh8BIvOYlPGC9KkIUmFaE="} +crypto.PrivKeySecp256k1 - {"type":"019E82E1B0F798","value":"zx4Pnh67N+g2V+5vZbQzEyRerX9c4ccNZOVzM9RvJ0Y="} +crypto.SignatureSecp256k1 - {"type":"6D1EA416E1FEE8","value":"MEUCIQCIg5TqS1l7I+MKTrSPIuUN2+4m5tA29dcauqn3NhEJ2wIgICaZ+lgRc5aOTVahU/XoLopXKn8BZcl0bnuYWLvohR8="} +crypto.PubKeySecp256k1 - {"type":"F8CCEAEB5AE980","value":"A8lPKJXcNl5VHt1FK8a244K9EJuS4WX1hFBnwisi0IJx"} +``` diff --git a/crypto/amino.go b/crypto/amino.go new file mode 100644 index 000000000..6a8703fc9 --- /dev/null +++ b/crypto/amino.go @@ -0,0 +1,37 @@ +package crypto + +import ( + amino "github.com/tendermint/go-amino" +) + +var cdc = amino.NewCodec() + +func init() { + // NOTE: It's important that there be no conflicts here, + // as that would change the canonical representations, + // and therefore change the address. + // TODO: Add feature to go-amino to ensure that there + // are no conflicts. + RegisterAmino(cdc) +} + +// RegisterAmino registers all crypto related types in the given (amino) codec. +func RegisterAmino(cdc *amino.Codec) { + cdc.RegisterInterface((*PubKey)(nil), nil) + cdc.RegisterConcrete(PubKeyEd25519{}, + "tendermint/PubKeyEd25519", nil) + cdc.RegisterConcrete(PubKeySecp256k1{}, + "tendermint/PubKeySecp256k1", nil) + + cdc.RegisterInterface((*PrivKey)(nil), nil) + cdc.RegisterConcrete(PrivKeyEd25519{}, + "tendermint/PrivKeyEd25519", nil) + cdc.RegisterConcrete(PrivKeySecp256k1{}, + "tendermint/PrivKeySecp256k1", nil) + + cdc.RegisterInterface((*Signature)(nil), nil) + cdc.RegisterConcrete(SignatureEd25519{}, + "tendermint/SignatureEd25519", nil) + cdc.RegisterConcrete(SignatureSecp256k1{}, + "tendermint/SignatureSecp256k1", nil) +} diff --git a/crypto/armor.go b/crypto/armor.go new file mode 100644 index 000000000..4146048ad --- /dev/null +++ b/crypto/armor.go @@ -0,0 +1,39 @@ +package crypto + +import ( + "bytes" + "fmt" + "io/ioutil" + + "golang.org/x/crypto/openpgp/armor" +) + +func EncodeArmor(blockType string, headers map[string]string, data []byte) string { + buf := new(bytes.Buffer) + w, err := armor.Encode(buf, blockType, headers) + if err != nil { + panic(fmt.Errorf("could not encode ascii armor: %s", err)) + } + _, err = w.Write(data) + if err != nil { + panic(fmt.Errorf("could not encode ascii armor: %s", err)) + } + err = w.Close() + if err != nil { + panic(fmt.Errorf("could not encode ascii armor: %s", err)) + } + return buf.String() +} + +func DecodeArmor(armorStr string) (blockType string, headers map[string]string, data []byte, err error) { + buf := bytes.NewBufferString(armorStr) + block, err := armor.Decode(buf) + if err != nil { + return "", nil, nil, err + } + data, err = ioutil.ReadAll(block.Body) + if err != nil { + return "", nil, nil, err + } + return block.Type, block.Header, data, nil +} diff --git a/crypto/armor_test.go b/crypto/armor_test.go new file mode 100644 index 000000000..5eae87c00 --- /dev/null +++ b/crypto/armor_test.go @@ -0,0 +1,20 @@ +package crypto + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSimpleArmor(t *testing.T) { + blockType := "MINT TEST" + data := []byte("somedata") + armorStr := EncodeArmor(blockType, nil, data) + + // Decode armorStr and test for equivalence. + blockType2, _, data2, err := DecodeArmor(armorStr) + require.Nil(t, err, "%+v", err) + assert.Equal(t, blockType, blockType2) + assert.Equal(t, data, data2) +} diff --git a/crypto/doc.go b/crypto/doc.go new file mode 100644 index 000000000..544e0df36 --- /dev/null +++ b/crypto/doc.go @@ -0,0 +1,45 @@ +// crypto is a customized/convenience cryptography package for supporting +// Tendermint. + +// It wraps select functionality of equivalent functions in the +// Go standard library, for easy usage with our libraries. + +// Keys: + +// All key generation functions return an instance of the PrivKey interface +// which implements methods + +// AssertIsPrivKeyInner() +// Bytes() []byte +// Sign(msg []byte) Signature +// PubKey() PubKey +// Equals(PrivKey) bool +// Wrap() PrivKey + +// From the above method we can: +// a) Retrieve the public key if needed + +// pubKey := key.PubKey() + +// For example: +// privKey, err := crypto.GenPrivKeyEd25519() +// if err != nil { +// ... +// } +// pubKey := privKey.PubKey() +// ... +// // And then you can use the private and public key +// doSomething(privKey, pubKey) + +// We also provide hashing wrappers around algorithms: + +// Sha256 +// sum := crypto.Sha256([]byte("This is Tendermint")) +// fmt.Printf("%x\n", sum) + +// Ripemd160 +// sum := crypto.Ripemd160([]byte("This is consensus")) +// fmt.Printf("%x\n", sum) +package crypto + +// TODO: Add more docs in here diff --git a/crypto/encode_test.go b/crypto/encode_test.go new file mode 100644 index 000000000..16555bf71 --- /dev/null +++ b/crypto/encode_test.go @@ -0,0 +1,119 @@ +package crypto + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type byter interface { + Bytes() []byte +} + +func checkAminoBinary(t *testing.T, src byter, dst interface{}, size int) { + // Marshal to binary bytes. + bz, err := cdc.MarshalBinaryBare(src) + require.Nil(t, err, "%+v", err) + // Make sure this is compatible with current (Bytes()) encoding. + assert.Equal(t, src.Bytes(), bz, "Amino binary vs Bytes() mismatch") + // Make sure we have the expected length. + if size != -1 { + assert.Equal(t, size, len(bz), "Amino binary size mismatch") + } + // Unmarshal. + err = cdc.UnmarshalBinaryBare(bz, dst) + require.Nil(t, err, "%+v", err) +} + +func checkAminoJSON(t *testing.T, src interface{}, dst interface{}, isNil bool) { + // Marshal to JSON bytes. + js, err := cdc.MarshalJSON(src) + require.Nil(t, err, "%+v", err) + if isNil { + assert.Equal(t, string(js), `null`) + } else { + assert.Contains(t, string(js), `"type":`) + assert.Contains(t, string(js), `"value":`) + } + // Unmarshal. + err = cdc.UnmarshalJSON(js, dst) + require.Nil(t, err, "%+v", err) +} + +func ExamplePrintRegisteredTypes() { + cdc.PrintTypes(os.Stdout) + // Output: | Type | Name | Prefix | Length | Notes | + //| ---- | ---- | ------ | ----- | ------ | + //| PubKeyEd25519 | tendermint/PubKeyEd25519 | 0x1624DE64 | 0x20 | | + //| PubKeySecp256k1 | tendermint/PubKeySecp256k1 | 0xEB5AE987 | 0x21 | | + //| PrivKeyEd25519 | tendermint/PrivKeyEd25519 | 0xA3288910 | 0x40 | | + //| PrivKeySecp256k1 | tendermint/PrivKeySecp256k1 | 0xE1B0F79B | 0x20 | | + //| SignatureEd25519 | tendermint/SignatureEd25519 | 0x2031EA53 | 0x40 | | + //| SignatureSecp256k1 | tendermint/SignatureSecp256k1 | 0x7FC4A495 | variable | | +} + +func TestKeyEncodings(t *testing.T) { + cases := []struct { + privKey PrivKey + privSize, pubSize int // binary sizes + }{ + { + privKey: GenPrivKeyEd25519(), + privSize: 69, + pubSize: 37, + }, + { + privKey: GenPrivKeySecp256k1(), + privSize: 37, + pubSize: 38, + }, + } + + for _, tc := range cases { + + // Check (de/en)codings of PrivKeys. + var priv2, priv3 PrivKey + checkAminoBinary(t, tc.privKey, &priv2, tc.privSize) + assert.EqualValues(t, tc.privKey, priv2) + checkAminoJSON(t, tc.privKey, &priv3, false) // TODO also check Prefix bytes. + assert.EqualValues(t, tc.privKey, priv3) + + // Check (de/en)codings of Signatures. + var sig1, sig2, sig3 Signature + sig1, err := tc.privKey.Sign([]byte("something")) + assert.NoError(t, err) + checkAminoBinary(t, sig1, &sig2, -1) // Signature size changes for Secp anyways. + assert.EqualValues(t, sig1, sig2) + checkAminoJSON(t, sig1, &sig3, false) // TODO also check Prefix bytes. + assert.EqualValues(t, sig1, sig3) + + // Check (de/en)codings of PubKeys. + pubKey := tc.privKey.PubKey() + var pub2, pub3 PubKey + checkAminoBinary(t, pubKey, &pub2, tc.pubSize) + assert.EqualValues(t, pubKey, pub2) + checkAminoJSON(t, pubKey, &pub3, false) // TODO also check Prefix bytes. + assert.EqualValues(t, pubKey, pub3) + } +} + +func TestNilEncodings(t *testing.T) { + + // Check nil Signature. + var a, b Signature + checkAminoJSON(t, &a, &b, true) + assert.EqualValues(t, a, b) + + // Check nil PubKey. + var c, d PubKey + checkAminoJSON(t, &c, &d, true) + assert.EqualValues(t, c, d) + + // Check nil PrivKey. + var e, f PrivKey + checkAminoJSON(t, &e, &f, true) + assert.EqualValues(t, e, f) + +} diff --git a/crypto/example_test.go b/crypto/example_test.go new file mode 100644 index 000000000..904e1c610 --- /dev/null +++ b/crypto/example_test.go @@ -0,0 +1,35 @@ +// Copyright 2017 Tendermint. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crypto_test + +import ( + "fmt" + + "github.com/tendermint/tendermint/crypto" +) + +func ExampleSha256() { + sum := crypto.Sha256([]byte("This is Tendermint")) + fmt.Printf("%x\n", sum) + // Output: + // f91afb642f3d1c87c17eb01aae5cb65c242dfdbe7cf1066cc260f4ce5d33b94e +} + +func ExampleRipemd160() { + sum := crypto.Ripemd160([]byte("This is Tendermint")) + fmt.Printf("%x\n", sum) + // Output: + // 051e22663e8f0fd2f2302f1210f954adff009005 +} diff --git a/crypto/hash.go b/crypto/hash.go new file mode 100644 index 000000000..165b1e153 --- /dev/null +++ b/crypto/hash.go @@ -0,0 +1,18 @@ +package crypto + +import ( + "crypto/sha256" + "golang.org/x/crypto/ripemd160" +) + +func Sha256(bytes []byte) []byte { + hasher := sha256.New() + hasher.Write(bytes) + return hasher.Sum(nil) +} + +func Ripemd160(bytes []byte) []byte { + hasher := ripemd160.New() + hasher.Write(bytes) + return hasher.Sum(nil) +} diff --git a/crypto/hkdfchacha20poly1305/hkdfchachapoly.go b/crypto/hkdfchacha20poly1305/hkdfchachapoly.go new file mode 100644 index 000000000..ab3b9df3a --- /dev/null +++ b/crypto/hkdfchacha20poly1305/hkdfchachapoly.go @@ -0,0 +1,105 @@ +// Package hkdfchacha20poly1305 creates an AEAD using hkdf, chacha20, and poly1305 +// When sealing and opening, the hkdf is used to obtain the nonce and subkey for +// chacha20. Other than the change for the how the subkey and nonce for chacha +// are obtained, this is the same as chacha20poly1305 +package hkdfchacha20poly1305 + +import ( + "crypto/cipher" + "crypto/sha256" + "errors" + "io" + + "golang.org/x/crypto/chacha20poly1305" + "golang.org/x/crypto/hkdf" +) + +type hkdfchacha20poly1305 struct { + key [KeySize]byte +} + +const ( + // KeySize is the size of the key used by this AEAD, in bytes. + KeySize = 32 + // NonceSize is the size of the nonce used with this AEAD, in bytes. + NonceSize = 24 + // TagSize is the size added from poly1305 + TagSize = 16 + // MaxPlaintextSize is the max size that can be passed into a single call of Seal + MaxPlaintextSize = (1 << 38) - 64 + // MaxCiphertextSize is the max size that can be passed into a single call of Open, + // this differs from plaintext size due to the tag + MaxCiphertextSize = (1 << 38) - 48 + // HkdfInfo is the parameter used internally for Hkdf's info parameter. + HkdfInfo = "TENDERMINT_SECRET_CONNECTION_FRAME_KEY_DERIVE" +) + +//New xChaChapoly1305 AEAD with 24 byte nonces +func New(key []byte) (cipher.AEAD, error) { + if len(key) != KeySize { + return nil, errors.New("chacha20poly1305: bad key length") + } + ret := new(hkdfchacha20poly1305) + copy(ret.key[:], key) + return ret, nil + +} +func (c *hkdfchacha20poly1305) NonceSize() int { + return NonceSize +} + +func (c *hkdfchacha20poly1305) Overhead() int { + return TagSize +} + +func (c *hkdfchacha20poly1305) Seal(dst, nonce, plaintext, additionalData []byte) []byte { + if len(nonce) != NonceSize { + panic("hkdfchacha20poly1305: bad nonce length passed to Seal") + } + + if uint64(len(plaintext)) > MaxPlaintextSize { + panic("hkdfchacha20poly1305: plaintext too large") + } + + subKey, chachaNonce := getSubkeyAndChachaNonceFromHkdf(&c.key, &nonce) + + aead, err := chacha20poly1305.New(subKey[:]) + if err != nil { + panic("hkdfchacha20poly1305: failed to initialize chacha20poly1305") + } + + return aead.Seal(dst, chachaNonce[:], plaintext, additionalData) +} + +func (c *hkdfchacha20poly1305) Open(dst, nonce, ciphertext, additionalData []byte) ([]byte, error) { + if len(nonce) != NonceSize { + return nil, errors.New("hkdfchacha20poly1305: bad nonce length passed to Open") + } + if uint64(len(ciphertext)) > MaxCiphertextSize { + return nil, errors.New("hkdfchacha20poly1305: ciphertext too large") + } + + subKey, chachaNonce := getSubkeyAndChachaNonceFromHkdf(&c.key, &nonce) + + aead, err := chacha20poly1305.New(subKey[:]) + if err != nil { + panic("hkdfchacha20poly1305: failed to initialize chacha20poly1305") + } + + return aead.Open(dst, chachaNonce[:], ciphertext, additionalData) +} + +func getSubkeyAndChachaNonceFromHkdf(cKey *[32]byte, nonce *[]byte) ( + subKey [KeySize]byte, chachaNonce [chacha20poly1305.NonceSize]byte) { + hash := sha256.New + hkdf := hkdf.New(hash, (*cKey)[:], *nonce, []byte(HkdfInfo)) + _, err := io.ReadFull(hkdf, subKey[:]) + if err != nil { + panic("hkdfchacha20poly1305: failed to read subkey from hkdf") + } + _, err = io.ReadFull(hkdf, chachaNonce[:]) + if err != nil { + panic("hkdfchacha20poly1305: failed to read chachaNonce from hkdf") + } + return +} diff --git a/crypto/hkdfchacha20poly1305/hkdfchachapoly_test.go b/crypto/hkdfchacha20poly1305/hkdfchachapoly_test.go new file mode 100644 index 000000000..854a312e6 --- /dev/null +++ b/crypto/hkdfchacha20poly1305/hkdfchachapoly_test.go @@ -0,0 +1,143 @@ +package hkdfchacha20poly1305 + +import ( + "bytes" + cr "crypto/rand" + "encoding/hex" + mr "math/rand" + "testing" + + "github.com/stretchr/testify/assert" +) + +// Test that a test vector we generated is valid. (Ensures backwards +// compatibility) +func TestVector(t *testing.T) { + key, _ := hex.DecodeString("56f8de45d3c294c7675bcaf457bdd4b71c380b9b2408ce9412b348d0f08b69ee") + aead, err := New(key[:]) + if err != nil { + t.Fatal(err) + } + cts := []string{"e20a8bf42c535ac30125cfc52031577f0b", + "657695b37ba30f67b25860d90a6f1d00d8", + "e9aa6f3b7f625d957fd50f05bcdf20d014", + "8a00b3b5a6014e0d2033bebc5935086245", + "aadd74867b923879e6866ea9e03c009039", + "fc59773c2c864ee3b4cc971876b3c7bed4", + "caec14e3a9a52ce1a2682c6737defa4752", + "0b89511ffe490d2049d6950494ee51f919", + "7de854ea71f43ca35167a07566c769083d", + "cd477327f4ea4765c71e311c5fec1edbfb"} + + for i := 0; i < 10; i++ { + ct, _ := hex.DecodeString(cts[i]) + + byteArr := []byte{byte(i)} + nonce := make([]byte, 24) + nonce[0] = byteArr[0] + + // Test that we get the expected plaintext on open + plaintext, err := aead.Open(nil, nonce, ct, byteArr) + if err != nil { + t.Errorf("%dth Open failed", i) + continue + } + assert.Equal(t, byteArr, plaintext) + // Test that sealing yields the expected ciphertext + ciphertext := aead.Seal(nil, nonce, plaintext, byteArr) + assert.Equal(t, ct, ciphertext) + } +} + +// The following test is taken from +// https://github.com/golang/crypto/blob/master/chacha20poly1305/chacha20poly1305_test.go#L69 +// It requires the below copyright notice, where "this source code" refers to the following function. +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found at the bottom of this file. +func TestRandom(t *testing.T) { + // Some random tests to verify Open(Seal) == Plaintext + for i := 0; i < 256; i++ { + var nonce [24]byte + var key [32]byte + + al := mr.Intn(128) + pl := mr.Intn(16384) + ad := make([]byte, al) + plaintext := make([]byte, pl) + cr.Read(key[:]) + cr.Read(nonce[:]) + cr.Read(ad) + cr.Read(plaintext) + + aead, err := New(key[:]) + if err != nil { + t.Fatal(err) + } + + ct := aead.Seal(nil, nonce[:], plaintext, ad) + + plaintext2, err := aead.Open(nil, nonce[:], ct, ad) + if err != nil { + t.Errorf("Random #%d: Open failed", i) + continue + } + + if !bytes.Equal(plaintext, plaintext2) { + t.Errorf("Random #%d: plaintext's don't match: got %x vs %x", i, plaintext2, plaintext) + continue + } + + if len(ad) > 0 { + alterAdIdx := mr.Intn(len(ad)) + ad[alterAdIdx] ^= 0x80 + if _, err := aead.Open(nil, nonce[:], ct, ad); err == nil { + t.Errorf("Random #%d: Open was successful after altering additional data", i) + } + ad[alterAdIdx] ^= 0x80 + } + + alterNonceIdx := mr.Intn(aead.NonceSize()) + nonce[alterNonceIdx] ^= 0x80 + if _, err := aead.Open(nil, nonce[:], ct, ad); err == nil { + t.Errorf("Random #%d: Open was successful after altering nonce", i) + } + nonce[alterNonceIdx] ^= 0x80 + + alterCtIdx := mr.Intn(len(ct)) + ct[alterCtIdx] ^= 0x80 + if _, err := aead.Open(nil, nonce[:], ct, ad); err == nil { + t.Errorf("Random #%d: Open was successful after altering ciphertext", i) + } + ct[alterCtIdx] ^= 0x80 + } +} + +// AFOREMENTIONED LICENCE +// Copyright (c) 2009 The Go Authors. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/crypto/merkle/README.md b/crypto/merkle/README.md new file mode 100644 index 000000000..c44978368 --- /dev/null +++ b/crypto/merkle/README.md @@ -0,0 +1,4 @@ +## Simple Merkle Tree + +For smaller static data structures that don't require immutable snapshots or mutability; +for instance the transactions and validation signatures of a block can be hashed using this simple merkle tree logic. diff --git a/crypto/merkle/doc.go b/crypto/merkle/doc.go new file mode 100644 index 000000000..865c30217 --- /dev/null +++ b/crypto/merkle/doc.go @@ -0,0 +1,31 @@ +/* +Package merkle computes a deterministic minimal height Merkle tree hash. +If the number of items is not a power of two, some leaves +will be at different levels. Tries to keep both sides of +the tree the same size, but the left may be one greater. + +Use this for short deterministic trees, such as the validator list. +For larger datasets, use IAVLTree. + +Be aware that the current implementation by itself does not prevent +second pre-image attacks. Hence, use this library with caution. +Otherwise you might run into similar issues as, e.g., in early Bitcoin: +https://bitcointalk.org/?topic=102395 + + * + / \ + / \ + / \ + / \ + * * + / \ / \ + / \ / \ + / \ / \ + * * * h6 + / \ / \ / \ + h0 h1 h2 h3 h4 h5 + +TODO(ismail): add 2nd pre-image protection or clarify further on how we use this and why this secure. + +*/ +package merkle diff --git a/crypto/merkle/simple_map.go b/crypto/merkle/simple_map.go new file mode 100644 index 000000000..ba4b9309a --- /dev/null +++ b/crypto/merkle/simple_map.go @@ -0,0 +1,88 @@ +package merkle + +import ( + "github.com/tendermint/tendermint/crypto/tmhash" + cmn "github.com/tendermint/tendermint/libs/common" +) + +// Merkle tree from a map. +// Leaves are `hash(key) | hash(value)`. +// Leaves are sorted before Merkle hashing. +type simpleMap struct { + kvs cmn.KVPairs + sorted bool +} + +func newSimpleMap() *simpleMap { + return &simpleMap{ + kvs: nil, + sorted: false, + } +} + +// Set hashes the key and value and appends it to the kv pairs. +func (sm *simpleMap) Set(key string, value Hasher) { + sm.sorted = false + + // The value is hashed, so you can + // check for equality with a cached value (say) + // and make a determination to fetch or not. + vhash := value.Hash() + + sm.kvs = append(sm.kvs, cmn.KVPair{ + Key: []byte(key), + Value: vhash, + }) +} + +// Hash Merkle root hash of items sorted by key +// (UNSTABLE: and by value too if duplicate key). +func (sm *simpleMap) Hash() []byte { + sm.Sort() + return hashKVPairs(sm.kvs) +} + +func (sm *simpleMap) Sort() { + if sm.sorted { + return + } + sm.kvs.Sort() + sm.sorted = true +} + +// Returns a copy of sorted KVPairs. +// NOTE these contain the hashed key and value. +func (sm *simpleMap) KVPairs() cmn.KVPairs { + sm.Sort() + kvs := make(cmn.KVPairs, len(sm.kvs)) + copy(kvs, sm.kvs) + return kvs +} + +//---------------------------------------- + +// A local extension to KVPair that can be hashed. +// Key and value are length prefixed and concatenated, +// then hashed. +type KVPair cmn.KVPair + +func (kv KVPair) Hash() []byte { + hasher := tmhash.New() + err := encodeByteSlice(hasher, kv.Key) + if err != nil { + panic(err) + } + err = encodeByteSlice(hasher, kv.Value) + if err != nil { + panic(err) + } + return hasher.Sum(nil) +} + +func hashKVPairs(kvs cmn.KVPairs) []byte { + kvsH := make([]Hasher, len(kvs)) + for i, kvp := range kvs { + kvsH[i] = KVPair(kvp) + } + return SimpleHashFromHashers(kvsH) +} diff --git a/crypto/merkle/simple_map_test.go b/crypto/merkle/simple_map_test.go new file mode 100644 index 000000000..34febcf16 --- /dev/null +++ b/crypto/merkle/simple_map_test.go @@ -0,0 +1,54 @@ +package merkle + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/tendermint/tendermint/crypto/tmhash" +) + +type strHasher string + +func (str strHasher) Hash() []byte { + return tmhash.Sum([]byte(str)) +} + +func TestSimpleMap(t *testing.T) { + { + db := newSimpleMap() + db.Set("key1", strHasher("value1")) + assert.Equal(t, "fa9bc106ffd932d919bee935ceb6cf2b3dd72d8f", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + } + { + db := newSimpleMap() + db.Set("key1", strHasher("value2")) + assert.Equal(t, "e00e7dcfe54e9fafef5111e813a587f01ba9c3e8", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + } + { + db := newSimpleMap() + db.Set("key1", strHasher("value1")) + db.Set("key2", strHasher("value2")) + assert.Equal(t, "eff12d1c703a1022ab509287c0f196130123d786", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + } + { + db := newSimpleMap() + db.Set("key2", strHasher("value2")) // NOTE: out of order + db.Set("key1", strHasher("value1")) + assert.Equal(t, "eff12d1c703a1022ab509287c0f196130123d786", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + } + { + db := newSimpleMap() + db.Set("key1", strHasher("value1")) + db.Set("key2", strHasher("value2")) + db.Set("key3", strHasher("value3")) + assert.Equal(t, "b2c62a277c08dbd2ad73ca53cd1d6bfdf5830d26", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + } + { + db := newSimpleMap() + db.Set("key2", strHasher("value2")) // NOTE: out of order + db.Set("key1", strHasher("value1")) + db.Set("key3", strHasher("value3")) + assert.Equal(t, "b2c62a277c08dbd2ad73ca53cd1d6bfdf5830d26", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + } +} diff --git a/crypto/merkle/simple_proof.go b/crypto/merkle/simple_proof.go new file mode 100644 index 000000000..2541b6d38 --- /dev/null +++ b/crypto/merkle/simple_proof.go @@ -0,0 +1,160 @@ +package merkle + +import ( + "bytes" + "fmt" +) + +// SimpleProof represents a simple merkle proof. +type SimpleProof struct { + Aunts [][]byte `json:"aunts"` // Hashes from leaf's sibling to a root's child. +} + +// SimpleProofsFromHashers computes inclusion proof for given items. +// proofs[0] is the proof for items[0]. +func SimpleProofsFromHashers(items []Hasher) (rootHash []byte, proofs []*SimpleProof) { + trails, rootSPN := trailsFromHashers(items) + rootHash = rootSPN.Hash + proofs = make([]*SimpleProof, len(items)) + for i, trail := range trails { + proofs[i] = &SimpleProof{ + Aunts: trail.FlattenAunts(), + } + } + return +} + +// SimpleProofsFromMap generates proofs from a map. The keys/values of the map will be used as the keys/values +// in the underlying key-value pairs. +// The keys are sorted before the proofs are computed. +func SimpleProofsFromMap(m map[string]Hasher) (rootHash []byte, proofs map[string]*SimpleProof, keys []string) { + sm := newSimpleMap() + for k, v := range m { + sm.Set(k, v) + } + sm.Sort() + kvs := sm.kvs + kvsH := make([]Hasher, 0, len(kvs)) + for _, kvp := range kvs { + kvsH = append(kvsH, KVPair(kvp)) + } + + rootHash, proofList := SimpleProofsFromHashers(kvsH) + proofs = make(map[string]*SimpleProof) + keys = make([]string, len(proofList)) + for i, kvp := range kvs { + proofs[string(kvp.Key)] = proofList[i] + keys[i] = string(kvp.Key) + } + return +} + +// Verify that leafHash is a leaf hash of the simple-merkle-tree +// which hashes to rootHash. +func (sp *SimpleProof) Verify(index int, total int, leafHash []byte, rootHash []byte) bool { + computedHash := computeHashFromAunts(index, total, leafHash, sp.Aunts) + return computedHash != nil && bytes.Equal(computedHash, rootHash) +} + +// String implements the stringer interface for SimpleProof. +// It is a wrapper around StringIndented. +func (sp *SimpleProof) String() string { + return sp.StringIndented("") +} + +// StringIndented generates a canonical string representation of a SimpleProof. +func (sp *SimpleProof) StringIndented(indent string) string { + return fmt.Sprintf(`SimpleProof{ +%s Aunts: %X +%s}`, + indent, sp.Aunts, + indent) +} + +// Use the leafHash and innerHashes to get the root merkle hash. +// If the length of the innerHashes slice isn't exactly correct, the result is nil. +// Recursive impl. +func computeHashFromAunts(index int, total int, leafHash []byte, innerHashes [][]byte) []byte { + if index >= total || index < 0 || total <= 0 { + return nil + } + switch total { + case 0: + panic("Cannot call computeHashFromAunts() with 0 total") + case 1: + if len(innerHashes) != 0 { + return nil + } + return leafHash + default: + if len(innerHashes) == 0 { + return nil + } + numLeft := (total + 1) / 2 + if index < numLeft { + leftHash := computeHashFromAunts(index, numLeft, leafHash, innerHashes[:len(innerHashes)-1]) + if leftHash == nil { + return nil + } + return SimpleHashFromTwoHashes(leftHash, innerHashes[len(innerHashes)-1]) + } + rightHash := computeHashFromAunts(index-numLeft, total-numLeft, leafHash, innerHashes[:len(innerHashes)-1]) + if rightHash == nil { + return nil + } + return SimpleHashFromTwoHashes(innerHashes[len(innerHashes)-1], rightHash) + } +} + +// SimpleProofNode is a helper structure to construct merkle proof. +// The node and the tree is thrown away afterwards. +// Exactly one of node.Left and node.Right is nil, unless node is the root, in which case both are nil. +// node.Parent.Hash = hash(node.Hash, node.Right.Hash) or +// hash(node.Left.Hash, node.Hash), depending on whether node is a left/right child. +type SimpleProofNode struct { + Hash []byte + Parent *SimpleProofNode + Left *SimpleProofNode // Left sibling (only one of Left,Right is set) + Right *SimpleProofNode // Right sibling (only one of Left,Right is set) +} + +// FlattenAunts will return the inner hashes for the item corresponding to the leaf, +// starting from a leaf SimpleProofNode. +func (spn *SimpleProofNode) FlattenAunts() [][]byte { + // Nonrecursive impl. + innerHashes := [][]byte{} + for spn != nil { + if spn.Left != nil { + innerHashes = append(innerHashes, spn.Left.Hash) + } else if spn.Right != nil { + innerHashes = append(innerHashes, spn.Right.Hash) + } else { + break + } + spn = spn.Parent + } + return innerHashes +} + +// trails[0].Hash is the leaf hash for items[0]. +// trails[i].Parent.Parent....Parent == root for all i. +func trailsFromHashers(items []Hasher) (trails []*SimpleProofNode, root *SimpleProofNode) { + // Recursive impl. + switch len(items) { + case 0: + return nil, nil + case 1: + trail := &SimpleProofNode{items[0].Hash(), nil, nil, nil} + return []*SimpleProofNode{trail}, trail + default: + lefts, leftRoot := trailsFromHashers(items[:(len(items)+1)/2]) + rights, rightRoot := trailsFromHashers(items[(len(items)+1)/2:]) + rootHash := SimpleHashFromTwoHashes(leftRoot.Hash, rightRoot.Hash) + root := &SimpleProofNode{rootHash, nil, nil, nil} + leftRoot.Parent = root + leftRoot.Right = rightRoot + rightRoot.Parent = root + rightRoot.Left = leftRoot + return append(lefts, rights...), root + } +} diff --git a/crypto/merkle/simple_tree.go b/crypto/merkle/simple_tree.go new file mode 100644 index 000000000..46a075909 --- /dev/null +++ b/crypto/merkle/simple_tree.go @@ -0,0 +1,58 @@ +package merkle + +import ( + "github.com/tendermint/tendermint/crypto/tmhash" +) + +// SimpleHashFromTwoHashes is the basic operation of the Merkle tree: Hash(left | right). +func SimpleHashFromTwoHashes(left, right []byte) []byte { + var hasher = tmhash.New() + err := encodeByteSlice(hasher, left) + if err != nil { + panic(err) + } + err = encodeByteSlice(hasher, right) + if err != nil { + panic(err) + } + return hasher.Sum(nil) +} + +// SimpleHashFromHashers computes a Merkle tree from items that can be hashed. +func SimpleHashFromHashers(items []Hasher) []byte { + hashes := make([][]byte, len(items)) + for i, item := range items { + hash := item.Hash() + hashes[i] = hash + } + return simpleHashFromHashes(hashes) +} + +// SimpleHashFromMap computes a Merkle tree from sorted map. +// Like calling SimpleHashFromHashers with +// `item = []byte(Hash(key) | Hash(value))`, +// sorted by `item`. +func SimpleHashFromMap(m map[string]Hasher) []byte { + sm := newSimpleMap() + for k, v := range m { + sm.Set(k, v) + } + return sm.Hash() +} + +//---------------------------------------------------------------- + +// Expects hashes! +func simpleHashFromHashes(hashes [][]byte) []byte { + // Recursive impl. + switch len(hashes) { + case 0: + return nil + case 1: + return hashes[0] + default: + left := simpleHashFromHashes(hashes[:(len(hashes)+1)/2]) + right := simpleHashFromHashes(hashes[(len(hashes)+1)/2:]) + return SimpleHashFromTwoHashes(left, right) + } +} diff --git a/crypto/merkle/simple_tree_test.go b/crypto/merkle/simple_tree_test.go new file mode 100644 index 000000000..488e0c907 --- /dev/null +++ b/crypto/merkle/simple_tree_test.go @@ -0,0 +1,88 @@ +package merkle + +import ( + "bytes" + + cmn "github.com/tendermint/tendermint/libs/common" + . "github.com/tendermint/tendermint/libs/test" + + "github.com/tendermint/tendermint/crypto/tmhash" + "testing" +) + +type testItem []byte + +func (tI testItem) Hash() []byte { + return []byte(tI) +} + +func TestSimpleProof(t *testing.T) { + + total := 100 + + items := make([]Hasher, total) + for i := 0; i < total; i++ { + items[i] = testItem(cmn.RandBytes(tmhash.Size)) + } + + rootHash := SimpleHashFromHashers(items) + + rootHash2, proofs := SimpleProofsFromHashers(items) + + if !bytes.Equal(rootHash, rootHash2) { + t.Errorf("Unmatched root hashes: %X vs %X", rootHash, rootHash2) + } + + // For each item, check the trail. + for i, item := range items { + itemHash := item.Hash() + proof := proofs[i] + + // Verify success + ok := proof.Verify(i, total, itemHash, rootHash) + if !ok { + t.Errorf("Verification failed for index %v.", i) + } + + // Wrong item index should make it fail + { + ok = proof.Verify((i+1)%total, total, itemHash, rootHash) + if ok { + t.Errorf("Expected verification to fail for wrong index %v.", i) + } + } + + // Trail too long should make it fail + origAunts := proof.Aunts + proof.Aunts = append(proof.Aunts, cmn.RandBytes(32)) + { + ok = proof.Verify(i, total, itemHash, rootHash) + if ok { + t.Errorf("Expected verification to fail for wrong trail length.") + } + } + proof.Aunts = origAunts + + // Trail too short should make it fail + proof.Aunts = proof.Aunts[0 : len(proof.Aunts)-1] + { + ok = proof.Verify(i, total, itemHash, rootHash) + if ok { + t.Errorf("Expected verification to fail for wrong trail length.") + } + } + proof.Aunts = origAunts + + // Mutating the itemHash should make it fail. + ok = proof.Verify(i, total, MutateByteSlice(itemHash), rootHash) + if ok { + t.Errorf("Expected verification to fail for mutated leaf hash") + } + + // Mutating the rootHash should make it fail. + ok = proof.Verify(i, total, itemHash, MutateByteSlice(rootHash)) + if ok { + t.Errorf("Expected verification to fail for mutated root hash") + } + } +} diff --git a/crypto/merkle/types.go b/crypto/merkle/types.go new file mode 100644 index 000000000..2fcb3f39d --- /dev/null +++ b/crypto/merkle/types.go @@ -0,0 +1,38 @@ +package merkle + +import ( + "io" + + amino "github.com/tendermint/go-amino" +) + +// Tree is a Merkle tree interface. +type Tree interface { + Size() (size int) + Height() (height int8) + Has(key []byte) (has bool) + Proof(key []byte) (value []byte, proof []byte, exists bool) // TODO make it return an index + Get(key []byte) (index int, value []byte, exists bool) + GetByIndex(index int) (key []byte, value []byte) + Set(key []byte, value []byte) (updated bool) + Remove(key []byte) (value []byte, removed bool) + HashWithCount() (hash []byte, count int) + Hash() (hash []byte) + Save() (hash []byte) + Load(hash []byte) + Copy() Tree + Iterate(func(key []byte, value []byte) (stop bool)) (stopped bool) + IterateRange(start []byte, end []byte, ascending bool, fx func(key []byte, value []byte) (stop bool)) (stopped bool) +} + +// Hasher represents a hashable piece of data which can be hashed in the Tree. +type Hasher interface { + Hash() []byte +} + +//----------------------------------------------------------------------- + +// Uvarint length prefixed byteslice +func encodeByteSlice(w io.Writer, bz []byte) (err error) { + return amino.EncodeByteSlice(w, bz) +} diff --git a/crypto/priv_key.go b/crypto/priv_key.go new file mode 100644 index 000000000..dbfe64c33 --- /dev/null +++ b/crypto/priv_key.go @@ -0,0 +1,164 @@ +package crypto + +import ( + "crypto/subtle" + + secp256k1 "github.com/btcsuite/btcd/btcec" + "github.com/tendermint/ed25519" + "github.com/tendermint/ed25519/extra25519" +) + +func PrivKeyFromBytes(privKeyBytes []byte) (privKey PrivKey, err error) { + err = cdc.UnmarshalBinaryBare(privKeyBytes, &privKey) + return +} + +//---------------------------------------- + +type PrivKey interface { + Bytes() []byte + Sign(msg []byte) (Signature, error) + PubKey() PubKey + Equals(PrivKey) bool +} + +//------------------------------------- + +var _ PrivKey = PrivKeyEd25519{} + +// Implements PrivKey +type PrivKeyEd25519 [64]byte + +func (privKey PrivKeyEd25519) Bytes() []byte { + return cdc.MustMarshalBinaryBare(privKey) +} + +func (privKey PrivKeyEd25519) Sign(msg []byte) (Signature, error) { + privKeyBytes := [64]byte(privKey) + signatureBytes := ed25519.Sign(&privKeyBytes, msg) + return SignatureEd25519(*signatureBytes), nil +} + +func (privKey PrivKeyEd25519) PubKey() PubKey { + privKeyBytes := [64]byte(privKey) + pubBytes := *ed25519.MakePublicKey(&privKeyBytes) + return PubKeyEd25519(pubBytes) +} + +// Equals - you probably don't need to use this. +// Runs in constant time based on length of the keys. +func (privKey PrivKeyEd25519) Equals(other PrivKey) bool { + if otherEd, ok := other.(PrivKeyEd25519); ok { + return subtle.ConstantTimeCompare(privKey[:], otherEd[:]) == 1 + } else { + return false + } +} + +func (privKey PrivKeyEd25519) ToCurve25519() *[32]byte { + keyCurve25519 := new([32]byte) + privKeyBytes := [64]byte(privKey) + extra25519.PrivateKeyToCurve25519(keyCurve25519, &privKeyBytes) + return keyCurve25519 +} + +// Deterministically generates new priv-key bytes from key. +func (privKey PrivKeyEd25519) Generate(index int) PrivKeyEd25519 { + bz, err := cdc.MarshalBinaryBare(struct { + PrivKey [64]byte + Index int + }{privKey, index}) + if err != nil { + panic(err) + } + newBytes := Sha256(bz) + newKey := new([64]byte) + copy(newKey[:32], newBytes) + ed25519.MakePublicKey(newKey) + return PrivKeyEd25519(*newKey) +} + +func GenPrivKeyEd25519() PrivKeyEd25519 { + privKeyBytes := new([64]byte) + copy(privKeyBytes[:32], CRandBytes(32)) + ed25519.MakePublicKey(privKeyBytes) + return PrivKeyEd25519(*privKeyBytes) +} + +// NOTE: secret should be the output of a KDF like bcrypt, +// if it's derived from user input. +func GenPrivKeyEd25519FromSecret(secret []byte) PrivKeyEd25519 { + privKey32 := Sha256(secret) // Not Ripemd160 because we want 32 bytes. + privKeyBytes := new([64]byte) + copy(privKeyBytes[:32], privKey32) + ed25519.MakePublicKey(privKeyBytes) + return PrivKeyEd25519(*privKeyBytes) +} + +//------------------------------------- + +var _ PrivKey = PrivKeySecp256k1{} + +// Implements PrivKey +type PrivKeySecp256k1 [32]byte + +func (privKey PrivKeySecp256k1) Bytes() []byte { + return cdc.MustMarshalBinaryBare(privKey) +} + +func (privKey PrivKeySecp256k1) Sign(msg []byte) (Signature, error) { + priv__, _ := secp256k1.PrivKeyFromBytes(secp256k1.S256(), privKey[:]) + sig__, err := priv__.Sign(Sha256(msg)) + if err != nil { + return nil, err + } + return SignatureSecp256k1(sig__.Serialize()), nil +} + +func (privKey PrivKeySecp256k1) PubKey() PubKey { + _, pub__ := secp256k1.PrivKeyFromBytes(secp256k1.S256(), privKey[:]) + var pub PubKeySecp256k1 + copy(pub[:], pub__.SerializeCompressed()) + return pub +} + +// Equals - you probably don't need to use this. +// Runs in constant time based on length of the keys. +func (privKey PrivKeySecp256k1) Equals(other PrivKey) bool { + if otherSecp, ok := other.(PrivKeySecp256k1); ok { + return subtle.ConstantTimeCompare(privKey[:], otherSecp[:]) == 1 + } else { + return false + } +} + +/* +// Deterministically generates new priv-key bytes from key. +func (key PrivKeySecp256k1) Generate(index int) PrivKeySecp256k1 { + newBytes := cdc.BinarySha256(struct { + PrivKey [64]byte + Index int + }{key, index}) + var newKey [64]byte + copy(newKey[:], newBytes) + return PrivKeySecp256k1(newKey) +} +*/ + +func GenPrivKeySecp256k1() PrivKeySecp256k1 { + privKeyBytes := [32]byte{} + copy(privKeyBytes[:], CRandBytes(32)) + priv, _ := secp256k1.PrivKeyFromBytes(secp256k1.S256(), privKeyBytes[:]) + copy(privKeyBytes[:], priv.Serialize()) + return PrivKeySecp256k1(privKeyBytes) +} + +// NOTE: secret should be the output of a KDF like bcrypt, +// if it's derived from user input. +func GenPrivKeySecp256k1FromSecret(secret []byte) PrivKeySecp256k1 { + privKey32 := Sha256(secret) // Not Ripemd160 because we want 32 bytes. + priv, _ := secp256k1.PrivKeyFromBytes(secp256k1.S256(), privKey32) + privKeyBytes := [32]byte{} + copy(privKeyBytes[:], priv.Serialize()) + return PrivKeySecp256k1(privKeyBytes) +} diff --git a/crypto/priv_key_test.go b/crypto/priv_key_test.go new file mode 100644 index 000000000..c1ae33ed1 --- /dev/null +++ b/crypto/priv_key_test.go @@ -0,0 +1,60 @@ +package crypto_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/tendermint/tendermint/crypto" +) + +func TestGeneratePrivKey(t *testing.T) { + testPriv := crypto.GenPrivKeyEd25519() + testGenerate := testPriv.Generate(1) + signBytes := []byte("something to sign") + pub := testGenerate.PubKey() + sig, err := testGenerate.Sign(signBytes) + assert.NoError(t, err) + assert.True(t, pub.VerifyBytes(signBytes, sig)) +} + +/* + +type BadKey struct { + PrivKeyEd25519 +} + +func TestReadPrivKey(t *testing.T) { + assert, require := assert.New(t), require.New(t) + + // garbage in, garbage out + garbage := []byte("hjgewugfbiewgofwgewr") + XXX This test wants to register BadKey globally to crypto, + but we don't want to support that. + _, err := PrivKeyFromBytes(garbage) + require.Error(err) + + edKey := GenPrivKeyEd25519() + badKey := BadKey{edKey} + + cases := []struct { + key PrivKey + valid bool + }{ + {edKey, true}, + {badKey, false}, + } + + for i, tc := range cases { + data := tc.key.Bytes() + fmt.Println(">>>", data) + key, err := PrivKeyFromBytes(data) + fmt.Printf("!!! %#v\n", key, err) + if tc.valid { + assert.NoError(err, "%d", i) + assert.Equal(tc.key, key, "%d", i) + } else { + assert.Error(err, "%d: %#v", i, key) + } + } +} +*/ diff --git a/crypto/pub_key.go b/crypto/pub_key.go new file mode 100644 index 000000000..588c54113 --- /dev/null +++ b/crypto/pub_key.go @@ -0,0 +1,153 @@ +package crypto + +import ( + "bytes" + "crypto/sha256" + "fmt" + + "golang.org/x/crypto/ripemd160" + + secp256k1 "github.com/btcsuite/btcd/btcec" + + "github.com/tendermint/ed25519" + "github.com/tendermint/ed25519/extra25519" + + cmn "github.com/tendermint/tendermint/libs/common" + + "github.com/tendermint/tendermint/crypto/tmhash" +) + +// An address is a []byte, but hex-encoded even in JSON. +// []byte leaves us the option to change the address length. +// Use an alias so Unmarshal methods (with ptr receivers) are available too. +type Address = cmn.HexBytes + +func PubKeyFromBytes(pubKeyBytes []byte) (pubKey PubKey, err error) { + err = cdc.UnmarshalBinaryBare(pubKeyBytes, &pubKey) + return +} + +//---------------------------------------- + +type PubKey interface { + Address() Address + Bytes() []byte + VerifyBytes(msg []byte, sig Signature) bool + Equals(PubKey) bool +} + +//------------------------------------- + +var _ PubKey = PubKeyEd25519{} + +const PubKeyEd25519Size = 32 + +// Implements PubKeyInner +type PubKeyEd25519 [PubKeyEd25519Size]byte + +// Address is the SHA256-20 of the raw pubkey bytes. +func (pubKey PubKeyEd25519) Address() Address { + return Address(tmhash.Sum(pubKey[:])) +} + +func (pubKey PubKeyEd25519) Bytes() []byte { + bz, err := cdc.MarshalBinaryBare(pubKey) + if err != nil { + panic(err) + } + return bz +} + +func (pubKey PubKeyEd25519) VerifyBytes(msg []byte, sig_ Signature) bool { + // make sure we use the same algorithm to sign + sig, ok := sig_.(SignatureEd25519) + if !ok { + return false + } + pubKeyBytes := [PubKeyEd25519Size]byte(pubKey) + sigBytes := [SignatureEd25519Size]byte(sig) + return ed25519.Verify(&pubKeyBytes, msg, &sigBytes) +} + +// For use with golang/crypto/nacl/box +// If error, returns nil. +func (pubKey PubKeyEd25519) ToCurve25519() *[PubKeyEd25519Size]byte { + keyCurve25519, pubKeyBytes := new([PubKeyEd25519Size]byte), [PubKeyEd25519Size]byte(pubKey) + ok := extra25519.PublicKeyToCurve25519(keyCurve25519, &pubKeyBytes) + if !ok { + return nil + } + return keyCurve25519 +} + +func (pubKey PubKeyEd25519) String() string { + return fmt.Sprintf("PubKeyEd25519{%X}", pubKey[:]) +} + +func (pubKey PubKeyEd25519) Equals(other PubKey) bool { + if otherEd, ok := other.(PubKeyEd25519); ok { + return bytes.Equal(pubKey[:], otherEd[:]) + } else { + return false + } +} + +//------------------------------------- + +var _ PubKey = PubKeySecp256k1{} + +const PubKeySecp256k1Size = 33 + +// Implements PubKey. +// Compressed pubkey (just the x-cord), +// prefixed with 0x02 or 0x03, depending on the y-cord. +type PubKeySecp256k1 [PubKeySecp256k1Size]byte + +// Implements Bitcoin style addresses: RIPEMD160(SHA256(pubkey)) +func (pubKey PubKeySecp256k1) Address() Address { + hasherSHA256 := sha256.New() + hasherSHA256.Write(pubKey[:]) // does not error + sha := hasherSHA256.Sum(nil) + + hasherRIPEMD160 := ripemd160.New() + hasherRIPEMD160.Write(sha) // does not error + return Address(hasherRIPEMD160.Sum(nil)) +} + +func (pubKey PubKeySecp256k1) Bytes() []byte { + bz, err := cdc.MarshalBinaryBare(pubKey) + if err != nil { + panic(err) + } + return bz +} + +func (pubKey PubKeySecp256k1) VerifyBytes(msg []byte, sig_ Signature) bool { + // and assert same algorithm to sign and verify + sig, ok := sig_.(SignatureSecp256k1) + if !ok { + return false + } + + pub__, err := secp256k1.ParsePubKey(pubKey[:], secp256k1.S256()) + if err != nil { + return false + } + sig__, err := secp256k1.ParseDERSignature(sig[:], secp256k1.S256()) + if err != nil { + return false + } + return sig__.Verify(Sha256(msg), pub__) +} + +func (pubKey PubKeySecp256k1) String() string { + return fmt.Sprintf("PubKeySecp256k1{%X}", pubKey[:]) +} + +func (pubKey PubKeySecp256k1) Equals(other PubKey) bool { + if otherSecp, ok := other.(PubKeySecp256k1); ok { + return bytes.Equal(pubKey[:], otherSecp[:]) + } else { + return false + } +} diff --git a/crypto/pub_key_test.go b/crypto/pub_key_test.go new file mode 100644 index 000000000..7b856cf18 --- /dev/null +++ b/crypto/pub_key_test.go @@ -0,0 +1,50 @@ +package crypto + +import ( + "encoding/hex" + "testing" + + "github.com/btcsuite/btcutil/base58" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type keyData struct { + priv string + pub string + addr string +} + +var secpDataTable = []keyData{ + { + priv: "a96e62ed3955e65be32703f12d87b6b5cf26039ecfa948dc5107a495418e5330", + pub: "02950e1cdfcb133d6024109fd489f734eeb4502418e538c28481f22bce276f248c", + addr: "1CKZ9Nx4zgds8tU7nJHotKSDr4a9bYJCa3", + }, +} + +func TestPubKeySecp256k1Address(t *testing.T) { + for _, d := range secpDataTable { + privB, _ := hex.DecodeString(d.priv) + pubB, _ := hex.DecodeString(d.pub) + addrBbz, _, _ := base58.CheckDecode(d.addr) + addrB := Address(addrBbz) + + var priv PrivKeySecp256k1 + copy(priv[:], privB) + + pubKey := priv.PubKey() + pubT, _ := pubKey.(PubKeySecp256k1) + pub := pubT[:] + addr := pubKey.Address() + + assert.Equal(t, pub, pubB, "Expected pub keys to match") + assert.Equal(t, addr, addrB, "Expected addresses to match") + } +} + +func TestPubKeyInvalidDataProperReturnsEmpty(t *testing.T) { + pk, err := PubKeyFromBytes([]byte("foo")) + require.NotNil(t, err, "expecting a non-nil error") + require.Nil(t, pk, "expecting an empty public key on error") +} diff --git a/crypto/random.go b/crypto/random.go new file mode 100644 index 000000000..5c5057d30 --- /dev/null +++ b/crypto/random.go @@ -0,0 +1,108 @@ +package crypto + +import ( + "crypto/aes" + "crypto/cipher" + crand "crypto/rand" + "crypto/sha256" + "encoding/hex" + "io" + "sync" + + . "github.com/tendermint/tendermint/libs/common" +) + +var gRandInfo *randInfo + +func init() { + gRandInfo = &randInfo{} + gRandInfo.MixEntropy(randBytes(32)) // Init +} + +// Mix additional bytes of randomness, e.g. from hardware, user-input, etc. +// It is OK to call it multiple times. It does not diminish security. +func MixEntropy(seedBytes []byte) { + gRandInfo.MixEntropy(seedBytes) +} + +// This only uses the OS's randomness +func randBytes(numBytes int) []byte { + b := make([]byte, numBytes) + _, err := crand.Read(b) + if err != nil { + PanicCrisis(err) + } + return b +} + +// This uses the OS and the Seed(s). +func CRandBytes(numBytes int) []byte { + b := make([]byte, numBytes) + _, err := gRandInfo.Read(b) + if err != nil { + PanicCrisis(err) + } + return b +} + +// CRandHex returns a hex encoded string that's floor(numDigits/2) * 2 long. +// +// Note: CRandHex(24) gives 96 bits of randomness that +// are usually strong enough for most purposes. +func CRandHex(numDigits int) string { + return hex.EncodeToString(CRandBytes(numDigits / 2)) +} + +// Returns a crand.Reader mixed with user-supplied entropy +func CReader() io.Reader { + return gRandInfo +} + +//-------------------------------------------------------------------------------- + +type randInfo struct { + mtx sync.Mutex + seedBytes [32]byte + cipherAES256 cipher.Block + streamAES256 cipher.Stream + reader io.Reader +} + +// You can call this as many times as you'd like. +// XXX TODO review +func (ri *randInfo) MixEntropy(seedBytes []byte) { + ri.mtx.Lock() + defer ri.mtx.Unlock() + // Make new ri.seedBytes using passed seedBytes and current ri.seedBytes: + // ri.seedBytes = sha256( seedBytes || ri.seedBytes ) + h := sha256.New() + h.Write(seedBytes) + h.Write(ri.seedBytes[:]) + hashBytes := h.Sum(nil) + hashBytes32 := [32]byte{} + copy(hashBytes32[:], hashBytes) + ri.seedBytes = xorBytes32(ri.seedBytes, hashBytes32) + // Create new cipher.Block + var err error + ri.cipherAES256, err = aes.NewCipher(ri.seedBytes[:]) + if err != nil { + PanicSanity("Error creating AES256 cipher: " + err.Error()) + } + // Create new stream + ri.streamAES256 = cipher.NewCTR(ri.cipherAES256, randBytes(aes.BlockSize)) + // Create new reader + ri.reader = &cipher.StreamReader{S: ri.streamAES256, R: crand.Reader} +} + +func (ri *randInfo) Read(b []byte) (n int, err error) { + ri.mtx.Lock() + defer ri.mtx.Unlock() + return ri.reader.Read(b) +} + +func xorBytes32(bytesA [32]byte, bytesB [32]byte) (res [32]byte) { + for i, b := range bytesA { + res[i] = b ^ bytesB[i] + } + return res +} diff --git a/crypto/signature.go b/crypto/signature.go new file mode 100644 index 000000000..ae447da64 --- /dev/null +++ b/crypto/signature.go @@ -0,0 +1,90 @@ +package crypto + +import ( + "fmt" + + "crypto/subtle" + + . "github.com/tendermint/tendermint/libs/common" +) + +func SignatureFromBytes(pubKeyBytes []byte) (pubKey Signature, err error) { + err = cdc.UnmarshalBinaryBare(pubKeyBytes, &pubKey) + return +} + +//---------------------------------------- + +type Signature interface { + Bytes() []byte + IsZero() bool + Equals(Signature) bool +} + +//------------------------------------- + +var _ Signature = SignatureEd25519{} + +const SignatureEd25519Size = 64 + +// Implements Signature +type SignatureEd25519 [SignatureEd25519Size]byte + +func (sig SignatureEd25519) Bytes() []byte { + bz, err := cdc.MarshalBinaryBare(sig) + if err != nil { + panic(err) + } + return bz +} + +func (sig SignatureEd25519) IsZero() bool { return len(sig) == 0 } + +func (sig SignatureEd25519) String() string { return fmt.Sprintf("/%X.../", Fingerprint(sig[:])) } + +func (sig SignatureEd25519) Equals(other Signature) bool { + if otherEd, ok := other.(SignatureEd25519); ok { + return subtle.ConstantTimeCompare(sig[:], otherEd[:]) == 1 + } else { + return false + } +} + +func SignatureEd25519FromBytes(data []byte) Signature { + var sig SignatureEd25519 + copy(sig[:], data) + return sig +} + +//------------------------------------- + +var _ Signature = SignatureSecp256k1{} + +// Implements Signature +type SignatureSecp256k1 []byte + +func (sig SignatureSecp256k1) Bytes() []byte { + bz, err := cdc.MarshalBinaryBare(sig) + if err != nil { + panic(err) + } + return bz +} + +func (sig SignatureSecp256k1) IsZero() bool { return len(sig) == 0 } + +func (sig SignatureSecp256k1) String() string { return fmt.Sprintf("/%X.../", Fingerprint(sig[:])) } + +func (sig SignatureSecp256k1) Equals(other Signature) bool { + if otherSecp, ok := other.(SignatureSecp256k1); ok { + return subtle.ConstantTimeCompare(sig[:], otherSecp[:]) == 1 + } else { + return false + } +} + +func SignatureSecp256k1FromBytes(data []byte) Signature { + sig := make(SignatureSecp256k1, len(data)) + copy(sig[:], data) + return sig +} diff --git a/crypto/signature_test.go b/crypto/signature_test.go new file mode 100644 index 000000000..d6ae2b7a9 --- /dev/null +++ b/crypto/signature_test.go @@ -0,0 +1,46 @@ +package crypto + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSignAndValidateEd25519(t *testing.T) { + + privKey := GenPrivKeyEd25519() + pubKey := privKey.PubKey() + + msg := CRandBytes(128) + sig, err := privKey.Sign(msg) + require.Nil(t, err) + + // Test the signature + assert.True(t, pubKey.VerifyBytes(msg, sig)) + + // Mutate the signature, just one bit. + sigEd := sig.(SignatureEd25519) + sigEd[7] ^= byte(0x01) + sig = sigEd + + assert.False(t, pubKey.VerifyBytes(msg, sig)) +} + +func TestSignAndValidateSecp256k1(t *testing.T) { + privKey := GenPrivKeySecp256k1() + pubKey := privKey.PubKey() + + msg := CRandBytes(128) + sig, err := privKey.Sign(msg) + require.Nil(t, err) + + assert.True(t, pubKey.VerifyBytes(msg, sig)) + + // Mutate the signature, just one bit. + sigEd := sig.(SignatureSecp256k1) + sigEd[3] ^= byte(0x01) + sig = sigEd + + assert.False(t, pubKey.VerifyBytes(msg, sig)) +} diff --git a/crypto/symmetric.go b/crypto/symmetric.go new file mode 100644 index 000000000..62379c15f --- /dev/null +++ b/crypto/symmetric.go @@ -0,0 +1,51 @@ +package crypto + +import ( + "errors" + + . "github.com/tendermint/tendermint/libs/common" + "golang.org/x/crypto/nacl/secretbox" +) + +const nonceLen = 24 +const secretLen = 32 + +// secret must be 32 bytes long. Use something like Sha256(Bcrypt(passphrase)) +// The ciphertext is (secretbox.Overhead + 24) bytes longer than the plaintext. +// NOTE: call crypto.MixEntropy() first. +func EncryptSymmetric(plaintext []byte, secret []byte) (ciphertext []byte) { + if len(secret) != secretLen { + PanicSanity(Fmt("Secret must be 32 bytes long, got len %v", len(secret))) + } + nonce := CRandBytes(nonceLen) + nonceArr := [nonceLen]byte{} + copy(nonceArr[:], nonce) + secretArr := [secretLen]byte{} + copy(secretArr[:], secret) + ciphertext = make([]byte, nonceLen+secretbox.Overhead+len(plaintext)) + copy(ciphertext, nonce) + secretbox.Seal(ciphertext[nonceLen:nonceLen], plaintext, &nonceArr, &secretArr) + return ciphertext +} + +// secret must be 32 bytes long. Use something like Sha256(Bcrypt(passphrase)) +// The ciphertext is (secretbox.Overhead + 24) bytes longer than the plaintext. +func DecryptSymmetric(ciphertext []byte, secret []byte) (plaintext []byte, err error) { + if len(secret) != secretLen { + PanicSanity(Fmt("Secret must be 32 bytes long, got len %v", len(secret))) + } + if len(ciphertext) <= secretbox.Overhead+nonceLen { + return nil, errors.New("Ciphertext is too short") + } + nonce := ciphertext[:nonceLen] + nonceArr := [nonceLen]byte{} + copy(nonceArr[:], nonce) + secretArr := [secretLen]byte{} + copy(secretArr[:], secret) + plaintext = make([]byte, len(ciphertext)-nonceLen-secretbox.Overhead) + _, ok := secretbox.Open(plaintext[:0], ciphertext[nonceLen:], &nonceArr, &secretArr) + if !ok { + return nil, errors.New("Ciphertext decryption failed") + } + return plaintext, nil +} diff --git a/crypto/symmetric_test.go b/crypto/symmetric_test.go new file mode 100644 index 000000000..d92bff1aa --- /dev/null +++ b/crypto/symmetric_test.go @@ -0,0 +1,42 @@ +package crypto + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "golang.org/x/crypto/bcrypt" +) + +func TestSimple(t *testing.T) { + + MixEntropy([]byte("someentropy")) + + plaintext := []byte("sometext") + secret := []byte("somesecretoflengththirtytwo===32") + ciphertext := EncryptSymmetric(plaintext, secret) + plaintext2, err := DecryptSymmetric(ciphertext, secret) + + require.Nil(t, err, "%+v", err) + assert.Equal(t, plaintext, plaintext2) +} + +func TestSimpleWithKDF(t *testing.T) { + + MixEntropy([]byte("someentropy")) + + plaintext := []byte("sometext") + secretPass := []byte("somesecret") + secret, err := bcrypt.GenerateFromPassword(secretPass, 12) + if err != nil { + t.Error(err) + } + secret = Sha256(secret) + + ciphertext := EncryptSymmetric(plaintext, secret) + plaintext2, err := DecryptSymmetric(ciphertext, secret) + + require.Nil(t, err, "%+v", err) + assert.Equal(t, plaintext, plaintext2) +} diff --git a/crypto/tmhash/hash.go b/crypto/tmhash/hash.go new file mode 100644 index 000000000..1b29d8680 --- /dev/null +++ b/crypto/tmhash/hash.go @@ -0,0 +1,48 @@ +package tmhash + +import ( + "crypto/sha256" + "hash" +) + +const ( + Size = 20 + BlockSize = sha256.BlockSize +) + +type sha256trunc struct { + sha256 hash.Hash +} + +func (h sha256trunc) Write(p []byte) (n int, err error) { + return h.sha256.Write(p) +} +func (h sha256trunc) Sum(b []byte) []byte { + shasum := h.sha256.Sum(b) + return shasum[:Size] +} + +func (h sha256trunc) Reset() { + h.sha256.Reset() +} + +func (h sha256trunc) Size() int { + return Size +} + +func (h sha256trunc) BlockSize() int { + return h.sha256.BlockSize() +} + +// New returns a new hash.Hash. +func New() hash.Hash { + return sha256trunc{ + sha256: sha256.New(), + } +} + +// Sum returns the first 20 bytes of SHA256 of the bz. +func Sum(bz []byte) []byte { + hash := sha256.Sum256(bz) + return hash[:Size] +} diff --git a/crypto/tmhash/hash_test.go b/crypto/tmhash/hash_test.go new file mode 100644 index 000000000..27938039a --- /dev/null +++ b/crypto/tmhash/hash_test.go @@ -0,0 +1,23 @@ +package tmhash_test + +import ( + "crypto/sha256" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/tendermint/tendermint/crypto/tmhash" +) + +func TestHash(t *testing.T) { + testVector := []byte("abc") + hasher := tmhash.New() + hasher.Write(testVector) + bz := hasher.Sum(nil) + + hasher = sha256.New() + hasher.Write(testVector) + bz2 := hasher.Sum(nil) + bz2 = bz2[:20] + + assert.Equal(t, bz, bz2) +} diff --git a/crypto/version.go b/crypto/version.go new file mode 100644 index 000000000..77c0bed8a --- /dev/null +++ b/crypto/version.go @@ -0,0 +1,3 @@ +package crypto + +const Version = "0.9.0-dev" diff --git a/docs/README.md b/docs/README.md index 180acdcb9..0ed3aaec9 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,14 +1,28 @@ -Here lies our documentation. After making edits, run: +# Tendermint -``` -pip install -r requirements.txt -make html -``` +Welcome to the Tendermint Core documentation! The introduction below provides +an overview to help you navigate to your area of interest. -to build the docs locally then open the file `_build/html/index.html` in your browser. +## Introduction -**WARNING:** This documentation is intended to be viewed at: +Tendermint Core is Byzantine Fault Tolerant (BFT) middleware that takes a state +transition machine - written in any programming language - and securely +replicates it on many machines. In other words, a blockchain. -https://tendermint.readthedocs.io +Tendermint requires an application running over the Application Blockchain +Interface (ABCI) - and comes packaged with an example application to do so. +Follow the [installation instructions](./install) to get up and running +quickly. For more details on [using tendermint](./using-tendermint) see that +and the following sections. -and may contain broken internal links when viewed from Github. +## Networks + +Testnets can be setup manually on one or more machines, or automatically on one +or more machine, using a variety of methods described in the [deploy testnets +section](./deploy-testnets). For more information (and to join) about the +Cosmos Network testnets, see [here](/getting-started/full-node.md). + +## Application Development + +The first step to building application on Tendermint is to [install +ABCI-CLI](./getting-started) and play with the example applications. diff --git a/docs/_static/custom_collapsible_code.css b/docs/_static/custom_collapsible_code.css deleted file mode 100644 index 695268a83..000000000 --- a/docs/_static/custom_collapsible_code.css +++ /dev/null @@ -1,17 +0,0 @@ -.toggle { - padding-bottom: 1em ; -} - -.toggle .header { - display: block; - clear: both; - cursor: pointer; -} - -.toggle .header:after { - content: " ▼"; -} - -.toggle .header.open:after { - content: " ▲"; -} \ No newline at end of file diff --git a/docs/_static/custom_collapsible_code.js b/docs/_static/custom_collapsible_code.js deleted file mode 100644 index f4ff22adc..000000000 --- a/docs/_static/custom_collapsible_code.js +++ /dev/null @@ -1,10 +0,0 @@ -let makeCodeBlocksCollapsible = function() { - $(".toggle > *").hide(); - $(".toggle .header").show(); - $(".toggle .header").click(function() { - $(this).parent().children().not(".header").toggle({"duration": 400}); - $(this).parent().children(".header").toggleClass("open"); - }); -}; -// we could use the }(); way if we would have access to jQuery in HEAD, i.e. we would need to force the theme -// to load jQuery before our custom scripts diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html deleted file mode 100644 index 736460bc0..000000000 --- a/docs/_templates/layout.html +++ /dev/null @@ -1,20 +0,0 @@ -{% extends "!layout.html" %} - -{% set css_files = css_files + ["_static/custom_collapsible_code.css"] %} - -# sadly, I didn't find a css style way to add custom JS to a list that is automagically added to head like CSS (above) #} -{% block extrahead %} - -{% endblock %} - -{% block footer %} - -{% endblock %} - - diff --git a/docs/abci-cli.md b/docs/abci-cli.md index 14095d162..246e8b1a2 100644 --- a/docs/abci-cli.md +++ b/docs/abci-cli.md @@ -10,15 +10,14 @@ Make sure you [have Go installed](https://golang.org/doc/install). Next, install the `abci-cli` tool and example applications: - go get -u github.com/tendermint/abci/cmd/abci-cli + go get github.com/tendermint/tendermint -If this fails, you may need to use [dep](https://github.com/golang/dep) to get vendored dependencies: - cd $GOPATH/src/github.com/tendermint/abci + cd $GOPATH/src/github.com/tendermint/tendermint make get_tools make get_vendor_deps - make install + make install_abci Now run `abci-cli` to see the list of commands: @@ -61,7 +60,7 @@ as `abci-cli` above. The kvstore just stores transactions in a merkle tree. Its code can be found -[here](https://github.com/tendermint/abci/blob/master/cmd/abci-cli/abci-cli.go) +[here](https://github.com/tendermint/tendermint/blob/develop/abci/cmd/abci-cli/abci-cli.go) and looks like: func cmdKVStore(cmd *cobra.Command, args []string) error { @@ -124,7 +123,7 @@ response. The server may be generic for a particular language, and we provide a [reference implementation in -Golang](https://github.com/tendermint/abci/tree/master/server). See the +Golang](https://github.com/tendermint/tendermint/tree/develop/abci/server). See the [list of other ABCI implementations](./ecosystem.html) for servers in other languages. @@ -204,7 +203,7 @@ Now that we've got the hang of it, let's try another application, the "counter" app. Like the kvstore app, its code can be found -[here](https://github.com/tendermint/abci/blob/master/cmd/abci-cli/abci-cli.go) +[here](https://github.com/tendermint/tendermint/blob/master/abci/cmd/abci-cli/abci-cli.go) and looks like: func cmdCounter(cmd *cobra.Command, args []string) error { @@ -301,7 +300,7 @@ But the ultimate flexibility comes from being able to write the application easily in any language. We have implemented the counter in a number of languages [see the -example directory](https://github.com/tendermint/abci/tree/master/example). +example directory](https://github.com/tendermint/tendermint/tree/develop/abci/example). To run the Node JS version, `cd` to `example/js` and run @@ -324,6 +323,6 @@ connects to the app using three separate connections, each with its own pattern of messages. For more information, see the [application developers -guide](./app-development.html). For examples of running an ABCI app with -Tendermint, see the [getting started guide](./getting-started.html). +guide](./app-development.md). For examples of running an ABCI app with +Tendermint, see the [getting started guide](./getting-started.md). Next is the ABCI specification. diff --git a/docs/abci-spec.md b/docs/abci-spec.md new file mode 100644 index 000000000..f9677f448 --- /dev/null +++ b/docs/abci-spec.md @@ -0,0 +1,324 @@ +# ABCI Specification + +## Message Types + +ABCI requests/responses are defined as simple Protobuf messages in [this +schema file](https://github.com/tendermint/tendermint/blob/develop/abci/types/types.proto). +TendermintCore sends the requests, and the ABCI application sends the +responses. Here, we provide an overview of the messages types and how +they are used by Tendermint. Then we describe each request-response pair +as a function with arguments and return values, and add some notes on +usage. + +Some messages (`Echo, Info, InitChain, BeginBlock, EndBlock, Commit`), +don't return errors because an error would indicate a critical failure +in the application and there's nothing Tendermint can do. The problem +should be addressed and both Tendermint and the application restarted. +All other messages (`SetOption, Query, CheckTx, DeliverTx`) return an +application-specific response `Code uint32`, where only `0` is reserved +for `OK`. + +Some messages (`SetOption, Query, CheckTx, DeliverTx`) return +non-deterministic data in the form of `Info` and `Log`. The `Log` is +intended for the literal output from the application's logger, while the +`Info` is any additional info that should be returned. + +The first time a new blockchain is started, Tendermint calls +`InitChain`. From then on, the Block Execution Sequence that causes the +committed state to be updated is as follows: + +`BeginBlock, [DeliverTx], EndBlock, Commit` + +where one `DeliverTx` is called for each transaction in the block. +Cryptographic commitments to the results of DeliverTx, EndBlock, and +Commit are included in the header of the next block. + +Tendermint opens three connections to the application to handle the +different message types: + +- `Consensus Connection - InitChain, BeginBlock, DeliverTx, EndBlock, Commit` +- `Mempool Connection - CheckTx` +- `Info Connection - Info, SetOption, Query` + +The `Flush` message is used on every connection, and the `Echo` message +is only used for debugging. + +Note that messages may be sent concurrently across all connections -a +typical application will thus maintain a distinct state for each +connection. They may be referred to as the `DeliverTx state`, the +`CheckTx state`, and the `Commit state` respectively. + +See below for more details on the message types and how they are used. + +## Request/Response Messages + +### Echo + +- **Request**: + - `Message (string)`: A string to echo back +- **Response**: + - `Message (string)`: The input string +- **Usage**: + - Echo a string to test an abci client/server implementation + +### Flush + +- **Usage**: + - Signals that messages queued on the client should be flushed to + the server. It is called periodically by the client + implementation to ensure asynchronous requests are actually + sent, and is called immediately to make a synchronous request, + which returns when the Flush response comes back. + +### Info + +- **Request**: + - `Version (string)`: The Tendermint version +- **Response**: + - `Data (string)`: Some arbitrary information + - `Version (Version)`: Version information + - `LastBlockHeight (int64)`: Latest block for which the app has + called Commit + - `LastBlockAppHash ([]byte)`: Latest result of Commit +- **Usage**: + - Return information about the application state. + - Used to sync Tendermint with the application during a handshake + that happens on startup. + - Tendermint expects `LastBlockAppHash` and `LastBlockHeight` to + be updated during `Commit`, ensuring that `Commit` is never + called twice for the same block height. + +### SetOption + +- **Request**: + - `Key (string)`: Key to set + - `Value (string)`: Value to set for key +- **Response**: + - `Code (uint32)`: Response code + - `Log (string)`: The output of the application's logger. May + be non-deterministic. + - `Info (string)`: Additional information. May + be non-deterministic. +- **Usage**: + - Set non-consensus critical application specific options. + - e.g. Key="min-fee", Value="100fermion" could set the minimum fee + required for CheckTx (but not DeliverTx - that would be + consensus critical). + +### InitChain + +- **Request**: + - `Validators ([]Validator)`: Initial genesis validators + - `AppStateBytes ([]byte)`: Serialized initial application state +- **Response**: + - `ConsensusParams (ConsensusParams)`: Initial + consensus-critical parameters. + - `Validators ([]Validator)`: Initial validator set. +- **Usage**: + - Called once upon genesis. + +### Query + +- **Request**: + - `Data ([]byte)`: Raw query bytes. Can be used with or in lieu + of Path. + - `Path (string)`: Path of request, like an HTTP GET path. Can be + used with or in liue of Data. + - Apps MUST interpret '/store' as a query by key on the + underlying store. The key SHOULD be specified in the Data field. + - Apps SHOULD allow queries over specific types like + '/accounts/...' or '/votes/...' + - `Height (int64)`: The block height for which you want the query + (default=0 returns data for the latest committed block). Note + that this is the height of the block containing the + application's Merkle root hash, which represents the state as it + was after committing the block at Height-1 + - `Prove (bool)`: Return Merkle proof with response if possible +- **Response**: + - `Code (uint32)`: Response code. + - `Log (string)`: The output of the application's logger. May + be non-deterministic. + - `Info (string)`: Additional information. May + be non-deterministic. + - `Index (int64)`: The index of the key in the tree. + - `Key ([]byte)`: The key of the matching data. + - `Value ([]byte)`: The value of the matching data. + - `Proof ([]byte)`: Proof for the data, if requested. + - `Height (int64)`: The block height from which data was derived. + Note that this is the height of the block containing the + application's Merkle root hash, which represents the state as it + was after committing the block at Height-1 +- **Usage**: + - Query for data from the application at current or past height. + - Optionally return Merkle proof. + +### BeginBlock + +- **Request**: + - `Hash ([]byte)`: The block's hash. This can be derived from the + block header. + - `Header (struct{})`: The block header + - `Validators ([]SigningValidator)`: List of validators in the current validator + set and whether or not they signed a vote in the LastCommit + - `ByzantineValidators ([]Evidence)`: List of evidence of + validators that acted maliciously +- **Response**: + - `Tags ([]cmn.KVPair)`: Key-Value tags for filtering and indexing +- **Usage**: + - Signals the beginning of a new block. Called prior to + any DeliverTxs. + - The header is expected to at least contain the Height. + - The `Validators` and `ByzantineValidators` can be used to + determine rewards and punishments for the validators. + +### CheckTx + +- **Request**: + - `Tx ([]byte)`: The request transaction bytes +- **Response**: + - `Code (uint32)`: Response code + - `Data ([]byte)`: Result bytes, if any. + - `Log (string)`: The output of the application's logger. May + be non-deterministic. + - `Info (string)`: Additional information. May + be non-deterministic. + - `GasWanted (int64)`: Amount of gas request for transaction. + - `GasUsed (int64)`: Amount of gas consumed by transaction. + - `Tags ([]cmn.KVPair)`: Key-Value tags for filtering and indexing + transactions (eg. by account). + - `Fee (cmn.KI64Pair)`: Fee paid for the transaction. +- **Usage**: Validate a mempool transaction, prior to broadcasting + or proposing. CheckTx should perform stateful but light-weight + checks of the validity of the transaction (like checking signatures + and account balances), but need not execute in full (like running a + smart contract). + + Tendermint runs CheckTx and DeliverTx concurrently with eachother, + though on distinct ABCI connections - the mempool connection and the + consensus connection, respectively. + + The application should maintain a separate state to support CheckTx. + This state can be reset to the latest committed state during + `Commit`, where Tendermint ensures the mempool is locked and not + sending new `CheckTx`. After `Commit`, the mempool will rerun + CheckTx on all remaining transactions, throwing out any that are no + longer valid. + + Keys and values in Tags must be UTF-8 encoded strings (e.g. + "account.owner": "Bob", "balance": "100.0", "date": "2018-01-02") + +### DeliverTx + +- **Request**: + - `Tx ([]byte)`: The request transaction bytes. +- **Response**: + - `Code (uint32)`: Response code. + - `Data ([]byte)`: Result bytes, if any. + - `Log (string)`: The output of the application's logger. May + be non-deterministic. + - `Info (string)`: Additional information. May + be non-deterministic. + - `GasWanted (int64)`: Amount of gas requested for transaction. + - `GasUsed (int64)`: Amount of gas consumed by transaction. + - `Tags ([]cmn.KVPair)`: Key-Value tags for filtering and indexing + transactions (eg. by account). + - `Fee (cmn.KI64Pair)`: Fee paid for the transaction. +- **Usage**: + - Deliver a transaction to be executed in full by the application. + If the transaction is valid, returns CodeType.OK. + - Keys and values in Tags must be UTF-8 encoded strings (e.g. + "account.owner": "Bob", "balance": "100.0", + "time": "2018-01-02T12:30:00Z") + +### EndBlock + +- **Request**: + - `Height (int64)`: Height of the block just executed. +- **Response**: + - `ValidatorUpdates ([]Validator)`: Changes to validator set (set + voting power to 0 to remove). + - `ConsensusParamUpdates (ConsensusParams)`: Changes to + consensus-critical time, size, and other parameters. + - `Tags ([]cmn.KVPair)`: Key-Value tags for filtering and indexing +- **Usage**: + - Signals the end of a block. + - Called prior to each Commit, after all transactions. + - Validator set and consensus params are updated with the result. + - Validator pubkeys are expected to be go-wire encoded. + +### Commit + +- **Response**: + - `Data ([]byte)`: The Merkle root hash +- **Usage**: + - Persist the application state. + - Return a Merkle root hash of the application state. + - It's critical that all application instances return the + same hash. If not, they will not be able to agree on the next + block, because the hash is included in the next block! + +## Data Messages + +### Header + +- **Fields**: + - `ChainID (string)`: ID of the blockchain + - `Height (int64)`: Height of the block in the chain + - `Time (int64)`: Unix time of the block + - `NumTxs (int32)`: Number of transactions in the block + - `TotalTxs (int64)`: Total number of transactions in the blockchain until + now + - `LastBlockHash ([]byte)`: Hash of the previous (parent) block + - `ValidatorsHash ([]byte)`: Hash of the validator set for this block + - `AppHash ([]byte)`: Data returned by the last call to `Commit` - typically the + Merkle root of the application state after executing the previous block's + transactions + - `Proposer (Validator)`: Original proposer for the block +- **Usage**: + - Provided in RequestBeginBlock + - Provides important context about the current state of the blockchain - + especially height and time. + - Provides the proposer of the current block, for use in proposer-based + reward mechanisms. + +### Validator + +- **Fields**: + - `Address ([]byte)`: Address of the validator (hash of the public key) + - `PubKey (PubKey)`: Public key of the validator + - `Power (int64)`: Voting power of the validator +- **Usage**: + - Provides all identifying information about the validator + +### SigningValidator + +- **Fields**: + - `Validator (Validator)`: A validator + - `SignedLastBlock (bool)`: Indicated whether or not the validator signed + the last block +- **Usage**: + - Indicates whether a validator signed the last block, allowing for rewards + based on validator availability + +### PubKey + +- **Fields**: + - `Type (string)`: Type of the public key. A simple string like `"ed25519"`. + In the future, may indicate a serialization algorithm to parse the `Data`, + for instance `"amino"`. + - `Data ([]byte)`: Public key data. For a simple public key, it's just the + raw bytes. If the `Type` indicates an encoding algorithm, this is the + encoded public key. +- **Usage**: + - A generic and extensible typed public key + +### Evidence + +- **Fields**: + - `Type (string)`: Type of the evidence. A hierarchical path like + "duplicate/vote". + - `Validator (Validator`: The offending validator + - `Height (int64)`: Height when the offense was committed + - `Time (int64)`: Unix time of the block at height `Height` + - `TotalVotingPower (int64)`: Total voting power of the validator set at + height `Height` diff --git a/docs/app-architecture.md b/docs/app-architecture.md index 64b1a3794..401e28cca 100644 --- a/docs/app-architecture.md +++ b/docs/app-architecture.md @@ -47,4 +47,4 @@ See the following for more extensive documentation: - [Tendermint RPC Docs](https://tendermint.github.io/slate/) - [Tendermint in Production](https://github.com/tendermint/tendermint/pull/1618) - [Tendermint Basics](https://tendermint.readthedocs.io/en/master/using-tendermint.html) -- [ABCI spec](https://github.com/tendermint/abci/blob/develop/specification.md) +- [ABCI spec](https://github.com/tendermint/tendermint/blob/develop/abci/docs/abci-spec.md) diff --git a/docs/app-development.md b/docs/app-development.md index 488657670..f8c70f21b 100644 --- a/docs/app-development.md +++ b/docs/app-development.md @@ -49,7 +49,7 @@ The message protocol consists of pairs of requests and responses. Some messages have no fields, while others may include byte-arrays, strings, or integers. See the `message Request` and `message Response` definitions in [the protobuf definition -file](https://github.com/tendermint/abci/blob/master/types/types.proto), +file](https://github.com/tendermint/tendermint/blob/develop/abci/types/types.proto), and the [protobuf documentation](https://developers.google.com/protocol-buffers/docs/overview) for more details. @@ -72,9 +72,9 @@ Both can be tested using the `abci-cli` by setting the `--abci` flag appropriately (ie. to `socket` or `grpc`). See examples, in various stages of maintenance, in -[Go](https://github.com/tendermint/abci/tree/master/server), +[Go](https://github.com/tendermint/tendermint/tree/develop/abci/server), [JavaScript](https://github.com/tendermint/js-abci), -[Python](https://github.com/tendermint/abci/tree/master/example/python3/abci), +[Python](https://github.com/tendermint/tendermint/tree/develop/abci/example/python3/abci), [C++](https://github.com/mdyring/cpp-tmsp), and [Java](https://github.com/jTendermint/jabci). @@ -84,7 +84,7 @@ If GRPC is available in your language, this is the easiest approach, though it will have significant performance overhead. To get started with GRPC, copy in the [protobuf -file](https://github.com/tendermint/abci/blob/master/types/types.proto) +file](https://github.com/tendermint/tendermint/blob/develop/abci/types/types.proto) and compile it using the GRPC plugin for your language. For instance, for golang, the command is `protoc --go_out=plugins=grpc:. types.proto`. See the [grpc documentation for more details](http://www.grpc.io/docs/). @@ -99,14 +99,14 @@ performance, or otherwise enjoy programming, you may implement your own ABCI server using the Tendermint Socket Protocol, known affectionately as Teaspoon. The first step is still to auto-generate the relevant data types and codec in your language using `protoc`. Messages coming over -the socket are Protobuf3 encoded, but additionally length-prefixed to -facilitate use as a streaming protocol. Protobuf3 doesn't have an +the socket are proto3 encoded, but additionally length-prefixed to +facilitate use as a streaming protocol. proto3 doesn't have an official length-prefix standard, so we use our own. The first byte in the prefix represents the length of the Big Endian encoded length. The remaining bytes in the prefix are the Big Endian encoded length. -For example, if the Protobuf3 encoded ABCI message is 0xDEADBEEF (4 -bytes), the length-prefixed message is 0x0104DEADBEEF. If the Protobuf3 +For example, if the proto3 encoded ABCI message is 0xDEADBEEF (4 +bytes), the length-prefixed message is 0x0104DEADBEEF. If the proto3 encoded ABCI message is 65535 bytes long, the length-prefixed message would be like 0x02FFFF.... @@ -125,12 +125,12 @@ received or a block is committed. It is unlikely that you will need to implement a client. For details of our client, see -[here](https://github.com/tendermint/abci/tree/master/client). +[here](https://github.com/tendermint/tendermint/tree/develop/abci/client). Most of the examples below are from [kvstore -application](https://github.com/tendermint/abci/blob/master/example/kvstore/kvstore.go), +application](https://github.com/tendermint/tendermint/blob/develop/abci/example/kvstore/kvstore.go), which is a part of the abci repo. [persistent_kvstore -application](https://github.com/tendermint/abci/blob/master/example/kvstore/persistent_kvstore.go) +application](https://github.com/tendermint/tendermint/blob/develop/abci/example/kvstore/persistent_kvstore.go) is used to show `BeginBlock`, `EndBlock` and `InitChain` example implementations. @@ -188,9 +188,9 @@ In Java: ResponseCheckTx requestCheckTx(RequestCheckTx req) { byte[] transaction = req.getTx().toByteArray(); - + // validate transaction - + if (notValid) { return ResponseCheckTx.newBuilder().setCode(CodeType.BadNonce).setLog("invalid tx").build(); } else { @@ -260,15 +260,15 @@ In Java: */ ResponseDeliverTx deliverTx(RequestDeliverTx request) { byte[] transaction = request.getTx().toByteArray(); - + // validate your transaction - + if (notValid) { return ResponseDeliverTx.newBuilder().setCode(CodeType.BadNonce).setLog("transaction was invalid").build(); } else { ResponseDeliverTx.newBuilder().setCode(CodeType.OK).build(); } - + } ### Commit @@ -302,10 +302,10 @@ In go: In Java: ResponseCommit requestCommit(RequestCommit requestCommit) { - + // update the internal app-state byte[] newAppState = calculateAppState(); - + // and return it to the node return ResponseCommit.newBuilder().setCode(CodeType.OK).setData(ByteString.copyFrom(newAppState)).build(); } @@ -326,7 +326,7 @@ In go: func (app *PersistentKVStoreApplication) BeginBlock(params types.RequestBeginBlock) { // update latest block info app.blockHeader = params.Header - + // reset valset changes app.changes = make([]*types.Validator, 0) } @@ -337,14 +337,14 @@ In Java: * all types come from protobuf definition */ ResponseBeginBlock requestBeginBlock(RequestBeginBlock req) { - + Header header = req.getHeader(); byte[] prevAppHash = header.getAppHash().toByteArray(); long prevHeight = header.getHeight(); long numTxs = header.getNumTxs(); - + // run your pre-block logic. Maybe prepare a state snapshot, message components, etc - + return ResponseBeginBlock.newBuilder().build(); } @@ -377,10 +377,10 @@ In Java: ResponseEndBlock requestEndBlock(RequestEndBlock req) { final long currentHeight = req.getHeight(); final byte[] validatorPubKey = getValPubKey(); - + ResponseEndBlock.Builder builder = ResponseEndBlock.newBuilder(); builder.addDiffs(1, Types.Validator.newBuilder().setPower(10L).setPubKey(ByteString.copyFrom(validatorPubKey)).build()); - + return builder.build(); } @@ -394,13 +394,13 @@ serialize each query as a single byte array. Additionally, certain instance about which peers to connect to. Tendermint Core currently uses the Query connection to filter peers upon -connecting, according to IP address or public key. For instance, +connecting, according to IP address or node ID. For instance, returning non-OK ABCI response to either of the following queries will cause Tendermint to not connect to the corresponding peer: -- `p2p/filter/addr/`, where `` is an IP address. -- `p2p/filter/pubkey/`, where `` is the hex-encoded - ED25519 key of the node (not it's validator key) +- `p2p/filter/addr/`, where `` is an IP address. +- `p2p/filter/id/`, where `` is the hex-encoded node ID (the hash of + the node's p2p pubkey). Note: these query formats are subject to change! @@ -437,25 +437,25 @@ In Java: ResponseQuery requestQuery(RequestQuery req) { final boolean isProveQuery = req.getProve(); final ResponseQuery.Builder responseBuilder = ResponseQuery.newBuilder(); - + if (isProveQuery) { com.app.example.ProofResult proofResult = generateProof(req.getData().toByteArray()); final byte[] proofAsByteArray = proofResult.getAsByteArray(); - + responseBuilder.setProof(ByteString.copyFrom(proofAsByteArray)); responseBuilder.setKey(req.getData()); responseBuilder.setValue(ByteString.copyFrom(proofResult.getData())); responseBuilder.setLog(result.getLogValue()); } else { byte[] queryData = req.getData().toByteArray(); - + final com.app.example.QueryResult result = generateQueryResult(queryData); - + responseBuilder.setIndex(result.getIndex()); responseBuilder.setValue(ByteString.copyFrom(result.getValue())); responseBuilder.setLog(result.getLogValue()); } - + return responseBuilder.build(); } @@ -515,13 +515,13 @@ In Java: ResponseInitChain requestInitChain(RequestInitChain req) { final int validatorsCount = req.getValidatorsCount(); final List validatorsList = req.getValidatorsList(); - + validatorsList.forEach((validator) -> { long power = validator.getPower(); byte[] validatorPubKey = validator.getPubKey().toByteArray(); - + // do somehing for validator setup in app }); - + return ResponseInitChain.newBuilder().build(); } diff --git a/docs/architecture/README.md b/docs/architecture/README.md index 9e41d3064..1cfc7ddce 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -1,5 +1,22 @@ -# Architecture Decision Records +# Architecture Decision Records (ADR) -This is a location to record all high-level architecture decisions in the tendermint project. Not the implementation details, but the reasoning that happened. This should be refered to for guidance of the "right way" to extend the application. And if we notice that the original decisions were lacking, we should have another open discussion, record the new decisions here, and then modify the code to match. +This is a location to record all high-level architecture decisions in the tendermint project. -Read up on the concept in this [blog post](https://product.reverb.com/documenting-architecture-decisions-the-reverb-way-a3563bb24bd0#.78xhdix6t). +You can read more about the ADR concept in this [blog post](https://product.reverb.com/documenting-architecture-decisions-the-reverb-way-a3563bb24bd0#.78xhdix6t). + +An ADR should provide: + +- Context on the relevant goals and the current state +- Proposed changes to achieve the goals +- Summary of pros and cons +- References +- Changelog + +Note the distinction between an ADR and a spec. The ADR provides the context, intuition, reasoning, and +justification for a change in architecture, or for the architecture of something +new. The spec is much more compressed and streamlined summary of everything as +it stands today. + +If recorded decisions turned out to be lacking, convene a discussion, record the new decisions here, and then modify the code to match. + +Note the context/background should be written in the present tense. diff --git a/docs/architecture/adr-009-ABCI-design.md b/docs/architecture/adr-009-ABCI-design.md new file mode 100644 index 000000000..8b85679b8 --- /dev/null +++ b/docs/architecture/adr-009-ABCI-design.md @@ -0,0 +1,273 @@ +# ADR 009: ABCI UX Improvements + +## Changelog + +23-06-2018: Some minor fixes from review +07-06-2018: Some updates based on discussion with Jae +07-06-2018: Initial draft to match what was released in ABCI v0.11 + +## Context + +The ABCI was first introduced in late 2015. It's purpose is to be: + +- a generic interface between state machines and their replication engines +- agnostic to the language the state machine is written in +- agnostic to the replication engine that drives it + +This means ABCI should provide an interface for both pluggable applications and +pluggable consensus engines. + +To achieve this, it uses Protocol Buffers (proto3) for message types. The dominant +implementation is in Go. + +After some recent discussions with the community on github, the following were +identified as pain points: + +- Amino encoded types +- Managing validator sets +- Imports in the protobuf file + +See the [references](#references) for more. + +### Imports + +The native proto library in Go generates inflexible and verbose code. +Many in the Go community have adopted a fork called +[gogoproto](https://github.com/gogo/protobuf) that provides a +variety of features aimed to improve the developer experience. +While `gogoproto` is nice, it creates an additional dependency, and compiling +the protobuf types for other languages has been reported to fail when `gogoproto` is used. + +### Amino + +Amino is an encoding protocol designed to improve over insufficiencies of protobuf. +It's goal is to be proto4. + +Many people are frustrated by incompatibility with protobuf, +and with the requirement for Amino to be used at all within ABCI. + +We intend to make Amino successful enough that we can eventually use it for ABCI +message types directly. By then it should be called proto4. In the meantime, +we want it to be easy to use. + +### PubKey + +PubKeys are encoded using Amino (and before that, go-wire). +Ideally, PubKeys are an interface type where we don't know all the +implementation types, so its unfitting to use `oneof` or `enum`. + +### Addresses + +The address for ED25519 pubkey is the RIPEMD160 of the Amino +encoded pubkey. This introduces an Amino dependency in the address generation, +a functionality that is widely required and should be easy to compute as +possible. + +### Validators + +To change the validator set, applications can return a list of validator updates +with ResponseEndBlock. In these updates, the public key *must* be included, +because Tendermint requires the public key to verify validator signatures. This +means ABCI developers have to work with PubKeys. That said, it would also be +convenient to work with address information, and for it to be simple to do so. + +### AbsentValidators + +Tendermint also provides a list of validators in BeginBlock who did not sign the +last block. This allows applications to reflect availability behaviour in the +application, for instance by punishing validators for not having votes included +in commits. + +### InitChain + +Tendermint passes in a list of validators here, and nothing else. It would +benefit the application to be able to control the initial validator set. For +instance the genesis file could include application-based information about the +initial validator set that the application could process to determine the +initial validator set. Additionally, InitChain would benefit from getting all +the genesis information. + +### Header + +ABCI provides the Header in RequestBeginBlock so the application can have +important information about the latest state of the blockchain. + +## Decision + +### Imports + +Move away from gogoproto. In the short term, we will just maintain a second +protobuf file without the gogoproto annotations. In the medium term, we will +make copies of all the structs in Golang and shuttle back and forth. In the long +term, we will use Amino. + +### Amino + +To simplify ABCI application development in the short term, +Amino will be completely removed from the ABCI: + +- It will not be required for PubKey encoding +- It will not be required for computing PubKey addresses + +That said, we are working to make Amino a huge success, and to become proto4. +To facilitate adoption and cross-language compatibility in the near-term, Amino +v1 will: + +- be fully compatible with the subset of proto3 that excludes `oneof` +- use the Amino prefix system to provide interface types, as opposed to `oneof` + style union types. + +That said, an Amino v2 will be worked on to improve the performance of the +format and its useability in cryptographic applications. + + +### PubKey + +Encoding schemes infect software. As a generic middleware, ABCI aims to have +some cross scheme compatibility. For this it has no choice but to include opaque +bytes from time to time. While we will not enforce Amino encoding for these +bytes yet, we need to provide a type system. The simplest way to do this is to +use a type string. + +PubKey will now look like: + +``` +message PubKey { + string type + bytes data +} +``` + +where `type` can be: + +- "ed225519", with `data = ` +- "secp256k1", with `data = <33-byte OpenSSL compressed pubkey>` + + +As we want to retain flexibility here, and since ideally, PubKey would be an +interface type, we do not use `enum` or `oneof`. + +### Addresses + +To simplify and improve computing addresses, we change it to the first 20-bytes of the SHA256 +of the raw 32-byte public key. + +We continue to use the Bitcoin address scheme for secp256k1 keys. + +### Validators + +Add a `bytes address` field: + +``` +message Validator { + bytes address + PubKey pub_key + int64 power +} +``` + +### RequestBeginBlock and AbsentValidators + +To simplify this, RequestBeginBlock will include the complete validator set, +including the address, and voting power of each validator, along +with a boolean for whether or not they voted: + +``` +message RequestBeginBlock { + bytes hash + Header header + LastCommitInfo last_commit_info + repeated Evidence byzantine_validators +} + +message LastCommitInfo { + int32 CommitRound + repeated SigningValidator validators +} + +message SigningValidator { + Validator validator + bool signed_last_block +} +``` + +Note that in Validators in RequestBeginBlock, we DO NOT include public keys. Public keys are +larger than addresses and in the future, with quantum computers, will be much +larger. The overhead of passing them, especially during fast-sync, is +significant. + +Additional, addresses are changing to be simpler to compute, further removing +the need to include pubkeys here. + +In short, ABCI developers must be aware of both addresses and public keys. + +### ResponseEndBlock + +Since ResponseEndBlock includes Validator, it must now include their address. + +### InitChain + +Change RequestInitChain to give the app all the information from the genesis file: + +``` +message RequestInitChain { + int64 time + string chain_id + ConsensusParams consensus_params + repeated Validator validators + bytes app_state_bytes +} +``` + +Change ResponseInitChain to allow the app to specify the initial validator set +and consensus parameters. + +``` +message ResponseInitChain { + ConsensusParams consensus_params + repeated Validator validators +} +``` + +### Header + +Now that Tendermint Amino will be compatible with proto3, the Header in ABCI +should exactly match the Tendermint header - they will then be encoded +identically in ABCI and in Tendermint Core. + +## Status + +Accepted. + +## Consequences + +### Positive + +- Easier for developers to build on the ABCI +- ABCI and Tendermint headers are identically serialized + +### Negative + +- Maintenance overhead of alternative type encoding scheme +- Performance overhead of passing all validator info every block (at least its + only addresses, and not also pubkeys) +- Maintenance overhead of duplicate types + +### Neutral + +- ABCI developers must know about validator addresses + +## References + +- [ABCI v0.10.3 Specification (before this + proposal)](https://github.com/tendermint/abci/blob/v0.10.3/specification.rst) +- [ABCI v0.11.0 Specification (implementing first draft of this + proposal)](https://github.com/tendermint/abci/blob/v0.11.0/specification.md) +- [Ed25519 addresses](https://github.com/tendermint/go-crypto/issues/103) +- [InitChain contains the + Genesis](https://github.com/tendermint/abci/issues/216) +- [PubKeys](https://github.com/tendermint/tendermint/issues/1524) +- [Notes on + Header](https://github.com/tendermint/tendermint/issues/1605) +- [Gogoproto issues](https://github.com/tendermint/abci/issues/256) +- [Absent Validators](https://github.com/tendermint/abci/issues/231) diff --git a/docs/architecture/adr-010-crypto-changes.md b/docs/architecture/adr-010-crypto-changes.md new file mode 100644 index 000000000..cfe618421 --- /dev/null +++ b/docs/architecture/adr-010-crypto-changes.md @@ -0,0 +1,78 @@ +# ADR 010: Crypto Changes + +## Context + +Tendermint is a cryptographic protocol that uses and composes a variety of cryptographic primitives. + +After nearly 4 years of development, Tendermint has recently undergone multiple security reviews to search for vulnerabilities and to assess the the use and composition of cryptographic primitives. + +### Hash Functions + +Tendermint uses RIPEMD160 universally as a hash function, most notably in its Merkle tree implementation. + +RIPEMD160 was chosen because it provides the shortest fingerprint that is long enough to be considered secure (ie. birthday bound of 80-bits). +It was also developed in the open academic community, unlike NSA-designed algorithms like SHA256. + +That said, the cryptographic community appears to unanimously agree on the security of SHA256. It has become a universal standard, especially now that SHA1 is broken, being required in TLS connections and having optimized support in hardware. + +### Merkle Trees + +Tendermint uses a simple Merkle tree to compute digests of large structures like transaction batches +and even blockchain headers. The Merkle tree length prefixes byte arrays before concatenating and hashing them. +It uses RIPEMD160. + +### Addresses + +ED25519 addresses are computed using the RIPEMD160 of the Amino encoding of the public key. +RIPEMD160 is generally considered an outdated hash function, and is much slower +than more modern functions like SHA256 or Blake2. + +### Authenticated Encryption + +Tendermint P2P connections use authenticated encryption to provide privacy and authentication in the communications. +This is done using the simple Station-to-Station protocol with the NaCL Ed25519 library. + +While there have been no vulnerabilities found in the implementation, there are some concerns: + +- NaCL uses Salsa20, a not-widely used and relatively out-dated stream cipher that has been obsoleted by ChaCha20 +- Connections use RIPEMD160 to compute a value that is used for the encryption nonce with subtle requirements on how it's used + +## Decision + +### Hash Functions + +Use the first 20-bytes of the SHA256 hash instead of RIPEMD160 for everything + +### Merkle Trees + +TODO + +### Addresses + +Compute ED25519 addresses as the first 20-bytes of the SHA256 of the raw 32-byte public key + +### Authenticated Encryption + +Make the following changes: + +- Use xChaCha20 instead of xSalsa20 - https://github.com/tendermint/tendermint/issues/1124 +- Use an HKDF instead of RIPEMD160 to compute nonces - https://github.com/tendermint/tendermint/issues/1165 + +## Status + +## Consequences + +### Positive + +- More modern and standard cryptographic functions with wider adoption and hardware acceleration + + +### Negative + +- Exact authenticated encryption construction isn't already provided in a well-used library + + +### Neutral + +## References + diff --git a/docs/architecture/adr-011-monitoring.md b/docs/architecture/adr-011-monitoring.md new file mode 100644 index 000000000..ca16a9a1c --- /dev/null +++ b/docs/architecture/adr-011-monitoring.md @@ -0,0 +1,116 @@ +# ADR 011: Monitoring + +## Changelog + +08-06-2018: Initial draft +11-06-2018: Reorg after @xla comments +13-06-2018: Clarification about usage of labels + +## Context + +In order to bring more visibility into Tendermint, we would like it to report +metrics and, maybe later, traces of transactions and RPC queries. See +https://github.com/tendermint/tendermint/issues/986. + +A few solutions were considered: + +1. [Prometheus](https://prometheus.io) + a) Prometheus API + b) [go-kit metrics package](https://github.com/go-kit/kit/tree/master/metrics) as an interface plus Prometheus + c) [telegraf](https://github.com/influxdata/telegraf) + d) new service, which will listen to events emitted by pubsub and report metrics +5. [OpenCensus](https://opencensus.io/go/index.html) + +### 1. Prometheus + +Prometheus seems to be the most popular product out there for monitoring. It has +a Go client library, powerful queries, alerts. + +**a) Prometheus API** + +We can commit to using Prometheus in Tendermint, but I think Tendermint users +should be free to choose whatever monitoring tool they feel will better suit +their needs (if they don't have existing one already). So we should try to +abstract interface enough so people can switch between Prometheus and other +similar tools. + +**b) go-kit metrics package as an interface** + +metrics package provides a set of uniform interfaces for service +instrumentation and offers adapters to popular metrics packages: + +https://godoc.org/github.com/go-kit/kit/metrics#pkg-subdirectories + +Comparing to Prometheus API, we're losing customisability and control, but gaining +freedom in choosing any instrument from the above list given we will extract +metrics creation into a separate function (see "providers" in node/node.go). + +**c) telegraf** + +Unlike already discussed options, telegraf does not require modifying Tendermint +source code. You create something called an input plugin, which polls +Tendermint RPC every second and calculates the metrics itself. + +While it may sound good, but some metrics we want to report are not exposed via +RPC or pubsub, therefore can't be accessed externally. + +**d) service, listening to pubsub** + +Same issue as the above. + +### 2. opencensus + +opencensus provides both metrics and tracing, which may be important in the +future. It's API looks different from go-kit and Prometheus, but looks like it +covers everything we need. + +Unfortunately, OpenCensus go client does not define any +interfaces, so if we want to abstract away metrics we +will need to write interfaces ourselves. + +### List of metrics + +| | Name | Type | Description | +| - | --------------------------------------- | ------- | ----------------------------------------------------------------------------- | +| A | consensus_height | Gauge | | +| A | consensus_validators | Gauge | Number of validators who signed | +| A | consensus_validators_power | Gauge | Total voting power of all validators | +| A | consensus_missing_validators | Gauge | Number of validators who did not sign | +| A | consensus_missing_validators_power | Gauge | Total voting power of the missing validators | +| A | consensus_byzantine_validators | Gauge | Number of validators who tried to double sign | +| A | consensus_byzantine_validators_power | Gauge | Total voting power of the byzantine validators | +| A | consensus_block_interval | Timing | Time between this and last block (Block.Header.Time) | +| | consensus_block_time | Timing | Time to create a block (from creating a proposal to commit) | +| | consensus_time_between_blocks | Timing | Time between committing last block and (receiving proposal creating proposal) | +| A | consensus_rounds | Gauge | Number of rounds | +| | consensus_prevotes | Gauge | | +| | consensus_precommits | Gauge | | +| | consensus_prevotes_total_power | Gauge | | +| | consensus_precommits_total_power | Gauge | | +| A | consensus_num_txs | Gauge | | +| A | mempool_size | Gauge | | +| A | consensus_total_txs | Gauge | | +| A | consensus_block_size | Gauge | In bytes | +| A | p2p_peers | Gauge | Number of peers node's connected to | + +`A` - will be implemented in the fist place. + +**Proposed solution** + +## Status + +Proposed. + +## Consequences + +### Positive + +Better visibility, support of variety of monitoring backends + +### Negative + +One more library to audit, messing metrics reporting code with business domain. + +### Neutral + +- diff --git a/docs/conf.py b/docs/conf.py index 7d52a4932..0cfc05cdf 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -197,10 +197,3 @@ copyfile('../DOCKER/README.md', tools_dir+'/docker.md') urllib.urlretrieve(tools_repo+tools_branch+'/tm-bench/README.md', filename=tools_dir+'/benchmarking.md') urllib.urlretrieve(tools_repo+tools_branch+'/tm-monitor/README.md', filename=tools_dir+'/monitoring.md') - -#### abci spec ################################# - -abci_repo = "https://raw.githubusercontent.com/tendermint/abci/" -abci_branch = "develop" - -urllib.urlretrieve(abci_repo+abci_branch+'/specification.md', filename='abci-spec.md') diff --git a/docs/deploy-testnets.md b/docs/deploy-testnets.md index 0c74b2c5e..e5b300836 100644 --- a/docs/deploy-testnets.md +++ b/docs/deploy-testnets.md @@ -35,7 +35,7 @@ Here are the steps to setting up a testnet manually: After a few seconds, all the nodes should connect to each other and start making blocks! For more information, see the Tendermint Networks -section of [the guide to using Tendermint](using-tendermint.html). +section of [the guide to using Tendermint](./using-tendermint.md). But wait! Steps 3 and 4 are quite manual. Instead, use [this script](https://github.com/tendermint/tendermint/blob/develop/docs/examples/init_testnet.sh), diff --git a/docs/determinism.md b/docs/determinism.md index 95958bb1f..2e23476c6 100644 --- a/docs/determinism.md +++ b/docs/determinism.md @@ -1,5 +1,3 @@ # On Determinism Arguably, the most difficult part of blockchain programming is determinism - that is, ensuring that sources of indeterminism do not creep into the design of such systems. - -See [this issue](https://github.com/tendermint/abci/issues/56) for more information on the potential sources of indeterminism. diff --git a/docs/ecosystem.md b/docs/ecosystem.md index 9fbc56f8e..6b7f833a8 100644 --- a/docs/ecosystem.md +++ b/docs/ecosystem.md @@ -12,7 +12,7 @@ to include your project. ## Other Tools -See [deploy testnets](./deploy-testnets.html) for information about all +See [deploy testnets](./deploy-testnets) for information about all the tools built by Tendermint. We have Kubernetes, Ansible, and Terraform integrations. diff --git a/docs/examples/init_testnet.sh b/docs/examples/init_testnet.sh index cd83751ec..84e54674c 100644 --- a/docs/examples/init_testnet.sh +++ b/docs/examples/init_testnet.sh @@ -30,7 +30,7 @@ echo "{ \"validators\": [ { \"pub_key\": { - \"type\": \"AC26791624DE60\", + \"type\": \"tendermint/PubKeyEd25519\", \"value\": $value0 }, \"power:\": 10, @@ -38,7 +38,7 @@ echo "{ }, { \"pub_key\": { - \"type\": \"AC26791624DE60\", + \"type\": \"tendermint/PubKeyEd25519\", \"value\": $value1 }, \"power:\": 10, @@ -46,7 +46,7 @@ echo "{ }, { \"pub_key\": { - \"type\": \"AC26791624DE60\", + \"type\": \"tendermint/PubKeyEd25519\", \"value\": $value2 }, \"power:\": 10, @@ -54,7 +54,7 @@ echo "{ }, { \"pub_key\": { - \"type\": \"AC26791624DE60\", + \"type\": \"tendermint/PubKeyEd25519\", \"value\": $value3 }, \"power:\": 10, @@ -66,4 +66,4 @@ echo "{ cp $file0 $file1 cp $file0 $file2 -cp $file2 $file3 \ No newline at end of file +cp $file2 $file3 diff --git a/docs/examples/node0/config/genesis.json b/docs/examples/node0/config/genesis.json index b9c12e31b..ac306bd2f 100644 --- a/docs/examples/node0/config/genesis.json +++ b/docs/examples/node0/config/genesis.json @@ -4,34 +4,34 @@ "validators": [ { "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "D+k4AdjnYPWbB9wmad137Bdpo/kAulOoTRQrLy/Qc4k=" }, - "power": 10, + "power": "10", "name": "" }, { "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "b56N5GCR1adcVRuENjfKw/mrm2dkhT7wNZXV/SDsKsU=" }, - "power": 10, + "power": "10", "name": "" }, { "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "IgZDpJvGA0TAamicA8ircy+RX/BkUlj6DXwM791ywIU=" }, - "power": 10, + "power": "10", "name": "" }, { "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "KGAZfxZvIZ7abbeIQ85U1ECG6+I62KSdaH8ulc0+OiU=" }, - "power": 10, + "power": "10", "name": "" } ], diff --git a/docs/examples/node0/config/node_key.json b/docs/examples/node0/config/node_key.json index f4cdd093f..dec1fd42d 100644 --- a/docs/examples/node0/config/node_key.json +++ b/docs/examples/node0/config/node_key.json @@ -1 +1 @@ -{"priv_key":{"type":"954568A3288910","value":"7lY+k6EDllG8Q9gVbF5313t/ag2YGkBVKdVa0YHJ9xO5k0w3Q/hke0Z7UFT1KgVDGRUEKzwAwwjwFQUvgF0ZWg=="}} \ No newline at end of file +{"priv_key":{"type":"tendermint/PrivKeyEd25519","value":"7lY+k6EDllG8Q9gVbF5313t/ag2YGkBVKdVa0YHJ9xO5k0w3Q/hke0Z7UFT1KgVDGRUEKzwAwwjwFQUvgF0ZWg=="}} diff --git a/docs/examples/node0/config/priv_validator.json b/docs/examples/node0/config/priv_validator.json index e758b75b8..ef2daccbb 100644 --- a/docs/examples/node0/config/priv_validator.json +++ b/docs/examples/node0/config/priv_validator.json @@ -1,14 +1,14 @@ { "address": "122A9414774A2FCAD026201DA477EF3F41970EF0", "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "D+k4AdjnYPWbB9wmad137Bdpo/kAulOoTRQrLy/Qc4k=" }, - "last_height": 0, - "last_round": 0, + "last_height": "0", + "last_round": "0", "last_step": 0, "priv_key": { - "type": "954568A3288910", + "type": "tendermint/PrivKeyEd25519", "value": "YLxp3ho+kySgAnzjBptbxDzSGw2ntGZLsIHQsaVxY/cP6TgB2Odg9ZsH3CZp3XfsF2mj+QC6U6hNFCsvL9BziQ==" } -} \ No newline at end of file +} diff --git a/docs/examples/node1/config/genesis.json b/docs/examples/node1/config/genesis.json index b9c12e31b..ac306bd2f 100644 --- a/docs/examples/node1/config/genesis.json +++ b/docs/examples/node1/config/genesis.json @@ -4,34 +4,34 @@ "validators": [ { "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "D+k4AdjnYPWbB9wmad137Bdpo/kAulOoTRQrLy/Qc4k=" }, - "power": 10, + "power": "10", "name": "" }, { "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "b56N5GCR1adcVRuENjfKw/mrm2dkhT7wNZXV/SDsKsU=" }, - "power": 10, + "power": "10", "name": "" }, { "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "IgZDpJvGA0TAamicA8ircy+RX/BkUlj6DXwM791ywIU=" }, - "power": 10, + "power": "10", "name": "" }, { "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "KGAZfxZvIZ7abbeIQ85U1ECG6+I62KSdaH8ulc0+OiU=" }, - "power": 10, + "power": "10", "name": "" } ], diff --git a/docs/examples/node1/config/node_key.json b/docs/examples/node1/config/node_key.json index 374efe63b..05f4ea846 100644 --- a/docs/examples/node1/config/node_key.json +++ b/docs/examples/node1/config/node_key.json @@ -1 +1 @@ -{"priv_key":{"type":"954568A3288910","value":"H71dc/TIG7nTselfa9nG0WRArXLKYnm7P5eFCk2lk8ASKQ3sIHpbdxCSHQD/RcdHe7TiabJeuOssNPvPWiyQEQ=="}} \ No newline at end of file +{"priv_key":{"type":"tendermint/PrivKeyEd25519","value":"H71dc/TIG7nTselfa9nG0WRArXLKYnm7P5eFCk2lk8ASKQ3sIHpbdxCSHQD/RcdHe7TiabJeuOssNPvPWiyQEQ=="}} diff --git a/docs/examples/node1/config/priv_validator.json b/docs/examples/node1/config/priv_validator.json index caf3dbc59..3a38793e4 100644 --- a/docs/examples/node1/config/priv_validator.json +++ b/docs/examples/node1/config/priv_validator.json @@ -1,14 +1,14 @@ { "address": "BEA1B57F5806CF9AC4D54C8CF806DED5C0F102E1", "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "b56N5GCR1adcVRuENjfKw/mrm2dkhT7wNZXV/SDsKsU=" }, - "last_height": 0, - "last_round": 0, + "last_height": "0", + "last_round": "0", "last_step": 0, "priv_key": { - "type": "954568A3288910", + "type": "tendermint/PrivKeyEd25519", "value": "o0IqrHSPtd5YqGefodWxpJuRzvuVBjgbH785vbMgk7Vvno3kYJHVp1xVG4Q2N8rD+aubZ2SFPvA1ldX9IOwqxQ==" } -} \ No newline at end of file +} diff --git a/docs/examples/node2/config/genesis.json b/docs/examples/node2/config/genesis.json index b9c12e31b..ac306bd2f 100644 --- a/docs/examples/node2/config/genesis.json +++ b/docs/examples/node2/config/genesis.json @@ -4,34 +4,34 @@ "validators": [ { "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "D+k4AdjnYPWbB9wmad137Bdpo/kAulOoTRQrLy/Qc4k=" }, - "power": 10, + "power": "10", "name": "" }, { "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "b56N5GCR1adcVRuENjfKw/mrm2dkhT7wNZXV/SDsKsU=" }, - "power": 10, + "power": "10", "name": "" }, { "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "IgZDpJvGA0TAamicA8ircy+RX/BkUlj6DXwM791ywIU=" }, - "power": 10, + "power": "10", "name": "" }, { "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "KGAZfxZvIZ7abbeIQ85U1ECG6+I62KSdaH8ulc0+OiU=" }, - "power": 10, + "power": "10", "name": "" } ], diff --git a/docs/examples/node2/config/node_key.json b/docs/examples/node2/config/node_key.json index 52a978bb0..6f15a541b 100644 --- a/docs/examples/node2/config/node_key.json +++ b/docs/examples/node2/config/node_key.json @@ -1 +1 @@ -{"priv_key":{"type":"954568A3288910","value":"COHZ/Y2cWGWxJNkRwtpQBt5sYvOnb6Gpz0lO46XERRJFBIdSWD5x1UMGRSTmnvW1ec5G4bMdg6zUZKOZD+vVPg=="}} \ No newline at end of file +{"priv_key":{"type":"tendermint/PrivKeyEd25519","value":"COHZ/Y2cWGWxJNkRwtpQBt5sYvOnb6Gpz0lO46XERRJFBIdSWD5x1UMGRSTmnvW1ec5G4bMdg6zUZKOZD+vVPg=="}} diff --git a/docs/examples/node2/config/priv_validator.json b/docs/examples/node2/config/priv_validator.json index 65fa30484..2bcd31a76 100644 --- a/docs/examples/node2/config/priv_validator.json +++ b/docs/examples/node2/config/priv_validator.json @@ -1,14 +1,14 @@ { "address": "F0AA266949FB29ADA0B679C27889ED930BD1BDA1", "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "IgZDpJvGA0TAamicA8ircy+RX/BkUlj6DXwM791ywIU=" }, - "last_height": 0, - "last_round": 0, + "last_height": "0", + "last_round": "0", "last_step": 0, "priv_key": { - "type": "954568A3288910", + "type": "tendermint/PrivKeyEd25519", "value": "khADeZ5K/8u/L99DFaZNRq8V5g+EHWbwfqFjhCrppaAiBkOkm8YDRMBqaJwDyKtzL5Ff8GRSWPoNfAzv3XLAhQ==" } -} \ No newline at end of file +} diff --git a/docs/examples/node3/config/genesis.json b/docs/examples/node3/config/genesis.json index b9c12e31b..ac306bd2f 100644 --- a/docs/examples/node3/config/genesis.json +++ b/docs/examples/node3/config/genesis.json @@ -4,34 +4,34 @@ "validators": [ { "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "D+k4AdjnYPWbB9wmad137Bdpo/kAulOoTRQrLy/Qc4k=" }, - "power": 10, + "power": "10", "name": "" }, { "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "b56N5GCR1adcVRuENjfKw/mrm2dkhT7wNZXV/SDsKsU=" }, - "power": 10, + "power": "10", "name": "" }, { "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "IgZDpJvGA0TAamicA8ircy+RX/BkUlj6DXwM791ywIU=" }, - "power": 10, + "power": "10", "name": "" }, { "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "KGAZfxZvIZ7abbeIQ85U1ECG6+I62KSdaH8ulc0+OiU=" }, - "power": 10, + "power": "10", "name": "" } ], diff --git a/docs/examples/node3/config/node_key.json b/docs/examples/node3/config/node_key.json index bde4e0ede..01a14a3bf 100644 --- a/docs/examples/node3/config/node_key.json +++ b/docs/examples/node3/config/node_key.json @@ -1 +1 @@ -{"priv_key":{"type":"954568A3288910","value":"9Y9xp/tUJJ6pHTF5SUV0bGKYSdVbFtMHu+Lr8S0JBSZAwneaejnfOEU1LMKOnQ07skrDUaJcj5di3jAyjxJzqg=="}} \ No newline at end of file +{"priv_key":{"type":"tendermint/PrivKeyEd25519","value":"9Y9xp/tUJJ6pHTF5SUV0bGKYSdVbFtMHu+Lr8S0JBSZAwneaejnfOEU1LMKOnQ07skrDUaJcj5di3jAyjxJzqg=="}} diff --git a/docs/examples/node3/config/priv_validator.json b/docs/examples/node3/config/priv_validator.json index 1d985a00b..bc8a66513 100644 --- a/docs/examples/node3/config/priv_validator.json +++ b/docs/examples/node3/config/priv_validator.json @@ -1,14 +1,14 @@ { "address": "9A1A6914EB5F4FF0269C7EEEE627C27310CC64F9", "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "KGAZfxZvIZ7abbeIQ85U1ECG6+I62KSdaH8ulc0+OiU=" }, - "last_height": 0, - "last_round": 0, + "last_height": "0", + "last_round": "0", "last_step": 0, "priv_key": { - "type": "954568A3288910", + "type": "tendermint/PrivKeyEd25519", "value": "jb52LZ5gp+eQ8nJlFK1z06nBMp1gD8ICmyzdM1icGOgoYBl/Fm8hntptt4hDzlTUQIbr4jrYpJ1ofy6VzT46JQ==" } -} \ No newline at end of file +} diff --git a/docs/getting-started.md b/docs/getting-started.md index 9767dae5a..1fa1405f2 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -7,7 +7,7 @@ application you want to run. So, to run a complete blockchain that does something useful, you must start two programs: one is Tendermint Core, the other is your application, which can be written in any programming language. Recall from [the intro to -ABCI](introduction.html#ABCI-Overview) that Tendermint Core handles all +ABCI](./introduction.md#ABCI-Overview) that Tendermint Core handles all the p2p and consensus stuff, and just forwards transactions to the application when they need to be validated, or when they're ready to be committed to a block. @@ -25,15 +25,11 @@ more info. Then run - go get -u github.com/tendermint/abci/cmd/abci-cli - -If there is an error, install and run the -[dep](https://github.com/golang/dep) tool to pin the dependencies: - - cd $GOPATH/src/github.com/tendermint/abci + go get github.com/tendermint/tendermint + cd $GOPATH/src/github.com/tendermint/tendermint make get_tools make get_vendor_deps - make install + make install_abci Now you should have the `abci-cli` installed; you'll see a couple of commands (`counter` and `kvstore`) that are example applications written @@ -62,7 +58,7 @@ Tendermint before, use: If you have used Tendermint, you may want to reset the data for a new blockchain by running `tendermint unsafe_reset_all`. Then you can run `tendermint node` to start Tendermint, and connect to the app. For more -details, see [the guide on using Tendermint](./using-tendermint.html). +details, see [the guide on using Tendermint](./using-tendermint.md). You should see Tendermint making blocks! We can get the status of our Tendermint node as follows: @@ -129,10 +125,10 @@ The result should look like: Note the `value` in the result (`YWJjZA==`); this is the base64-encoding of the ASCII of `abcd`. You can verify this in a python 2 shell by -running `"61626364".decode('base64')` or in python 3 shell by running -`import codecs; codecs.decode("61626364", 'base64').decode('ascii')`. +running `"YWJjZA==".decode('base64')` or in python 3 shell by running +`import codecs; codecs.decode("YWJjZA==", 'base64').decode('ascii')`. Stay tuned for a future release that [makes this output more -human-readable](https://github.com/tendermint/abci/issues/32). +human-readable](https://github.com/tendermint/tendermint/issues/1794). Now let's try setting a different key and value: @@ -228,7 +224,7 @@ But if we send a `1`, it works again: } For more details on the `broadcast_tx` API, see [the guide on using -Tendermint](./using-tendermint.html). +Tendermint](./using-tendermint.md). ## CounterJS - Example in Another Language @@ -247,7 +243,7 @@ devs, we keep all our code under the `$GOPATH`, so run: Kill the previous `counter` and `tendermint` processes. Now run the app: - node example/app.js + node example/counter.js In another window, reset and start `tendermint`: diff --git a/docs/how-to-read-logs.md b/docs/how-to-read-logs.md index 92f563cff..2b019e549 100644 --- a/docs/how-to-read-logs.md +++ b/docs/how-to-read-logs.md @@ -53,8 +53,8 @@ Next follows a standard block creation cycle, where we enter a new round, propose a block, receive more than 2/3 of prevotes, then precommits and finally have a chance to commit a block. For details, please refer to [Consensus -Overview](introduction.html#consensus-overview) or [Byzantine Consensus -Algorithm](specification.html). +Overview](./introduction.md#consensus-overview) or [Byzantine Consensus +Algorithm](./spec/consensus). I[10-04|13:54:30.393] enterNewRound(91/0). Current: 91/0/RoundStepNewHeight module=consensus I[10-04|13:54:30.393] enterPropose(91/0). Current: 91/0/RoundStepNewRound module=consensus @@ -100,10 +100,10 @@ Algorithm](specification.html). Here is the list of modules you may encounter in Tendermint's log and a little overview what they do. -- `abci-client` As mentioned in [Application Development Guide](app-development.md#abci-design), Tendermint acts as an ABCI +- `abci-client` As mentioned in [Application Development Guide](./app-development.md), Tendermint acts as an ABCI client with respect to the application and maintains 3 connections: mempool, consensus and query. The code used by Tendermint Core can - be found [here](https://github.com/tendermint/abci/tree/master/client). + be found [here](https://github.com/tendermint/tendermint/tree/develop/abci/client). - `blockchain` Provides storage, pool (a group of peers), and reactor for both storing and exchanging blocks between peers. - `consensus` The heart of Tendermint core, which is the @@ -115,13 +115,13 @@ little overview what they do. found [here](https://github.com/tendermint/tendermint/blob/master/types/events.go). You can subscribe to them by calling `subscribe` RPC method. Refer - to [RPC docs](specification/rpc.html) for additional information. + to [RPC docs](./specification/rpc.md) for additional information. - `mempool` Mempool module handles all incoming transactions, whenever they are coming from peers or the application. - `p2p` Provides an abstraction around peer-to-peer communication. For more details, please check out the [README](https://github.com/tendermint/tendermint/blob/master/p2p/README.md). -- `rpc` [Tendermint's RPC](specification/rpc.html). +- `rpc` [Tendermint's RPC](./specification/rpc.md). - `rpc-server` RPC server. For implementation details, please read the [README](https://github.com/tendermint/tendermint/blob/master/rpc/lib/README.md). - `state` Represents the latest state and execution submodule, which diff --git a/docs/index.rst b/docs/index.rst index b40db3bf4..b3158c219 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -6,67 +6,10 @@ Welcome to Tendermint! ====================== +This location for our documentation has been deprecated, please see: -.. image:: assets/tmint-logo-blue.png - :height: 200px - :width: 200px - :align: center +- https://tendermint.com/docs -Introduction ------------- +The last version built by Read The Docs will still be available at: -.. toctree:: - :maxdepth: 1 - - introduction.md - install.md - getting-started.md - using-tendermint.md - deploy-testnets.md - ecosystem.md - -Tendermint Tools ----------------- - -.. the tools/ files are pulled in from the tools repo -.. see the bottom of conf.py -.. toctree:: - :maxdepth: 1 - - tools/docker.md - terraform-and-ansible.md - tools/benchmarking.md - tools/monitoring.md - -ABCI, Apps, Logging, Etc ------------------------- - -.. toctree:: - :maxdepth: 1 - - abci-cli.md - abci-spec.md - app-architecture.md - app-development.md - subscribing-to-events-via-websocket.md - indexing-transactions.md - how-to-read-logs.md - running-in-production.md - -Research & Specification ------------------------- - -.. toctree:: - :maxdepth: 1 - - determinism.md - transactional-semantics.md - -.. specification.md ## keep this file for legacy purpose. needs to be fixed though - -* For a deeper dive, see `this thesis `__. -* There is also the `original whitepaper `__, though it is now quite outdated. -* Readers might also be interested in the `Cosmos Whitepaper `__ which describes Tendermint, ABCI, and how to build a scalable, heterogeneous, cryptocurrency network. -* For example applications and related software built by the Tendermint team and other, see the `software ecosystem `__. - -Join the `community `__ to ask questions and discuss projects. +- https://tendermint.readthedocs.io/projects/tools/en/v0.21.0/ diff --git a/docs/install.md b/docs/install.md new file mode 100644 index 000000000..ff101715b --- /dev/null +++ b/docs/install.md @@ -0,0 +1,76 @@ +# Install Tendermint + +The fastest and easiest way to install the `tendermint` binary +is to run [this script](https://github.com/tendermint/tendermint/blob/develop/scripts/install/install_tendermint_ubuntu.sh) on +a fresh Ubuntu instance, +or [this script](https://github.com/tendermint/tendermint/blob/develop/scripts/install/install_tendermint_bsd.sh) +on a fresh FreeBSD instance. Read the comments / instructions carefully (i.e., reset your terminal after running the script, +make sure your okay with the network connections being made). + +## From Binary + +To download pre-built binaries, see the [releases page](https://github.com/tendermint/tendermint/releases). + +## From Source + +You'll need `go` [installed](https://golang.org/doc/install) and the required +[environment variables set](https://github.com/tendermint/tendermint/wiki/Setting-GOPATH) + +### Get Source Code + +``` +mkdir -p $GOPATH/src/github.com/tendermint +cd $GOPATH/src/github.com/tendermint +git clone https://github.com/tendermint/tendermint.git +cd tendermint +``` + +### Get Tools & Dependencies + +``` +make get_tools +make get_vendor_deps +``` + +### Compile + +``` +make install +``` + +to put the binary in `$GOPATH/bin` or use: + +``` +make build +``` + +to put the binary in `./build`. + +The latest `tendermint version` is now installed. + +## Reinstall + +If you already have Tendermint installed, and you make updates, simply + +``` +cd $GOPATH/src/github.com/tendermint/tendermint +make install +``` + +To upgrade, run + +``` +cd $GOPATH/src/github.com/tendermint/tendermint +git pull origin master +make get_vendor_deps +make install +``` + +## Run + +To start a one-node blockchain with a simple in-process application: + +``` +tendermint init +tendermint node --proxy_app=kvstore +``` diff --git a/docs/install.rst b/docs/install.rst deleted file mode 100644 index 7ae87eae7..000000000 --- a/docs/install.rst +++ /dev/null @@ -1,99 +0,0 @@ -Install Tendermint -================== - -The fastest and easiest way to install the ``tendermint`` binary -is to run `this script `__ on -a fresh Ubuntu instance, -or `this script `__ -on a fresh FreeBSD instance. Read the comments / instructions carefully (i.e., reset your terminal after running the script, -make sure your okay with the network connections being made). - -From Binary ------------ - -To download pre-built binaries, see the `releases page `__. - -From Source ------------ - -You'll need ``go`` `installed `__ and the required -`environment variables set `__ - -Get Source Code -^^^^^^^^^^^^^^^ - -:: - - mkdir -p $GOPATH/src/github.com/tendermint - cd $GOPATH/src/github.com/tendermint - git clone https://github.com/tendermint/tendermint.git - cd tendermint - -Get Tools & Dependencies -^^^^^^^^^^^^^^^^^^^^^^^^ - -:: - - make get_tools - make get_vendor_deps - -Compile -^^^^^^^ - -:: - - make install - -to put the binary in ``$GOPATH/bin`` or use: - -:: - - make build - -to put the binary in ``./build``. - -The latest ``tendermint version`` is now installed. - -Reinstall ---------- - -If you already have Tendermint installed, and you make updates, simply - -:: - - cd $GOPATH/src/github.com/tendermint/tendermint - make install - -To upgrade, there are a few options: - -- set a new ``$GOPATH`` and run - ``go get github.com/tendermint/tendermint/cmd/tendermint``. This - makes a fresh copy of everything for the new version. -- run ``go get -u github.com/tendermint/tendermint/cmd/tendermint``, - where the ``-u`` fetches the latest updates for the repository and - its dependencies -- fetch and checkout the latest master branch in - ``$GOPATH/src/github.com/tendermint/tendermint``, and then run - ``make get_vendor_deps && make install`` as above. - -Note the first two options should usually work, but may fail. If they -do, use ``dep``, as above: - -:: - - cd $GOPATH/src/github.com/tendermint/tendermint - make get_vendor_deps - make install - -Since the third option just uses ``dep`` right away, it should always -work. - -Run -^^^ - -To start a one-node blockchain with a simple in-process application: - -:: - - tendermint init - tendermint node --proxy_app=kvstore diff --git a/docs/introduction.md b/docs/introduction.md index 419071dca..4503e195e 100644 --- a/docs/introduction.md +++ b/docs/introduction.md @@ -119,9 +119,9 @@ consensus engine, and provides a particular application state. ## ABCI Overview The [Application BlockChain Interface -(ABCI)](https://github.com/tendermint/abci) allows for Byzantine Fault -Tolerant replication of applications written in any programming -language. +(ABCI)](https://github.com/tendermint/tendermint/tree/develop/abci) +allows for Byzantine Fault Tolerant replication of applications +written in any programming language. ### Motivation @@ -159,7 +159,7 @@ Teaspoon). [Tendermint Core](https://github.com/tendermint/tendermint) (the "consensus engine") communicates with the application via a socket -protocol that satisfies the [ABCI](https://github.com/tendermint/abci). +protocol that satisfies the ABCI. To draw an analogy, lets talk about a well-known cryptocurrency, Bitcoin. Bitcoin is a cryptocurrency blockchain where each node @@ -187,7 +187,7 @@ core to the application. The application replies with corresponding response messages. The messages are specified here: [ABCI Message -Types](https://github.com/tendermint/abci#message-types). +Types](https://github.com/tendermint/tendermint/blob/develop/abci/README.md#message-types). The **DeliverTx** message is the work horse of the application. Each transaction in the blockchain is delivered with this message. The diff --git a/docs/metrics.md b/docs/metrics.md new file mode 100644 index 000000000..b469c6890 --- /dev/null +++ b/docs/metrics.md @@ -0,0 +1,42 @@ +# Metrics + +Tendermint can report and serve the Prometheus metrics, which in their turn can +be consumed by Prometheus collector(s). + +This functionality is disabled by default. + +To enable the Prometheus metrics, set `instrumentation.prometheus=true` if your +config file. Metrics will be served under `/metrics` on 26660 port by default. +Listen address can be changed in the config file (see +`instrumentation.prometheus_listen_addr`). + +## List of available metrics + +The following metrics are available: + +``` +| Name | Type | Since | Description | +| --------------------------------------- | ------- | --------- | ----------------------------------------------------------------------------- | +| consensus_height | Gauge | 0.21.0 | Height of the chain | +| consensus_validators | Gauge | 0.21.0 | Number of validators | +| consensus_validators_power | Gauge | 0.21.0 | Total voting power of all validators | +| consensus_missing_validators | Gauge | 0.21.0 | Number of validators who did not sign | +| consensus_missing_validators_power | Gauge | 0.21.0 | Total voting power of the missing validators | +| consensus_byzantine_validators | Gauge | 0.21.0 | Number of validators who tried to double sign | +| consensus_byzantine_validators_power | Gauge | 0.21.0 | Total voting power of the byzantine validators | +| consensus_block_interval_seconds | Histogram | 0.21.0 | Time between this and last block (Block.Header.Time) in seconds | +| consensus_rounds | Gauge | 0.21.0 | Number of rounds | +| consensus_num_txs | Gauge | 0.21.0 | Number of transactions | +| mempool_size | Gauge | 0.21.0 | Number of uncommitted transactions | +| consensus_total_txs | Gauge | 0.21.0 | Total number of transactions committed | +| consensus_block_size_bytes | Gauge | 0.21.0 | Block size in bytes | +| p2p_peers | Gauge | 0.21.0 | Number of peers node's connected to | +``` + +## Useful queries + +Percentage of missing + byzantine validators: + +``` +((consensus_byzantine_validators_power + consensus_missing_validators_power) / consensus_validators_power) * 100 +``` diff --git a/docs/running-in-production.md b/docs/running-in-production.md index 7e6068d4b..225a97853 100644 --- a/docs/running-in-production.md +++ b/docs/running-in-production.md @@ -78,6 +78,9 @@ We have a small tool, called `tm-monitor`, which outputs information from the endpoints above plus some statistics. The tool can be found [here](https://github.com/tendermint/tools/tree/master/tm-monitor). +Tendermint also can report and serve Prometheus metrics. See +[Metrics](./metrics.md). + ## What happens when my app dies? You are supposed to run Tendermint under a [process @@ -204,3 +207,37 @@ ranges](https://github.com/tendermint/tendermint/blob/27bd1deabe4ba6a2d9b463b8f3 This may not be the case for private networks, where your IP range is usually strictly limited and private. If that case, you need to set `addr_book_strict` to `false` (turn off). + +- `rpc.max_open_connections` + +By default, the number of simultaneous connections is limited because most OS +give you limited number of file descriptors. + +If you want to accept greater number of connections, you will need to increase +these limits. + +[Sysctls to tune the system to be able to open more connections](https://github.com/satori-com/tcpkali/blob/master/doc/tcpkali.man.md#sysctls-to-tune-the-system-to-be-able-to-open-more-connections) + +...for N connections, such as 50k: + +``` +kern.maxfiles=10000+2*N # BSD +kern.maxfilesperproc=100+2*N # BSD +kern.ipc.maxsockets=10000+2*N # BSD +fs.file-max=10000+2*N # Linux +net.ipv4.tcp_max_orphans=N # Linux + +# For load-generating clients. +net.ipv4.ip_local_port_range="10000 65535" # Linux. +net.inet.ip.portrange.first=10000 # BSD/Mac. +net.inet.ip.portrange.last=65535 # (Enough for N < 55535) +net.ipv4.tcp_tw_reuse=1 # Linux +net.inet.tcp.maxtcptw=2*N # BSD + +# If using netfilter on Linux: +net.netfilter.nf_conntrack_max=N +echo $((N/8)) > /sys/module/nf_conntrack/parameters/hashsize +``` + +The similar option exists for limiting the number of gRPC connections - +`rpc.grpc_max_open_connections`. diff --git a/docs/spec/blockchain/encoding.md b/docs/spec/blockchain/encoding.md index aa1842107..16902d099 100644 --- a/docs/spec/blockchain/encoding.md +++ b/docs/spec/blockchain/encoding.md @@ -2,8 +2,8 @@ ## Amino -Tendermint uses the Protobuf3 derivative [Amino](https://github.com/tendermint/go-amino) for all data structures. -Think of Amino as an object-oriented Protobuf3 with native JSON support. +Tendermint uses the proto3 derivative [Amino](https://github.com/tendermint/go-amino) for all data structures. +Think of Amino as an object-oriented proto3 with native JSON support. The goal of the Amino encoding protocol is to bring parity between application logic objects and persistence objects. @@ -21,7 +21,7 @@ arbitrary object and return the Amino encoded bytes. ## Byte Arrays The encoding of a byte array is simply the raw-bytes prefixed with the length of -the array as a `UVarint` (what Protobuf calls a `Varint`). +the array as a `UVarint` (what proto calls a `Varint`). For details on varints, see the [protobuf spec](https://developers.google.com/protocol-buffers/docs/encoding#varints). @@ -54,17 +54,15 @@ familiar with amino encoding. You can simply use below table and concatenate Prefix || Length (of raw bytes) || raw bytes ( while || stands for byte concatenation here). -| Type | Name | Prefix | Length | -| ---- | ---- | ------ | ----- | -| PubKeyEd25519 | tendermint/PubKeyEd25519 | 0x1624DE62 | 0x20 | -| PubKeyLedgerEd25519 | tendermint/PubKeyLedgerEd25519 | 0x5C3453B2 | 0x20 | -| PubKeySecp256k1 | tendermint/PubKeySecp256k1 | 0xEB5AE982 | 0x21 | -| PrivKeyEd25519 | tendermint/PrivKeyEd25519 | 0xA3288912 | 0x40 | -| PrivKeySecp256k1 | tendermint/PrivKeySecp256k1 | 0xE1B0F79A | 0x20 | -| PrivKeyLedgerSecp256k1 | tendermint/PrivKeyLedgerSecp256k1 | 0x10CAB393 | variable | -| PrivKeyLedgerEd25519 | tendermint/PrivKeyLedgerEd25519 | 0x0CFEEF9B | variable | -| SignatureEd25519 | tendermint/SignatureKeyEd25519 | 0x3DA1DB2A | 0x40 | -| SignatureSecp256k1 | tendermint/SignatureKeySecp256k1 | 0x16E1FEEA | variable | +| Type | Name | Prefix | Length | Notes | +| ---- | ---- | ------ | ----- | ------ | +| PubKeyEd25519 | tendermint/PubKeyEd25519 | 0x1624DE64 | 0x20 | | +| PubKeySecp256k1 | tendermint/PubKeySecp256k1 | 0xEB5AE987 | 0x21 | | +| PrivKeyEd25519 | tendermint/PrivKeyEd25519 | 0xA3288910 | 0x40 | | +| PrivKeySecp256k1 | tendermint/PrivKeySecp256k1 | 0xE1B0F79B | 0x20 | | +| SignatureEd25519 | tendermint/SignatureEd25519 | 0x2031EA53 | 0x40 | | +| SignatureSecp256k1 | tendermint/SignatureSecp256k1 | 0x7FC4A495 | variable | +| ### Examples @@ -84,14 +82,13 @@ Addresses for each public key types are computed as follows: #### Ed25519 -RIPEMD160 hash of the Amino encoded public key: +First 20-bytes of the SHA256 hash of the raw 32-byte public key: ``` -address = RIPEMD160(AMINO(pubkey)) +address = SHA256(pubkey)[:20] ``` -NOTE: this will soon change to the truncated 20-bytes of the SHA256 of the raw -public key +NOTE: before v0.22.0, this was the RIPEMD160 of the Amino encoded public key. #### Secp256k1 @@ -154,7 +151,15 @@ func MakeParts(obj interface{}, partSize int) []Part Simple Merkle trees are used in numerous places in Tendermint to compute a cryptographic digest of a data structure. -RIPEMD160 is always used as the hashing function. +Tendermint always uses the `TMHASH` hash function, which is the first 20-bytes +of the SHA256: + +``` +func TMHASH(bz []byte) []byte { + shasum := SHA256(bz) + return shasum[:20] +} +``` ### Simple Merkle Root @@ -177,7 +182,7 @@ func SimpleMerkleRoot(hashes [][]byte) []byte{ func SimpleConcatHash(left, right []byte) []byte{ left = encodeByteSlice(left) right = encodeByteSlice(right) - return RIPEMD160 (append(left, right)) + return TMHASH(append(left, right)) } ``` @@ -185,8 +190,8 @@ Note that the leaves are Amino encoded as byte-arrays (ie. simple Uvarint length prefix) before being concatenated together and hashed. Note: we will abuse notion and invoke `SimpleMerkleRoot` with arguments of type `struct` or type `[]struct`. -For `struct` arguments, we compute a `[][]byte` by sorting elements of the `struct` according to -field name and then hashing them. +For `struct` arguments, we compute a `[][]byte` containing the hash of each +field in the struct sorted by the hash of the field name. For `[]struct` arguments, we compute a `[][]byte` by hashing the individual `struct` elements. ### Simple Merkle Proof @@ -249,7 +254,7 @@ For instance, an ED25519 PubKey would look like: ``` { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "uZ4h63OFWuQ36ZZ4Bd6NF+/w9fWUwrOncrQsackrsTk=" } ``` diff --git a/docs/spec/p2p/peer.md b/docs/spec/p2p/peer.md index 3cda0c8a4..69c5bbacf 100644 --- a/docs/spec/p2p/peer.md +++ b/docs/spec/p2p/peer.md @@ -7,7 +7,7 @@ For details on peer discovery, see the [peer exchange (PEX) reactor doc](https:/ ## Peer Identity Tendermint peers are expected to maintain long-term persistent identities in the form of a public key. -Each peer has an ID defined as `peer.ID == peer.PubKey.Address()`, where `Address` uses the scheme defined in go-crypto. +Each peer has an ID defined as `peer.ID == peer.PubKey.Address()`, where `Address` uses the scheme defined in `crypto` package. A single peer ID can have multiple IP addresses associated with it, but a node will only ever connect to one at a time. diff --git a/docs/spec/scripts/crypto.go b/docs/spec/scripts/crypto.go index e4dbd8a24..9ae800f8f 100644 --- a/docs/spec/scripts/crypto.go +++ b/docs/spec/scripts/crypto.go @@ -2,124 +2,15 @@ package main import ( "fmt" + "os" - crypto "github.com/tendermint/go-crypto" + amino "github.com/tendermint/go-amino" + crypto "github.com/tendermint/tendermint/crypto" ) -// SECRET -var SECRET = []byte("some secret") - -func printEd() { - priv := crypto.GenPrivKeyEd25519FromSecret(SECRET) - pub := priv.PubKey().(crypto.PubKeyEd25519) - sig := priv.Sign([]byte("hello")).(crypto.SignatureEd25519) - - name := "tendermint/PubKeyEd25519" - length := len(pub[:]) - - fmt.Println("### PubKeyEd25519") - fmt.Println("") - fmt.Println("```") - fmt.Printf("// Name: %s\n", name) - fmt.Printf("// PrefixBytes: 0x%X \n", pub.Bytes()[:4]) - fmt.Printf("// Length: 0x%X \n", length) - fmt.Println("// Notes: raw 32-byte Ed25519 pubkey") - fmt.Println("type PubKeyEd25519 [32]byte") - fmt.Println("") - fmt.Println(`func (pubkey PubKeyEd25519) Address() []byte { - // NOTE: hash of the Amino encoded bytes! - return RIPEMD160(AminoEncode(pubkey)) -}`) - fmt.Println("```") - fmt.Println("") - fmt.Printf("For example, the 32-byte Ed25519 pubkey `%X` would be encoded as `%X`.\n\n", pub[:], pub.Bytes()) - fmt.Printf("The address would then be `RIPEMD160(0x%X)` or `%X`\n", pub.Bytes(), pub.Address()) - fmt.Println("") - - name = "tendermint/SignatureKeyEd25519" - length = len(sig[:]) - - fmt.Println("### SignatureEd25519") - fmt.Println("") - fmt.Println("```") - fmt.Printf("// Name: %s\n", name) - fmt.Printf("// PrefixBytes: 0x%X \n", sig.Bytes()[:4]) - fmt.Printf("// Length: 0x%X \n", length) - fmt.Println("// Notes: raw 64-byte Ed25519 signature") - fmt.Println("type SignatureEd25519 [64]byte") - fmt.Println("```") - fmt.Println("") - fmt.Printf("For example, the 64-byte Ed25519 signature `%X` would be encoded as `%X`\n", sig[:], sig.Bytes()) - fmt.Println("") - - name = "tendermint/PrivKeyEd25519" - - fmt.Println("### PrivKeyEd25519") - fmt.Println("") - fmt.Println("```") - fmt.Println("// Name:", name) - fmt.Println("// Notes: raw 32-byte priv key concatenated to raw 32-byte pub key") - fmt.Println("type PrivKeyEd25519 [64]byte") - fmt.Println("```") -} - -func printSecp() { - priv := crypto.GenPrivKeySecp256k1FromSecret(SECRET) - pub := priv.PubKey().(crypto.PubKeySecp256k1) - sig := priv.Sign([]byte("hello")).(crypto.SignatureSecp256k1) - - name := "tendermint/PubKeySecp256k1" - length := len(pub[:]) - - fmt.Println("### PubKeySecp256k1") - fmt.Println("") - fmt.Println("```") - fmt.Printf("// Name: %s\n", name) - fmt.Printf("// PrefixBytes: 0x%X \n", pub.Bytes()[:4]) - fmt.Printf("// Length: 0x%X \n", length) - fmt.Println("// Notes: OpenSSL compressed pubkey prefixed with 0x02 or 0x03") - fmt.Println("type PubKeySecp256k1 [33]byte") - fmt.Println("") - fmt.Println(`func (pubkey PubKeySecp256k1) Address() []byte { - // NOTE: hash of the raw pubkey bytes (not Amino encoded!). - // Compatible with Bitcoin addresses. - return RIPEMD160(SHA256(pubkey[:])) -}`) - fmt.Println("```") - fmt.Println("") - fmt.Printf("For example, the 33-byte Secp256k1 pubkey `%X` would be encoded as `%X`\n\n", pub[:], pub.Bytes()) - fmt.Printf("The address would then be `RIPEMD160(SHA256(0x%X))` or `%X`\n", pub[:], pub.Address()) - fmt.Println("") - - name = "tendermint/SignatureKeySecp256k1" - - fmt.Println("### SignatureSecp256k1") - fmt.Println("") - fmt.Println("```") - fmt.Printf("// Name: %s\n", name) - fmt.Printf("// PrefixBytes: 0x%X \n", sig.Bytes()[:4]) - fmt.Printf("// Length: Variable\n") - fmt.Printf("// Encoding prefix: Variable\n") - fmt.Println("// Notes: raw bytes of the Secp256k1 signature") - fmt.Println("type SignatureSecp256k1 []byte") - fmt.Println("```") - fmt.Println("") - fmt.Printf("For example, the Secp256k1 signature `%X` would be encoded as `%X`\n", []byte(sig[:]), sig.Bytes()) - fmt.Println("") - - name = "tendermint/PrivKeySecp256k1" - - fmt.Println("### PrivKeySecp256k1") - fmt.Println("") - fmt.Println("```") - fmt.Println("// Name:", name) - fmt.Println("// Notes: raw 32-byte priv key") - fmt.Println("type PrivKeySecp256k1 [32]byte") - fmt.Println("```") -} - func main() { - printEd() + cdc := amino.NewCodec() + crypto.RegisterAmino(cdc) + cdc.PrintTypes(os.Stdout) fmt.Println("") - printSecp() } diff --git a/docs/spec/software/abci.md b/docs/spec/software/abci.md index 9c9e6a58b..613e181f0 100644 --- a/docs/spec/software/abci.md +++ b/docs/spec/software/abci.md @@ -4,14 +4,14 @@ ABCI is the interface between Tendermint (a state-machine replication engine) and an application (the actual state machine). The ABCI message types are defined in a [protobuf -file](https://github.com/tendermint/abci/blob/master/types/types.proto). +file](https://github.com/tendermint/tendermint/blob/develop/abci/types/types.proto). For full details on the ABCI message types and protocol, see the [ABCI -specificaiton](https://github.com/tendermint/abci/blob/master/specification.rst). +specification](https://github.com/tendermint/tendermint/blob/develop/docs/abci-spec.md). Be sure to read the specification if you're trying to build an ABCI app! For additional details on server implementation, see the [ABCI -readme](https://github.com/tendermint/abci#implementation). +readme](https://github.com/tendermint/tendermint/blob/develop/abci/README.md). Here we provide some more details around the use of ABCI by Tendermint and clarify common "gotchas". diff --git a/docs/specification.rst b/docs/specification.rst deleted file mode 100644 index 70ebf633f..000000000 --- a/docs/specification.rst +++ /dev/null @@ -1,21 +0,0 @@ -############# -Specification -############# - -Here you'll find details of the Tendermint specification. Tendermint's types are produced by `godoc `__. - -.. toctree:: - :maxdepth: 2 - - specification/block-structure.rst - specification/byzantine-consensus-algorithm.rst - specification/configuration.rst - specification/corruption.rst - specification/fast-sync.rst - specification/genesis.rst - specification/light-client-protocol.rst - specification/merkle.rst - specification/rpc.rst - specification/secure-p2p.rst - specification/validators.rst - specification/wire-protocol.rst diff --git a/docs/specification/block-structure.rst b/docs/specification/block-structure.rst index 1569f6d96..7d8f3464c 100644 --- a/docs/specification/block-structure.rst +++ b/docs/specification/block-structure.rst @@ -58,8 +58,7 @@ validators `__. It represents the +that come from the ABCI application. It represents the state of the actual application, rather that the state of the blockchain itself. This means it's necessary in order to perform any business logic, such as verifying an account balance. @@ -144,8 +143,7 @@ Transaction ~~~~~~~~~~~ A transaction is any sequence of bytes. It is up to your -`ABCI `__ application to accept or -reject transactions. +ABCI application to accept or reject transactions. BlockID ~~~~~~~ diff --git a/docs/specification/configuration.md b/docs/specification/configuration.md new file mode 100644 index 000000000..214757b9f --- /dev/null +++ b/docs/specification/configuration.md @@ -0,0 +1,213 @@ +# Configuration + +Tendermint Core can be configured via a TOML file in +`$TMHOME/config/config.toml`. Some of these parameters can be overridden by +command-line flags. For most users, the options in the `##### main +base configuration options #####` are intended to be modified while +config options further below are intended for advance power users. + +## Options + +The default configuration file create by `tendermint init` has all +the parameters set with their default values. It will look something +like the file below, however, double check by inspecting the +`config.toml` created with your version of `tendermint` installed: + +``` +# This is a TOML config file. +# For more information, see https://github.com/toml-lang/toml + +##### main base config options ##### + +# TCP or UNIX socket address of the ABCI application, +# or the name of an ABCI application compiled in with the Tendermint binary +proxy_app = "tcp://127.0.0.1:26658" + +# A custom human readable name for this node +moniker = "anonymous" + +# If this node is many blocks behind the tip of the chain, FastSync +# allows them to catchup quickly by downloading blocks in parallel +# and verifying their commits +fast_sync = true + +# Database backend: leveldb | memdb +db_backend = "leveldb" + +# Database directory +db_path = "data" + +# Output level for logging +log_level = "state:info,*:error" + +##### additional base config options ##### + +# The ID of the chain to join (should be signed with every transaction and vote) +chain_id = "" + +# Path to the JSON file containing the initial validator set and other meta data +genesis_file = "genesis.json" + +# Path to the JSON file containing the private key to use as a validator in the consensus protocol +priv_validator_file = "priv_validator.json" + +# Mechanism to connect to the ABCI application: socket | grpc +abci = "socket" + +# TCP or UNIX socket address for the profiling server to listen on +prof_laddr = "" + +# If true, query the ABCI app on connecting to a new peer +# so the app can decide if we should keep the connection or not +filter_peers = false + +##### advanced configuration options ##### + +##### rpc server configuration options ##### +[rpc] + +# TCP or UNIX socket address for the RPC server to listen on +laddr = "tcp://0.0.0.0:26657" + +# TCP or UNIX socket address for the gRPC server to listen on +# NOTE: This server only supports /broadcast_tx_commit +grpc_laddr = "" + +# Maximum number of simultaneous connections. +# Does not include RPC (HTTP&WebSocket) connections. See max_open_connections +# If you want to accept more significant number than the default, make sure +# you increase your OS limits. +# 0 - unlimited. +grpc_max_open_connections = 900 + +# Activate unsafe RPC commands like /dial_seeds and /unsafe_flush_mempool +unsafe = false + +# Maximum number of simultaneous connections (including WebSocket). +# Does not include gRPC connections. See grpc_max_open_connections +# If you want to accept more significant number than the default, make sure +# you increase your OS limits. +# 0 - unlimited. +max_open_connections = 450 + +##### peer to peer configuration options ##### +[p2p] + +# Address to listen for incoming connections +laddr = "tcp://0.0.0.0:26656" + +# Comma separated list of seed nodes to connect to +seeds = "" + +# Comma separated list of nodes to keep persistent connections to +# Do not add private peers to this list if you don't want them advertised +persistent_peers = "" + +# UPNP port forwarding +upnp = false + +# Path to address book +addr_book_file = "addrbook.json" + +# Set true for strict address routability rules +addr_book_strict = true + +# Time to wait before flushing messages out on the connection, in ms +flush_throttle_timeout = 100 + +# Maximum number of peers to connect to +max_num_peers = 50 + +# Maximum size of a message packet payload, in bytes +max_packet_msg_payload_size = 1024 + +# Rate at which packets can be sent, in bytes/second +send_rate = 512000 + +# Rate at which packets can be received, in bytes/second +recv_rate = 512000 + +# Set true to enable the peer-exchange reactor +pex = true + +# Seed mode, in which node constantly crawls the network and looks for +# peers. If another node asks it for addresses, it responds and disconnects. +# +# Does not work if the peer-exchange reactor is disabled. +seed_mode = false + +# Comma separated list of peer IDs to keep private (will not be gossiped to other peers) +private_peer_ids = "" + +##### mempool configuration options ##### +[mempool] + +recheck = true +recheck_empty = true +broadcast = true +wal_dir = "data/mempool.wal" + +# size of the mempool +size = 100000 + +# size of the cache (used to filter transactions we saw earlier) +cache_size = 100000 + +##### consensus configuration options ##### +[consensus] + +wal_file = "data/cs.wal/wal" + +# All timeouts are in milliseconds +timeout_propose = 3000 +timeout_propose_delta = 500 +timeout_prevote = 1000 +timeout_prevote_delta = 500 +timeout_precommit = 1000 +timeout_precommit_delta = 500 +timeout_commit = 1000 + +# Make progress as soon as we have all the precommits (as if TimeoutCommit = 0) +skip_timeout_commit = false + +# EmptyBlocks mode and possible interval between empty blocks in seconds +create_empty_blocks = true +create_empty_blocks_interval = 0 + +# Reactor sleep duration parameters are in milliseconds +peer_gossip_sleep_duration = 100 +peer_query_maj23_sleep_duration = 2000 + +##### transactions indexer configuration options ##### +[tx_index] + +# What indexer to use for transactions +# +# Options: +# 1) "null" +# 2) "kv" (default) - the simplest possible indexer, backed by key-value storage (defaults to levelDB; see DBBackend). +indexer = "kv" + +# Comma-separated list of tags to index (by default the only tag is tx hash) +# +# It's recommended to index only a subset of tags due to possible memory +# bloat. This is, of course, depends on the indexer's DB and the volume of +# transactions. +index_tags = "" + +# When set to true, tells indexer to index all tags. Note this may be not +# desirable (see the comment above). IndexTags has a precedence over +# IndexAllTags (i.e. when given both, IndexTags will be indexed). +index_all_tags = false + +##### instrumentation configuration options ##### +[instrumentation] + +# When true, Prometheus metrics are served under /metrics on +# PrometheusListenAddr. +# Check out the documentation for the list of available metrics. +prometheus = false + +# Address to listen for Prometheus collector(s) connections +prometheus_listen_addr = ":26660" +``` diff --git a/docs/specification/configuration.rst b/docs/specification/configuration.rst deleted file mode 100644 index d7a030039..000000000 --- a/docs/specification/configuration.rst +++ /dev/null @@ -1,191 +0,0 @@ -Configuration -============= - -Tendermint Core can be configured via a TOML file in -``$TMHOME/config/config.toml``. Some of these parameters can be overridden by -command-line flags. For most users, the options in the ``##### main -base configuration options #####`` are intended to be modified while -config options further below are intended for advance power users. - -Config options -~~~~~~~~~~~~~~ - -The default configuration file create by ``tendermint init`` has all -the parameters set with their default values. It will look something -like the file below, however, double check by inspecting the -``config.toml`` created with your version of ``tendermint`` installed: - -:: - - # This is a TOML config file. - # For more information, see https://github.com/toml-lang/toml - - ##### main base config options ##### - - # TCP or UNIX socket address of the ABCI application, - # or the name of an ABCI application compiled in with the Tendermint binary - proxy_app = "tcp://127.0.0.1:26658" - - # A custom human readable name for this node - moniker = "anonymous" - - # If this node is many blocks behind the tip of the chain, FastSync - # allows them to catchup quickly by downloading blocks in parallel - # and verifying their commits - fast_sync = true - - # Database backend: leveldb | memdb - db_backend = "leveldb" - - # Database directory - db_path = "data" - - # Output level for logging - log_level = "state:info,*:error" - - ##### additional base config options ##### - - # The ID of the chain to join (should be signed with every transaction and vote) - chain_id = "" - - # Path to the JSON file containing the initial validator set and other meta data - genesis_file = "genesis.json" - - # Path to the JSON file containing the private key to use as a validator in the consensus protocol - priv_validator_file = "priv_validator.json" - - # Mechanism to connect to the ABCI application: socket | grpc - abci = "socket" - - # TCP or UNIX socket address for the profiling server to listen on - prof_laddr = "" - - # If true, query the ABCI app on connecting to a new peer - # so the app can decide if we should keep the connection or not - filter_peers = false - - ##### advanced configuration options ##### - - ##### rpc server configuration options ##### - [rpc] - - # TCP or UNIX socket address for the RPC server to listen on - laddr = "tcp://0.0.0.0:26657" - - # TCP or UNIX socket address for the gRPC server to listen on - # NOTE: This server only supports /broadcast_tx_commit - grpc_laddr = "" - - # Activate unsafe RPC commands like /dial_seeds and /unsafe_flush_mempool - unsafe = false - - ##### peer to peer configuration options ##### - [p2p] - - # Address to listen for incoming connections - laddr = "tcp://0.0.0.0:26656" - - # Comma separated list of seed nodes to connect to - seeds = "" - - # Comma separated list of nodes to keep persistent connections to - # Do not add private peers to this list if you don't want them advertised - persistent_peers = "" - - # Path to address book - addr_book_file = "addrbook.json" - - # Set true for strict address routability rules - addr_book_strict = true - - # Time to wait before flushing messages out on the connection, in ms - flush_throttle_timeout = 100 - - # Maximum number of peers to connect to - max_num_peers = 50 - - # Maximum size of a message packet payload, in bytes - max_msg_packet_payload_size = 1024 - - # Rate at which packets can be sent, in bytes/second - send_rate = 512000 - - # Rate at which packets can be received, in bytes/second - recv_rate = 512000 - - # Set true to enable the peer-exchange reactor - pex = true - - # Seed mode, in which node constantly crawls the network and looks for - # peers. If another node asks it for addresses, it responds and disconnects. - # - # Does not work if the peer-exchange reactor is disabled. - seed_mode = false - - # Comma separated list of peer IDs to keep private (will not be gossiped to other peers) - private_peer_ids = "" - - ##### mempool configuration options ##### - [mempool] - - recheck = true - recheck_empty = true - broadcast = true - wal_dir = "data/mempool.wal" - - # size of the mempool - size = 100000 - - # size of the cache (used to filter transactions we saw earlier) - cache_size = 100000 - - ##### consensus configuration options ##### - [consensus] - - wal_file = "data/cs.wal/wal" - - # All timeouts are in milliseconds - timeout_propose = 3000 - timeout_propose_delta = 500 - timeout_prevote = 1000 - timeout_prevote_delta = 500 - timeout_precommit = 1000 - timeout_precommit_delta = 500 - timeout_commit = 1000 - - # Make progress as soon as we have all the precommits (as if TimeoutCommit = 0) - skip_timeout_commit = false - - # BlockSize - max_block_size_txs = 10000 - max_block_size_bytes = 1 - - # EmptyBlocks mode and possible interval between empty blocks in seconds - create_empty_blocks = true - create_empty_blocks_interval = 0 - - # Reactor sleep duration parameters are in milliseconds - peer_gossip_sleep_duration = 100 - peer_query_maj23_sleep_duration = 2000 - - ##### transactions indexer configuration options ##### - [tx_index] - - # What indexer to use for transactions - # - # Options: - # 1) "null" (default) - # 2) "kv" - the simplest possible indexer, backed by key-value storage (defaults to levelDB; see DBBackend). - indexer = "{{ .TxIndex.Indexer }}" - - # Comma-separated list of tags to index (by default the only tag is tx hash) - # - # It's recommended to index only a subset of tags due to possible memory - # bloat. This is, of course, depends on the indexer's DB and the volume of - # transactions. - index_tags = "{{ .TxIndex.IndexTags }}" - - # When set to true, tells indexer to index all tags. Note this may be not - # desirable (see the comment above). IndexTags has a precedence over - # IndexAllTags (i.e. when given both, IndexTags will be indexed). - index_all_tags = {{ .TxIndex.IndexAllTags }} diff --git a/docs/specification/rpc.md b/docs/specification/rpc.md new file mode 100644 index 000000000..2f3a72c74 --- /dev/null +++ b/docs/specification/rpc.md @@ -0,0 +1,3 @@ +# RPC + +The RPC documentation is hosted [here](https://tendermint.github.io/slate) and is generated by the CI from our [Slate repo](https://github.com/tendermint/slate). To update the documentation, edit the relevant `godoc` comments in the [rpc/core directory](https://github.com/tendermint/tendermint/tree/develop/rpc/core). diff --git a/docs/specification/rpc.rst b/docs/specification/rpc.rst deleted file mode 100644 index 1dd1165b9..000000000 --- a/docs/specification/rpc.rst +++ /dev/null @@ -1,4 +0,0 @@ -RPC -=== - -The RPC documentation is hosted `here `__ and is generated by the CI from our `Slate repo `__. To update the documentation, edit the relevant ``godoc`` comments in the `rpc/core directory `__. diff --git a/docs/specification/validators.rst b/docs/specification/validators.rst index fb6bbbaca..085994f3d 100644 --- a/docs/specification/validators.rst +++ b/docs/specification/validators.rst @@ -26,8 +26,7 @@ There are two ways to become validator. 1. They can be pre-established in the `genesis state <./genesis.html>`__ -2. The `ABCI app responds to the EndBlock - message `__ with changes to the +2. The ABCI app responds to the EndBlock message with changes to the existing validator set. Committing a Block diff --git a/docs/terraform-and-ansible.md b/docs/terraform-and-ansible.md index 13fbeec89..55c38cef7 100644 --- a/docs/terraform-and-ansible.md +++ b/docs/terraform-and-ansible.md @@ -34,13 +34,16 @@ These will be used by both `terraform` and `ansible`. This step will create four Digital Ocean droplets. First, go to the correct directory: - cd $GOPATH/src/github.com/tendermint/tendermint/networks/remote/terraform +``` +cd $GOPATH/src/github.com/tendermint/tendermint/networks/remote/terraform +``` then: - terraform init - terraform apply -var DO_API_TOKEN="$DO_API_TOKEN" -var SSH_KEY_FILE="$SSH_KEY_FILE" - +``` +terraform init +terraform apply -var DO_API_TOKEN="$DO_API_TOKEN" -var SSH_KEY_FILE="$SSH_KEY_FILE" +``` and you will get a list of IP addresses that belong to your droplets. With the droplets created and running, let's setup Ansible. @@ -66,14 +69,18 @@ review [manual deployments](./deploy-testnets.md). Here's the command to run: - ansible-playbook -i inventory/digital_ocean.py -l sentrynet config.yml -e BINARY=$GOPATH/src/github.com/tendermint/tendermint/build/tendermint -e CONFIGDIR=$GOPATH/src/github.com/tendermint/tendermint/docs/examples +``` +ansible-playbook -i inventory/digital_ocean.py -l sentrynet config.yml -e BINARY=$GOPATH/src/github.com/tendermint/tendermint/build/tendermint -e CONFIGDIR=$GOPATH/src/github.com/tendermint/tendermint/docs/examples +``` Voila! All your droplets now have the `tendermint` binary and required configuration files to run a testnet. Next, we run the install role: - ansible-playbook -i inventory/digital_ocean.py -l sentrynet install.yml +``` +ansible-playbook -i inventory/digital_ocean.py -l sentrynet install.yml +``` which as you'll see below, executes `tendermint node --proxy_app=kvstore` on all droplets. Although we'll @@ -88,35 +95,43 @@ increasing). Next, open `roles/install/templates/systemd.service.j2` and look for the line `ExecStart` which should look something like: - ExecStart=/usr/bin/tendermint node --proxy_app=kvstore +``` +ExecStart=/usr/bin/tendermint node --proxy_app=kvstore +``` and add the `--p2p.persistent_peers` flag with the relevant information for each node. The resulting file should look something like: - [Unit] - Description={{service}} - Requires=network-online.target - After=network-online.target +``` +[Unit] +Description={{service}} +Requires=network-online.target +After=network-online.target - [Service] - Restart=on-failure - User={{service}} - Group={{service}} - PermissionsStartOnly=true - ExecStart=/usr/bin/tendermint node --proxy_app=kvstore --p2p.persistent_peers=167b80242c300bf0ccfb3ced3dec60dc2a81776e@165.227.41.206:26656,3c7a5920811550c04bf7a0b2f1e02ab52317b5e6@165.227.43.146:26656,303a1a4312c30525c99ba66522dd81cca56a361a@159.89.115.32:26656,b686c2a7f4b1b46dca96af3a0f31a6a7beae0be4@159.89.119.125:26656 - ExecReload=/bin/kill -HUP $MAINPID - KillSignal=SIGTERM +[Service] +Restart=on-failure +User={{service}} +Group={{service}} +PermissionsStartOnly=true +ExecStart=/usr/bin/tendermint node --proxy_app=kvstore --p2p.persistent_peers=167b80242c300bf0ccfb3ced3dec60dc2a81776e@165.227.41.206:26656,3c7a5920811550c04bf7a0b2f1e02ab52317b5e6@165.227.43.146:26656,303a1a4312c30525c99ba66522dd81cca56a361a@159.89.115.32:26656,b686c2a7f4b1b46dca96af3a0f31a6a7beae0be4@159.89.119.125:26656 +ExecReload=/bin/kill -HUP $MAINPID +KillSignal=SIGTERM - [Install] - WantedBy=multi-user.target +[Install] +WantedBy=multi-user.target +``` Then, stop the nodes: - ansible-playbook -i inventory/digital_ocean.py -l sentrynet stop.yml +``` +ansible-playbook -i inventory/digital_ocean.py -l sentrynet stop.yml +``` Finally, we run the install role again: - ansible-playbook -i inventory/digital_ocean.py -l sentrynet install.yml +``` +ansible-playbook -i inventory/digital_ocean.py -l sentrynet install.yml +``` to re-run `tendermint node` with the new flag, on all droplets. The `latest_block_hash` should now be changing and `latest_block_height` @@ -124,7 +139,9 @@ increasing. Your testnet is now up and running :) Peek at the logs with the status role: - ansible-playbook -i inventory/digital_ocean.py -l sentrynet status.yml +``` +ansible-playbook -i inventory/digital_ocean.py -l sentrynet status.yml +``` ### Logging @@ -134,14 +151,18 @@ service provider. You can set up your nodes to log there automatically. Create an account and get your API key from the notes on [this page](https://app.logz.io/#/dashboard/data-sources/Filebeat), then: - yum install systemd-devel || echo "This will only work on RHEL-based systems." - apt-get install libsystemd-dev || echo "This will only work on Debian-based systems." - - go get github.com/mheese/journalbeat - ansible-playbook -i inventory/digital_ocean.py -l sentrynet logzio.yml -e LOGZIO_TOKEN=ABCDEFGHIJKLMNOPQRSTUVWXYZ012345 +``` +yum install systemd-devel || echo "This will only work on RHEL-based systems." +apt-get install libsystemd-dev || echo "This will only work on Debian-based systems." + +go get github.com/mheese/journalbeat +ansible-playbook -i inventory/digital_ocean.py -l sentrynet logzio.yml -e LOGZIO_TOKEN=ABCDEFGHIJKLMNOPQRSTUVWXYZ012345 +``` ### Cleanup To remove your droplets, run: - terraform destroy -var DO_API_TOKEN="$DO_API_TOKEN" -var SSH_KEY_FILE="$SSH_KEY_FILE" +``` +terraform destroy -var DO_API_TOKEN="$DO_API_TOKEN" -var SSH_KEY_FILE="$SSH_KEY_FILE" +``` diff --git a/docs/using-tendermint.md b/docs/using-tendermint.md index 86ec2529a..1cd910821 100644 --- a/docs/using-tendermint.md +++ b/docs/using-tendermint.md @@ -45,8 +45,7 @@ blocks are produced regularly, even if there are no transactions. See *No Empty Blocks*, below, to modify this setting. Tendermint supports in-process versions of the `counter`, `kvstore` and -`nil` apps that ship as examples in the [ABCI -repository](https://github.com/tendermint/abci). It's easy to compile +`nil` apps that ship as examples with `abci-cli`. It's easy to compile your own app in-process with Tendermint if it's written in Go. If your app is not written in Go, simply run it in another process, and use the `--proxy_app` flag to specify the address of the socket it is listening @@ -130,7 +129,7 @@ new blockchain will not make any blocks. ## Configuration Tendermint uses a `config.toml` for configuration. For details, see [the -config specification](./specification/configuration.html). +config specification](./specification/configuration.md). Notable options include the socket address of the application (`proxy_app`), the listening address of the Tendermint peer @@ -214,7 +213,7 @@ When `tendermint init` is run, both a `genesis.json` and { "pub_key" : { "value" : "h3hk+QE8c6QLTySp8TcfzclJw/BG79ziGB/pIA+DfPE=", - "type" : "AC26791624DE60" + "type" : "tendermint/PubKeyEd25519" }, "power" : 10, "name" : "" @@ -229,16 +228,16 @@ And the `priv_validator.json`: { "last_step" : 0, - "last_round" : 0, + "last_round" : "0", "address" : "B788DEDE4F50AD8BC9462DE76741CCAFF87D51E2", "pub_key" : { "value" : "h3hk+QE8c6QLTySp8TcfzclJw/BG79ziGB/pIA+DfPE=", - "type" : "AC26791624DE60" + "type" : "tendermint/PubKeyEd25519" }, - "last_height" : 0, + "last_height" : "0", "priv_key" : { "value" : "JPivl82x+LfVkp8i3ztoTjY6c6GJ4pBxQexErOCyhwqHeGT5ATxzpAtPJKnxNx/NyUnD8Ebv3OIYH+kgD4N88Q==", - "type" : "954568A3288910" + "type" : "tendermint/PrivKeyEd25519" } } @@ -261,7 +260,7 @@ but must be positive, thus the range is: 0 through 9223372036854775807. Because of how the current proposer selection algorithm works, we do not recommend having voting powers greater than 10\^12 (ie. 1 trillion) (see [Proposals section of Byzantine Consensus -Algorithm](./specification/byzantine-consensus-algorithm.html#proposals) +Algorithm](./specification/byzantine-consensus-algorithm.md#proposals) for details). If we want to add more nodes to the network, we have two choices: we can @@ -348,15 +347,15 @@ Now we can update our genesis file. For instance, if the new "address" : "5AF49D2A2D4F5AD4C7C8C4CC2FB020131E9C4902", "pub_key" : { "value" : "l9X9+fjkeBzDfPGbUM7AMIRE6uJN78zN5+lk5OYotek=", - "type" : "AC26791624DE60" + "type" : "tendermint/PubKeyEd25519" }, "priv_key" : { "value" : "EDJY9W6zlAw+su6ITgTKg2nTZcHAH1NMTW5iwlgmNDuX1f35+OR4HMN88ZtQzsAwhETq4k3vzM3n6WTk5ii16Q==", - "type" : "954568A3288910" + "type" : "tendermint/PrivKeyEd25519" }, "last_step" : 0, - "last_round" : 0, - "last_height" : 0 + "last_round" : "0", + "last_height" : "0" } then the new `genesis.json` will be: @@ -366,7 +365,7 @@ then the new `genesis.json` will be: { "pub_key" : { "value" : "h3hk+QE8c6QLTySp8TcfzclJw/BG79ziGB/pIA+DfPE=", - "type" : "AC26791624DE60" + "type" : "tendermint/PubKeyEd25519" }, "power" : 10, "name" : "" @@ -374,7 +373,7 @@ then the new `genesis.json` will be: { "pub_key" : { "value" : "l9X9+fjkeBzDfPGbUM7AMIRE6uJN78zN5+lk5OYotek=", - "type" : "AC26791624DE60" + "type" : "tendermint/PubKeyEd25519" }, "power" : 10, "name" : "" @@ -399,7 +398,7 @@ failing, you need at least four validator nodes (e.g., 2/3). Updating validators in a live network is supported but must be explicitly programmed by the application developer. See the [application -developers guide](./app-development.html) for more details. +developers guide](./app-development.md) for more details. ### Local Network diff --git a/evidence/pool.go b/evidence/pool.go index 4bad355f7..247629b6b 100644 --- a/evidence/pool.go +++ b/evidence/pool.go @@ -4,9 +4,9 @@ import ( "fmt" "sync" - clist "github.com/tendermint/tmlibs/clist" - dbm "github.com/tendermint/tmlibs/db" - "github.com/tendermint/tmlibs/log" + clist "github.com/tendermint/tendermint/libs/clist" + dbm "github.com/tendermint/tendermint/libs/db" + "github.com/tendermint/tendermint/libs/log" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" diff --git a/evidence/pool_test.go b/evidence/pool_test.go index 019076234..915cba327 100644 --- a/evidence/pool_test.go +++ b/evidence/pool_test.go @@ -9,7 +9,7 @@ import ( sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" - dbm "github.com/tendermint/tmlibs/db" + dbm "github.com/tendermint/tendermint/libs/db" ) var mockState = sm.State{} diff --git a/evidence/reactor.go b/evidence/reactor.go index 5159572e3..7b22b8dba 100644 --- a/evidence/reactor.go +++ b/evidence/reactor.go @@ -6,8 +6,8 @@ import ( "time" "github.com/tendermint/go-amino" - clist "github.com/tendermint/tmlibs/clist" - "github.com/tendermint/tmlibs/log" + clist "github.com/tendermint/tendermint/libs/clist" + "github.com/tendermint/tendermint/libs/log" "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/types" diff --git a/evidence/reactor_test.go b/evidence/reactor_test.go index 2f1c34e6e..1687f25a3 100644 --- a/evidence/reactor_test.go +++ b/evidence/reactor_test.go @@ -10,8 +10,8 @@ import ( "github.com/go-kit/kit/log/term" - dbm "github.com/tendermint/tmlibs/db" - "github.com/tendermint/tmlibs/log" + dbm "github.com/tendermint/tendermint/libs/db" + "github.com/tendermint/tendermint/libs/log" cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/p2p" diff --git a/evidence/store.go b/evidence/store.go index 6af5d75d8..20b37bdb2 100644 --- a/evidence/store.go +++ b/evidence/store.go @@ -4,7 +4,7 @@ import ( "fmt" "github.com/tendermint/tendermint/types" - dbm "github.com/tendermint/tmlibs/db" + dbm "github.com/tendermint/tendermint/libs/db" ) /* diff --git a/evidence/store_test.go b/evidence/store_test.go index 3fdb3ba6e..30dc1c4d5 100644 --- a/evidence/store_test.go +++ b/evidence/store_test.go @@ -5,7 +5,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/tendermint/tendermint/types" - dbm "github.com/tendermint/tmlibs/db" + dbm "github.com/tendermint/tendermint/libs/db" ) //------------------------------------------- diff --git a/evidence/wire.go b/evidence/wire.go index 842e0707a..fb3a177cc 100644 --- a/evidence/wire.go +++ b/evidence/wire.go @@ -2,7 +2,7 @@ package evidence import ( "github.com/tendermint/go-amino" - "github.com/tendermint/go-crypto" + "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/types" ) diff --git a/libs/.editorconfig b/libs/.editorconfig new file mode 100644 index 000000000..82f774362 --- /dev/null +++ b/libs/.editorconfig @@ -0,0 +1,19 @@ +# top-most EditorConfig file +root = true + +# Unix-style newlines with a newline ending every file +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true + +[Makefile] +indent_style = tab + +[*.sh] +indent_style = tab + +[*.proto] +indent_style = space +indent_size = 2 diff --git a/libs/.gitignore b/libs/.gitignore new file mode 100644 index 000000000..a2ebfde29 --- /dev/null +++ b/libs/.gitignore @@ -0,0 +1,5 @@ +*.sw[opqr] +vendor +.glide + +pubsub/query/fuzz_test/output diff --git a/libs/CHANGELOG.md b/libs/CHANGELOG.md new file mode 100644 index 000000000..0f900c57f --- /dev/null +++ b/libs/CHANGELOG.md @@ -0,0 +1,438 @@ +# Changelog + +## 0.9.0 + +*June 24th, 2018* + +BREAKING: + - [events, pubsub] Removed - moved to github.com/tendermint/tendermint + - [merkle] Use 20-bytes of SHA256 instead of RIPEMD160. NOTE: this package is + moving to github.com/tendermint/go-crypto ! + - [common] Remove gogoproto from KVPair types + - [common] Error simplification, #220 + +FEATURES: + + - [db/remotedb] New DB type using an external CLevelDB process via + GRPC + - [autofile] logjack command for piping stdin to a rotating file + - [bech32] New package. NOTE: should move out of here - it's just two small + functions + - [common] ColoredBytes([]byte) string for printing mixed ascii and bytes + - [db] DebugDB uses ColoredBytes() + +## 0.8.4 + +*June 5, 2018* + +IMPROVEMENTS: + + - [autofile] Flush on Stop; Close() method to Flush and close file + +## 0.8.3 + +*May 21, 2018* + +FEATURES: + + - [common] ASCIITrim() + +## 0.8.2 (April 23rd, 2018) + +FEATURES: + + - [pubsub] TagMap, NewTagMap + - [merkle] SimpleProofsFromMap() + - [common] IsASCIIText() + - [common] PrefixEndBytes // e.g. increment or nil + - [common] BitArray.MarshalJSON/.UnmarshalJSON + - [common] BitArray uses 'x' not 'X' for String() and above. + - [db] DebugDB shows better colorized output + +BUG FIXES: + + - [common] Fix TestParallelAbort nondeterministic failure #201/#202 + - [db] PrefixDB Iterator/ReverseIterator fixes + - [db] DebugDB fixes + +## 0.8.1 (April 5th, 2018) + +FEATURES: + + - [common] Error.Error() includes cause + - [common] IsEmpty() for 0 length + +## 0.8.0 (April 4th, 2018) + +BREAKING: + + - [merkle] `PutVarint->PutUvarint` in encodeByteSlice + - [db] batch.WriteSync() + - [common] Refactored and fixed `Parallel` function + - [common] Refactored `Rand` functionality + - [common] Remove unused `Right/LeftPadString` functions + - [common] Remove StackError, introduce Error interface (to replace use of pkg/errors) + +FEATURES: + + - [db] NewPrefixDB for a DB with all keys prefixed + - [db] NewDebugDB prints everything during operation + - [common] SplitAndTrim func + - [common] rand.Float64(), rand.Int63n(n), rand.Int31n(n) and global equivalents + - [common] HexBytes Format() + +BUG FIXES: + + - [pubsub] Fix unsubscribing + - [cli] Return config errors + - [common] Fix WriteFileAtomic Windows bug + +## 0.7.1 (March 22, 2018) + +IMPROVEMENTS: + + - glide -> dep + +BUG FIXES: + + - [common] Fix panic in NewBitArray for negative bits + - [common] Fix and simplify WriteFileAtomic so it cleans up properly + +## 0.7.0 (February 20, 2018) + +BREAKING: + + - [db] Major API upgrade. See `db/types.go`. + - [common] added `Quit() <-chan struct{}` to Service interface. + The returned channel is closed when service is stopped. + - [common] Remove HTTP functions + - [common] Heap.Push takes an `int`, new Heap.PushComparable takes the comparable. + - [logger] Removed. Use `log` + - [merkle] Major API updade - uses cmn.KVPairs. + - [cli] WriteDemoConfig -> WriteConfigValues + - [all] Remove go-wire dependency! + +FEATURES: + + - [db] New FSDB that uses the filesystem directly + - [common] HexBytes + - [common] KVPair and KI64Pair (protobuf based key-value pair objects) + +IMPROVEMENTS: + + - [clist] add WaitChan() to CList, NextWaitChan() and PrevWaitChan() + to CElement. These can be used instead of blocking `*Wait()` methods + if you need to be able to send quit signal and not block forever + - [common] IsHex handles 0x-prefix + +BUG FIXES: + + - [common] BitArray check for nil arguments + - [common] Fix memory leak in RepeatTimer + +## 0.6.0 (December 29, 2017) + +BREAKING: + - [cli] remove --root + - [pubsub] add String() method to Query interface + +IMPROVEMENTS: + - [common] use a thread-safe and well seeded non-crypto rng + +BUG FIXES + - [clist] fix misuse of wait group + - [common] introduce Ticker interface and logicalTicker for better testing of timers + +## 0.5.0 (December 5, 2017) + +BREAKING: + - [common] replace Service#Start, Service#Stop first return value (bool) with an + error (ErrAlreadyStarted, ErrAlreadyStopped) + - [common] replace Service#Reset first return value (bool) with an error + - [process] removed + +FEATURES: + - [common] IntInSlice and StringInSlice functions + - [pubsub/query] introduce `Condition` struct, expose `Operator`, and add `query.Conditions()` + +## 0.4.1 (November 27, 2017) + +FEATURES: + - [common] `Keys()` method on `CMap` + +IMPROVEMENTS: + - [log] complex types now encoded as "%+v" by default if `String()` method is undefined (previously resulted in error) + - [log] logger logs its own errors + +BUG FIXES: + - [common] fixed `Kill()` to build on Windows (Windows does not have `syscall.Kill`) + +## 0.4.0 (October 26, 2017) + +BREAKING: + - [common] GoPath is now a function + - [db] `DB` and `Iterator` interfaces have new methods to better support iteration + +FEATURES: + - [autofile] `Read([]byte)` and `Write([]byte)` methods on `Group` to support binary WAL + - [common] `Kill()` sends SIGTERM to the current process + +IMPROVEMENTS: + - comments and linting + +BUG FIXES: + - [events] fix allocation error prefixing cache with 1000 empty events + +## 0.3.2 (October 2, 2017) + +BUG FIXES: + +- [autofile] fix AutoFile.Sync() to open file if it's been closed +- [db] fix MemDb.Close() to not empty the database (ie. its just a noop) + + +## 0.3.1 (September 22, 2017) + +BUG FIXES: + +- [common] fix WriteFileAtomic to not use /tmp, which can be on another device + +## 0.3.0 (September 22, 2017) + +BREAKING CHANGES: + +- [log] logger functions no longer returns an error +- [common] NewBaseService takes the new logger +- [cli] RunCaptureWithArgs now captures stderr and stdout + - +func RunCaptureWithArgs(cmd Executable, args []string, env map[string]string) (stdout, stderr string, err error) + - -func RunCaptureWithArgs(cmd Executable, args []string, env map[string]string) (output string, err error) + +FEATURES: + +- [common] various common HTTP functionality +- [common] Date range parsing from string (ex. "2015-12-31:2017-12-31") +- [common] ProtocolAndAddress function +- [pubsub] New package for publish-subscribe with more advanced filtering + +BUG FIXES: + +- [common] fix atomicity of WriteFileAtomic by calling fsync +- [db] fix memDb iteration index out of range +- [autofile] fix Flush by calling fsync + +## 0.2.2 (June 16, 2017) + +FEATURES: + +- [common] IsHex and StripHex for handling `0x` prefixed hex strings +- [log] NewTracingLogger returns a logger that output error traces, ala `github.com/pkg/errors` + +IMPROVEMENTS: + +- [cli] Error handling for tests +- [cli] Support dashes in ENV variables + +BUG FIXES: + +- [flowrate] Fix non-deterministic test failures + +## 0.2.1 (June 2, 2017) + +FEATURES: + +- [cli] Log level parsing moved here from tendermint repo + +## 0.2.0 (May 18, 2017) + +BREAKING CHANGES: + +- [common] NewBaseService takes the new logger + + +FEATURES: + +- [cli] New library to standardize building command line tools +- [log] New logging library + +BUG FIXES: + +- [autofile] Close file before rotating + +## 0.1.0 (May 1, 2017) + +Initial release, combines what were previously independent repos: + +- go-autofile +- go-clist +- go-common +- go-db +- go-events +- go-flowrate +- go-logger +- go-merkle +- go-process + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/libs/CODEOWNERS b/libs/CODEOWNERS new file mode 100644 index 000000000..d2dddf85a --- /dev/null +++ b/libs/CODEOWNERS @@ -0,0 +1,3 @@ +* @melekes @ebuchman +*.md @zramsay +*.rst @zramsay diff --git a/libs/Gopkg.lock b/libs/Gopkg.lock new file mode 100644 index 000000000..f0eaee18d --- /dev/null +++ b/libs/Gopkg.lock @@ -0,0 +1,281 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + branch = "master" + name = "github.com/btcsuite/btcutil" + packages = ["bech32"] + revision = "d4cc87b860166d00d6b5b9e0d3b3d71d6088d4d4" + +[[projects]] + name = "github.com/davecgh/go-spew" + packages = ["spew"] + revision = "346938d642f2ec3594ed81d874461961cd0faa76" + version = "v1.1.0" + +[[projects]] + branch = "master" + name = "github.com/fortytw2/leaktest" + packages = ["."] + revision = "3b724c3d7b8729a35bf4e577f71653aec6e53513" + +[[projects]] + name = "github.com/fsnotify/fsnotify" + packages = ["."] + revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9" + version = "v1.4.7" + +[[projects]] + name = "github.com/go-kit/kit" + packages = [ + "log", + "log/level", + "log/term" + ] + revision = "4dc7be5d2d12881735283bcab7352178e190fc71" + version = "v0.6.0" + +[[projects]] + name = "github.com/go-logfmt/logfmt" + packages = ["."] + revision = "390ab7935ee28ec6b286364bba9b4dd6410cb3d5" + version = "v0.3.0" + +[[projects]] + name = "github.com/go-stack/stack" + packages = ["."] + revision = "817915b46b97fd7bb80e8ab6b69f01a53ac3eebf" + version = "v1.6.0" + +[[projects]] + name = "github.com/golang/protobuf" + packages = [ + "proto", + "ptypes", + "ptypes/any", + "ptypes/duration", + "ptypes/timestamp" + ] + revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265" + version = "v1.1.0" + +[[projects]] + branch = "master" + name = "github.com/golang/snappy" + packages = ["."] + revision = "553a641470496b2327abcac10b36396bd98e45c9" + +[[projects]] + branch = "master" + name = "github.com/hashicorp/hcl" + packages = [ + ".", + "hcl/ast", + "hcl/parser", + "hcl/scanner", + "hcl/strconv", + "hcl/token", + "json/parser", + "json/scanner", + "json/token" + ] + revision = "23c074d0eceb2b8a5bfdbb271ab780cde70f05a8" + +[[projects]] + name = "github.com/inconshreveable/mousetrap" + packages = ["."] + revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75" + version = "v1.0" + +[[projects]] + branch = "master" + name = "github.com/jmhodges/levigo" + packages = ["."] + revision = "c42d9e0ca023e2198120196f842701bb4c55d7b9" + +[[projects]] + branch = "master" + name = "github.com/kr/logfmt" + packages = ["."] + revision = "b84e30acd515aadc4b783ad4ff83aff3299bdfe0" + +[[projects]] + name = "github.com/magiconair/properties" + packages = ["."] + revision = "49d762b9817ba1c2e9d0c69183c2b4a8b8f1d934" + +[[projects]] + name = "github.com/mitchellh/mapstructure" + packages = ["."] + revision = "b4575eea38cca1123ec2dc90c26529b5c5acfcff" + +[[projects]] + name = "github.com/pelletier/go-toml" + packages = ["."] + revision = "acdc4509485b587f5e675510c4f2c63e90ff68a8" + version = "v1.1.0" + +[[projects]] + name = "github.com/pkg/errors" + packages = ["."] + revision = "645ef00459ed84a119197bfb8d8205042c6df63d" + version = "v0.8.0" + +[[projects]] + name = "github.com/pmezard/go-difflib" + packages = ["difflib"] + revision = "792786c7400a136282c1664665ae0a8db921c6c2" + version = "v1.0.0" + +[[projects]] + name = "github.com/spf13/afero" + packages = [ + ".", + "mem" + ] + revision = "bb8f1927f2a9d3ab41c9340aa034f6b803f4359c" + version = "v1.0.2" + +[[projects]] + name = "github.com/spf13/cast" + packages = ["."] + revision = "acbeb36b902d72a7a4c18e8f3241075e7ab763e4" + version = "v1.1.0" + +[[projects]] + name = "github.com/spf13/cobra" + packages = ["."] + revision = "7b2c5ac9fc04fc5efafb60700713d4fa609b777b" + version = "v0.0.1" + +[[projects]] + branch = "master" + name = "github.com/spf13/jwalterweatherman" + packages = ["."] + revision = "7c0cea34c8ece3fbeb2b27ab9b59511d360fb394" + +[[projects]] + name = "github.com/spf13/pflag" + packages = ["."] + revision = "97afa5e7ca8a08a383cb259e06636b5e2cc7897f" + +[[projects]] + name = "github.com/spf13/viper" + packages = ["."] + revision = "25b30aa063fc18e48662b86996252eabdcf2f0c7" + version = "v1.0.0" + +[[projects]] + name = "github.com/stretchr/testify" + packages = [ + "assert", + "require" + ] + revision = "12b6f73e6084dad08a7c6e575284b177ecafbc71" + version = "v1.2.1" + +[[projects]] + name = "github.com/syndtr/goleveldb" + packages = [ + "leveldb", + "leveldb/cache", + "leveldb/comparer", + "leveldb/errors", + "leveldb/filter", + "leveldb/iterator", + "leveldb/journal", + "leveldb/memdb", + "leveldb/opt", + "leveldb/storage", + "leveldb/table", + "leveldb/util" + ] + revision = "b89cc31ef7977104127d34c1bd31ebd1a9db2199" + +[[projects]] + branch = "master" + name = "golang.org/x/net" + packages = [ + "context", + "http/httpguts", + "http2", + "http2/hpack", + "idna", + "internal/timeseries", + "trace" + ] + revision = "d11bb6cd8e3c4e60239c9cb20ef68586d74500d0" + +[[projects]] + name = "golang.org/x/sys" + packages = ["unix"] + revision = "37707fdb30a5b38865cfb95e5aab41707daec7fd" + +[[projects]] + name = "golang.org/x/text" + packages = [ + "collate", + "collate/build", + "internal/colltab", + "internal/gen", + "internal/tag", + "internal/triegen", + "internal/ucd", + "language", + "secure/bidirule", + "transform", + "unicode/bidi", + "unicode/cldr", + "unicode/norm", + "unicode/rangetable" + ] + revision = "c01e4764d870b77f8abe5096ee19ad20d80e8075" + +[[projects]] + branch = "master" + name = "google.golang.org/genproto" + packages = ["googleapis/rpc/status"] + revision = "86e600f69ee4704c6efbf6a2a40a5c10700e76c2" + +[[projects]] + name = "google.golang.org/grpc" + packages = [ + ".", + "balancer", + "balancer/base", + "balancer/roundrobin", + "codes", + "connectivity", + "credentials", + "encoding", + "encoding/proto", + "grpclb/grpc_lb_v1/messages", + "grpclog", + "internal", + "keepalive", + "metadata", + "naming", + "peer", + "resolver", + "resolver/dns", + "resolver/passthrough", + "stats", + "status", + "tap", + "transport" + ] + revision = "d11072e7ca9811b1100b80ca0269ac831f06d024" + version = "v1.11.3" + +[[projects]] + name = "gopkg.in/yaml.v2" + packages = ["."] + revision = "d670f9405373e636a5a2765eea47fac0c9bc91a4" + version = "v2.0.0" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + inputs-digest = "e0c0af880b57928787ea78a820abefd2759e6aee4cba18e67ab36b80e62ad581" + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/libs/Gopkg.toml b/libs/Gopkg.toml new file mode 100644 index 000000000..ff42087f8 --- /dev/null +++ b/libs/Gopkg.toml @@ -0,0 +1,69 @@ +# Gopkg.toml example +# +# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md +# for detailed Gopkg.toml documentation. +# +# required = ["github.com/user/thing/cmd/thing"] +# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] +# +# [[constraint]] +# name = "github.com/user/project" +# version = "1.0.0" +# +# [[constraint]] +# name = "github.com/user/project2" +# branch = "dev" +# source = "github.com/myfork/project2" +# +# [[override]] +# name = "github.com/x/y" +# version = "2.4.0" +# +# [prune] +# non-go = false +# go-tests = true +# unused-packages = true + + +[[constraint]] + branch = "master" + name = "github.com/fortytw2/leaktest" + +[[constraint]] + name = "github.com/go-kit/kit" + version = "0.6.0" + +[[constraint]] + name = "github.com/go-logfmt/logfmt" + version = "0.3.0" + +[[constraint]] + name = "github.com/gogo/protobuf" + version = "1.0.0" + +[[constraint]] + branch = "master" + name = "github.com/jmhodges/levigo" + +[[constraint]] + name = "github.com/pkg/errors" + version = "0.8.0" + +[[constraint]] + name = "github.com/spf13/cobra" + version = "0.0.1" + +[[constraint]] + name = "github.com/spf13/viper" + version = "1.0.0" + +[[constraint]] + name = "github.com/stretchr/testify" + version = "1.2.1" + +[[constraint]] + name = "github.com/btcsuite/btcutil" + branch ="master" +[prune] + go-tests = true + unused-packages = true diff --git a/libs/LICENSE b/libs/LICENSE new file mode 100644 index 000000000..06bc5e1c6 --- /dev/null +++ b/libs/LICENSE @@ -0,0 +1,193 @@ +Tendermint Libraries +Copyright (C) 2017 Tendermint + + + + Apache License + Version 2.0, January 2004 + https://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/libs/Makefile b/libs/Makefile new file mode 100644 index 000000000..a55bc139c --- /dev/null +++ b/libs/Makefile @@ -0,0 +1,137 @@ +GOTOOLS = \ + github.com/golang/dep/cmd/dep \ + github.com/golang/protobuf/protoc-gen-go \ + github.com/square/certstrap + # github.com/alecthomas/gometalinter.v2 \ + +GOTOOLS_CHECK = dep gometalinter.v2 protoc protoc-gen-go +INCLUDE = -I=. -I=${GOPATH}/src + +all: check get_vendor_deps protoc grpc_dbserver build test install metalinter + +check: check_tools + +######################################## +### Build + +protoc: + ## If you get the following error, + ## "error while loading shared libraries: libprotobuf.so.14: cannot open shared object file: No such file or directory" + ## See https://stackoverflow.com/a/25518702 + protoc $(INCLUDE) --go_out=plugins=grpc:. common/*.proto + @echo "--> adding nolint declarations to protobuf generated files" + @awk '/package common/ { print "//nolint: gas"; print; next }1' common/types.pb.go > common/types.pb.go.new + @mv common/types.pb.go.new common/types.pb.go + +build: + # Nothing to build! + +install: + # Nothing to install! + + +######################################## +### Tools & dependencies + +check_tools: + @# https://stackoverflow.com/a/25668869 + @echo "Found tools: $(foreach tool,$(GOTOOLS_CHECK),\ + $(if $(shell which $(tool)),$(tool),$(error "No $(tool) in PATH")))" + +get_tools: + @echo "--> Installing tools" + go get -u -v $(GOTOOLS) + # @gometalinter.v2 --install + +get_protoc: + @# https://github.com/google/protobuf/releases + curl -L https://github.com/google/protobuf/releases/download/v3.4.1/protobuf-cpp-3.4.1.tar.gz | tar xvz && \ + cd protobuf-3.4.1 && \ + DIST_LANG=cpp ./configure && \ + make && \ + make install && \ + cd .. && \ + rm -rf protobuf-3.4.1 + +update_tools: + @echo "--> Updating tools" + @go get -u $(GOTOOLS) + +get_vendor_deps: + @rm -rf vendor/ + @echo "--> Running dep ensure" + @dep ensure + + +######################################## +### Testing + +gen_certs: clean_certs + ## Generating certificates for TLS testing... + certstrap init --common-name "tendermint.com" --passphrase "" + certstrap request-cert -ip "::" --passphrase "" + certstrap sign "::" --CA "tendermint.com" --passphrase "" + mv out/::.crt out/::.key db/remotedb + +clean_certs: + ## Cleaning TLS testing certificates... + rm -rf out + rm -f db/remotedb/::.crt db/remotedb/::.key + +test: gen_certs + GOCACHE=off go test -tags gcc $(shell go list ./... | grep -v vendor) + make clean_certs + +test100: + @for i in {1..100}; do make test; done + + +######################################## +### Formatting, linting, and vetting + +fmt: + @go fmt ./... + +metalinter: + @echo "==> Running linter" + gometalinter.v2 --vendor --deadline=600s --disable-all \ + --enable=deadcode \ + --enable=goconst \ + --enable=goimports \ + --enable=gosimple \ + --enable=ineffassign \ + --enable=megacheck \ + --enable=misspell \ + --enable=staticcheck \ + --enable=safesql \ + --enable=structcheck \ + --enable=unconvert \ + --enable=unused \ + --enable=varcheck \ + --enable=vetshadow \ + ./... + + #--enable=maligned \ + #--enable=gas \ + #--enable=aligncheck \ + #--enable=dupl \ + #--enable=errcheck \ + #--enable=gocyclo \ + #--enable=golint \ <== comments on anything exported + #--enable=gotype \ + #--enable=interfacer \ + #--enable=unparam \ + #--enable=vet \ + +metalinter_all: + protoc $(INCLUDE) --lint_out=. types/*.proto + gometalinter.v2 --vendor --deadline=600s --enable-all --disable=lll ./... + + +# To avoid unintended conflicts with file names, always add to .PHONY +# unless there is a reason not to. +# https://www.gnu.org/software/make/manual/html_node/Phony-Targets.html +.PHONY: check protoc build check_tools get_tools get_protoc update_tools get_vendor_deps test fmt metalinter metalinter_all gen_certs clean_certs + +grpc_dbserver: + protoc -I db/remotedb/proto/ db/remotedb/proto/defs.proto --go_out=plugins=grpc:db/remotedb/proto diff --git a/libs/README.md b/libs/README.md new file mode 100644 index 000000000..9ea618dbd --- /dev/null +++ b/libs/README.md @@ -0,0 +1,49 @@ +# TMLIBS + +This repo is a home for various small packages. + +## autofile + +Autofile is file access with automatic log rotation. A group of files is maintained and rotation happens +when the leading file gets too big. Provides a reader for reading from the file group. + +## cli + +CLI wraps the `cobra` and `viper` packages and handles some common elements of building a CLI like flags and env vars for the home directory and the logger. + +## clist + +Clist provides a linekd list that is safe for concurrent access by many readers. + +## common + +Common provides a hodgepodge of useful functions. + +## db + +DB provides a database interface and a number of implementions, including ones using an in-memory map, the filesystem directory structure, +an implemention of LevelDB in Go, and the official LevelDB in C. + +## events + +Events is a synchronous PubSub package. + +## flowrate + +Flowrate is a fork of https://github.com/mxk/go-flowrate that added a `SetREMA` method. + +## log + +Log is a log package structured around key-value pairs that allows logging level to be set differently for different keys. + +## merkle + +Merkle provides a simple static merkle tree and corresponding proofs. + +## process + +Process is a simple utility for spawning OS processes. + +## pubsub + +PubSub is an asynchronous PubSub package. diff --git a/libs/autofile/README.md b/libs/autofile/README.md new file mode 100644 index 000000000..23799200c --- /dev/null +++ b/libs/autofile/README.md @@ -0,0 +1 @@ +# go-autofile diff --git a/libs/autofile/autofile.go b/libs/autofile/autofile.go new file mode 100644 index 000000000..313da6789 --- /dev/null +++ b/libs/autofile/autofile.go @@ -0,0 +1,142 @@ +package autofile + +import ( + "os" + "sync" + "time" + + cmn "github.com/tendermint/tendermint/libs/common" +) + +/* AutoFile usage + +// Create/Append to ./autofile_test +af, err := OpenAutoFile("autofile_test") +if err != nil { + panic(err) +} + +// Stream of writes. +// During this time, the file may be moved e.g. by logRotate. +for i := 0; i < 60; i++ { + af.Write([]byte(Fmt("LOOP(%v)", i))) + time.Sleep(time.Second) +} + +// Close the AutoFile +err = af.Close() +if err != nil { + panic(err) +} +*/ + +const autoFileOpenDuration = 1000 * time.Millisecond + +// Automatically closes and re-opens file for writing. +// This is useful for using a log file with the logrotate tool. +type AutoFile struct { + ID string + Path string + ticker *time.Ticker + mtx sync.Mutex + file *os.File +} + +func OpenAutoFile(path string) (af *AutoFile, err error) { + af = &AutoFile{ + ID: cmn.RandStr(12) + ":" + path, + Path: path, + ticker: time.NewTicker(autoFileOpenDuration), + } + if err = af.openFile(); err != nil { + return + } + go af.processTicks() + sighupWatchers.addAutoFile(af) + return +} + +func (af *AutoFile) Close() error { + af.ticker.Stop() + err := af.closeFile() + sighupWatchers.removeAutoFile(af) + return err +} + +func (af *AutoFile) processTicks() { + for { + _, ok := <-af.ticker.C + if !ok { + return // Done. + } + af.closeFile() + } +} + +func (af *AutoFile) closeFile() (err error) { + af.mtx.Lock() + defer af.mtx.Unlock() + + file := af.file + if file == nil { + return nil + } + af.file = nil + return file.Close() +} + +func (af *AutoFile) Write(b []byte) (n int, err error) { + af.mtx.Lock() + defer af.mtx.Unlock() + + if af.file == nil { + if err = af.openFile(); err != nil { + return + } + } + + n, err = af.file.Write(b) + return +} + +func (af *AutoFile) Sync() error { + af.mtx.Lock() + defer af.mtx.Unlock() + + if af.file == nil { + if err := af.openFile(); err != nil { + return err + } + } + return af.file.Sync() +} + +func (af *AutoFile) openFile() error { + file, err := os.OpenFile(af.Path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) + if err != nil { + return err + } + af.file = file + return nil +} + +func (af *AutoFile) Size() (int64, error) { + af.mtx.Lock() + defer af.mtx.Unlock() + + if af.file == nil { + err := af.openFile() + if err != nil { + if err == os.ErrNotExist { + return 0, nil + } + return -1, err + } + } + stat, err := af.file.Stat() + if err != nil { + return -1, err + } + return stat.Size(), nil + +} diff --git a/libs/autofile/autofile_test.go b/libs/autofile/autofile_test.go new file mode 100644 index 000000000..b39fb7cf3 --- /dev/null +++ b/libs/autofile/autofile_test.go @@ -0,0 +1,71 @@ +package autofile + +import ( + "os" + "sync/atomic" + "syscall" + "testing" + "time" + + cmn "github.com/tendermint/tendermint/libs/common" +) + +func TestSIGHUP(t *testing.T) { + + // First, create an AutoFile writing to a tempfile dir + file, name := cmn.Tempfile("sighup_test") + if err := file.Close(); err != nil { + t.Fatalf("Error creating tempfile: %v", err) + } + // Here is the actual AutoFile + af, err := OpenAutoFile(name) + if err != nil { + t.Fatalf("Error creating autofile: %v", err) + } + + // Write to the file. + _, err = af.Write([]byte("Line 1\n")) + if err != nil { + t.Fatalf("Error writing to autofile: %v", err) + } + _, err = af.Write([]byte("Line 2\n")) + if err != nil { + t.Fatalf("Error writing to autofile: %v", err) + } + + // Move the file over + err = os.Rename(name, name+"_old") + if err != nil { + t.Fatalf("Error moving autofile: %v", err) + } + + // Send SIGHUP to self. + oldSighupCounter := atomic.LoadInt32(&sighupCounter) + syscall.Kill(syscall.Getpid(), syscall.SIGHUP) + + // Wait a bit... signals are not handled synchronously. + for atomic.LoadInt32(&sighupCounter) == oldSighupCounter { + time.Sleep(time.Millisecond * 10) + } + + // Write more to the file. + _, err = af.Write([]byte("Line 3\n")) + if err != nil { + t.Fatalf("Error writing to autofile: %v", err) + } + _, err = af.Write([]byte("Line 4\n")) + if err != nil { + t.Fatalf("Error writing to autofile: %v", err) + } + if err := af.Close(); err != nil { + t.Fatalf("Error closing autofile") + } + + // Both files should exist + if body := cmn.MustReadFile(name + "_old"); string(body) != "Line 1\nLine 2\n" { + t.Errorf("Unexpected body %s", body) + } + if body := cmn.MustReadFile(name); string(body) != "Line 3\nLine 4\n" { + t.Errorf("Unexpected body %s", body) + } +} diff --git a/libs/autofile/cmd/logjack.go b/libs/autofile/cmd/logjack.go new file mode 100644 index 000000000..17b482bed --- /dev/null +++ b/libs/autofile/cmd/logjack.go @@ -0,0 +1,107 @@ +package main + +import ( + "flag" + "fmt" + "io" + "os" + "strconv" + "strings" + + auto "github.com/tendermint/tendermint/libs/autofile" + cmn "github.com/tendermint/tendermint/libs/common" +) + +const Version = "0.0.1" +const readBufferSize = 1024 // 1KB at a time + +// Parse command-line options +func parseFlags() (headPath string, chopSize int64, limitSize int64, version bool) { + var flagSet = flag.NewFlagSet(os.Args[0], flag.ExitOnError) + var chopSizeStr, limitSizeStr string + flagSet.StringVar(&headPath, "head", "logjack.out", "Destination (head) file.") + flagSet.StringVar(&chopSizeStr, "chop", "100M", "Move file if greater than this") + flagSet.StringVar(&limitSizeStr, "limit", "10G", "Only keep this much (for each specified file). Remove old files.") + flagSet.BoolVar(&version, "version", false, "Version") + flagSet.Parse(os.Args[1:]) + chopSize = parseBytesize(chopSizeStr) + limitSize = parseBytesize(limitSizeStr) + return +} + +func main() { + + // Read options + headPath, chopSize, limitSize, version := parseFlags() + if version { + fmt.Printf("logjack version %v\n", Version) + return + } + + // Open Group + group, err := auto.OpenGroup(headPath) + if err != nil { + fmt.Printf("logjack couldn't create output file %v\n", headPath) + os.Exit(1) + } + group.SetHeadSizeLimit(chopSize) + group.SetTotalSizeLimit(limitSize) + err = group.Start() + if err != nil { + fmt.Printf("logjack couldn't start with file %v\n", headPath) + os.Exit(1) + } + + go func() { + // Forever, read from stdin and write to AutoFile. + buf := make([]byte, readBufferSize) + for { + n, err := os.Stdin.Read(buf) + group.Write(buf[:n]) + group.Flush() + if err != nil { + group.Stop() + if err == io.EOF { + os.Exit(0) + } else { + fmt.Println("logjack errored") + os.Exit(1) + } + } + } + }() + + // Trap signal + cmn.TrapSignal(func() { + fmt.Println("logjack shutting down") + }) +} + +func parseBytesize(chopSize string) int64 { + // Handle suffix multiplier + var multiplier int64 = 1 + if strings.HasSuffix(chopSize, "T") { + multiplier = 1042 * 1024 * 1024 * 1024 + chopSize = chopSize[:len(chopSize)-1] + } + if strings.HasSuffix(chopSize, "G") { + multiplier = 1042 * 1024 * 1024 + chopSize = chopSize[:len(chopSize)-1] + } + if strings.HasSuffix(chopSize, "M") { + multiplier = 1042 * 1024 + chopSize = chopSize[:len(chopSize)-1] + } + if strings.HasSuffix(chopSize, "K") { + multiplier = 1042 + chopSize = chopSize[:len(chopSize)-1] + } + + // Parse the numeric part + chopSizeInt, err := strconv.Atoi(chopSize) + if err != nil { + panic(err) + } + + return int64(chopSizeInt) * multiplier +} diff --git a/libs/autofile/group.go b/libs/autofile/group.go new file mode 100644 index 000000000..b4368ed9e --- /dev/null +++ b/libs/autofile/group.go @@ -0,0 +1,747 @@ +package autofile + +import ( + "bufio" + "errors" + "fmt" + "io" + "log" + "os" + "path" + "path/filepath" + "regexp" + "strconv" + "strings" + "sync" + "time" + + cmn "github.com/tendermint/tendermint/libs/common" +) + +const ( + groupCheckDuration = 5000 * time.Millisecond + defaultHeadSizeLimit = 10 * 1024 * 1024 // 10MB + defaultTotalSizeLimit = 1 * 1024 * 1024 * 1024 // 1GB + maxFilesToRemove = 4 // needs to be greater than 1 +) + +/* +You can open a Group to keep restrictions on an AutoFile, like +the maximum size of each chunk, and/or the total amount of bytes +stored in the group. + +The first file to be written in the Group.Dir is the head file. + + Dir/ + - + +Once the Head file reaches the size limit, it will be rotated. + + Dir/ + - .000 // First rolled file + - // New head path, starts empty. + // The implicit index is 001. + +As more files are written, the index numbers grow... + + Dir/ + - .000 // First rolled file + - .001 // Second rolled file + - ... + - // New head path + +The Group can also be used to binary-search for some line, +assuming that marker lines are written occasionally. +*/ +type Group struct { + cmn.BaseService + + ID string + Head *AutoFile // The head AutoFile to write to + headBuf *bufio.Writer + Dir string // Directory that contains .Head + ticker *time.Ticker + mtx sync.Mutex + headSizeLimit int64 + totalSizeLimit int64 + minIndex int // Includes head + maxIndex int // Includes head, where Head will move to + + // TODO: When we start deleting files, we need to start tracking GroupReaders + // and their dependencies. +} + +// OpenGroup creates a new Group with head at headPath. It returns an error if +// it fails to open head file. +func OpenGroup(headPath string) (g *Group, err error) { + dir := path.Dir(headPath) + head, err := OpenAutoFile(headPath) + if err != nil { + return nil, err + } + + g = &Group{ + ID: "group:" + head.ID, + Head: head, + headBuf: bufio.NewWriterSize(head, 4096*10), + Dir: dir, + ticker: time.NewTicker(groupCheckDuration), + headSizeLimit: defaultHeadSizeLimit, + totalSizeLimit: defaultTotalSizeLimit, + minIndex: 0, + maxIndex: 0, + } + g.BaseService = *cmn.NewBaseService(nil, "Group", g) + + gInfo := g.readGroupInfo() + g.minIndex = gInfo.MinIndex + g.maxIndex = gInfo.MaxIndex + return +} + +// OnStart implements Service by starting the goroutine that checks file and +// group limits. +func (g *Group) OnStart() error { + go g.processTicks() + return nil +} + +// OnStop implements Service by stopping the goroutine described above. +// NOTE: g.Head must be closed separately using Close. +func (g *Group) OnStop() { + g.ticker.Stop() + g.Flush() // flush any uncommitted data +} + +// Close closes the head file. The group must be stopped by this moment. +func (g *Group) Close() { + g.Flush() // flush any uncommitted data + + g.mtx.Lock() + _ = g.Head.closeFile() + g.mtx.Unlock() +} + +// SetHeadSizeLimit allows you to overwrite default head size limit - 10MB. +func (g *Group) SetHeadSizeLimit(limit int64) { + g.mtx.Lock() + g.headSizeLimit = limit + g.mtx.Unlock() +} + +// HeadSizeLimit returns the current head size limit. +func (g *Group) HeadSizeLimit() int64 { + g.mtx.Lock() + defer g.mtx.Unlock() + return g.headSizeLimit +} + +// SetTotalSizeLimit allows you to overwrite default total size limit of the +// group - 1GB. +func (g *Group) SetTotalSizeLimit(limit int64) { + g.mtx.Lock() + g.totalSizeLimit = limit + g.mtx.Unlock() +} + +// TotalSizeLimit returns total size limit of the group. +func (g *Group) TotalSizeLimit() int64 { + g.mtx.Lock() + defer g.mtx.Unlock() + return g.totalSizeLimit +} + +// MaxIndex returns index of the last file in the group. +func (g *Group) MaxIndex() int { + g.mtx.Lock() + defer g.mtx.Unlock() + return g.maxIndex +} + +// MinIndex returns index of the first file in the group. +func (g *Group) MinIndex() int { + g.mtx.Lock() + defer g.mtx.Unlock() + return g.minIndex +} + +// Write writes the contents of p into the current head of the group. It +// returns the number of bytes written. If nn < len(p), it also returns an +// error explaining why the write is short. +// NOTE: Writes are buffered so they don't write synchronously +// TODO: Make it halt if space is unavailable +func (g *Group) Write(p []byte) (nn int, err error) { + g.mtx.Lock() + defer g.mtx.Unlock() + return g.headBuf.Write(p) +} + +// WriteLine writes line into the current head of the group. It also appends "\n". +// NOTE: Writes are buffered so they don't write synchronously +// TODO: Make it halt if space is unavailable +func (g *Group) WriteLine(line string) error { + g.mtx.Lock() + defer g.mtx.Unlock() + _, err := g.headBuf.Write([]byte(line + "\n")) + return err +} + +// Flush writes any buffered data to the underlying file and commits the +// current content of the file to stable storage. +func (g *Group) Flush() error { + g.mtx.Lock() + defer g.mtx.Unlock() + err := g.headBuf.Flush() + if err == nil { + err = g.Head.Sync() + } + return err +} + +func (g *Group) processTicks() { + for { + _, ok := <-g.ticker.C + if !ok { + return // Done. + } + g.checkHeadSizeLimit() + g.checkTotalSizeLimit() + } +} + +// NOTE: for testing +func (g *Group) stopTicker() { + g.ticker.Stop() +} + +// NOTE: this function is called manually in tests. +func (g *Group) checkHeadSizeLimit() { + limit := g.HeadSizeLimit() + if limit == 0 { + return + } + size, err := g.Head.Size() + if err != nil { + panic(err) + } + if size >= limit { + g.RotateFile() + } +} + +func (g *Group) checkTotalSizeLimit() { + limit := g.TotalSizeLimit() + if limit == 0 { + return + } + + gInfo := g.readGroupInfo() + totalSize := gInfo.TotalSize + for i := 0; i < maxFilesToRemove; i++ { + index := gInfo.MinIndex + i + if totalSize < limit { + return + } + if index == gInfo.MaxIndex { + // Special degenerate case, just do nothing. + log.Println("WARNING: Group's head " + g.Head.Path + "may grow without bound") + return + } + pathToRemove := filePathForIndex(g.Head.Path, index, gInfo.MaxIndex) + fileInfo, err := os.Stat(pathToRemove) + if err != nil { + log.Println("WARNING: Failed to fetch info for file @" + pathToRemove) + continue + } + err = os.Remove(pathToRemove) + if err != nil { + log.Println(err) + return + } + totalSize -= fileInfo.Size() + } +} + +// RotateFile causes group to close the current head and assign it some index. +// Note it does not create a new head. +func (g *Group) RotateFile() { + g.mtx.Lock() + defer g.mtx.Unlock() + + headPath := g.Head.Path + + if err := g.Head.closeFile(); err != nil { + panic(err) + } + + indexPath := filePathForIndex(headPath, g.maxIndex, g.maxIndex+1) + if err := os.Rename(headPath, indexPath); err != nil { + panic(err) + } + + g.maxIndex++ +} + +// NewReader returns a new group reader. +// CONTRACT: Caller must close the returned GroupReader. +func (g *Group) NewReader(index int) (*GroupReader, error) { + r := newGroupReader(g) + err := r.SetIndex(index) + if err != nil { + return nil, err + } + return r, nil +} + +// Returns -1 if line comes after, 0 if found, 1 if line comes before. +type SearchFunc func(line string) (int, error) + +// Searches for the right file in Group, then returns a GroupReader to start +// streaming lines. +// Returns true if an exact match was found, otherwise returns the next greater +// line that starts with prefix. +// CONTRACT: Caller must close the returned GroupReader +func (g *Group) Search(prefix string, cmp SearchFunc) (*GroupReader, bool, error) { + g.mtx.Lock() + minIndex, maxIndex := g.minIndex, g.maxIndex + g.mtx.Unlock() + // Now minIndex/maxIndex may change meanwhile, + // but it shouldn't be a big deal + // (maybe we'll want to limit scanUntil though) + + for { + curIndex := (minIndex + maxIndex + 1) / 2 + + // Base case, when there's only 1 choice left. + if minIndex == maxIndex { + r, err := g.NewReader(maxIndex) + if err != nil { + return nil, false, err + } + match, err := scanUntil(r, prefix, cmp) + if err != nil { + r.Close() + return nil, false, err + } + return r, match, err + } + + // Read starting roughly at the middle file, + // until we find line that has prefix. + r, err := g.NewReader(curIndex) + if err != nil { + return nil, false, err + } + foundIndex, line, err := scanNext(r, prefix) + r.Close() + if err != nil { + return nil, false, err + } + + // Compare this line to our search query. + val, err := cmp(line) + if err != nil { + return nil, false, err + } + if val < 0 { + // Line will come later + minIndex = foundIndex + } else if val == 0 { + // Stroke of luck, found the line + r, err := g.NewReader(foundIndex) + if err != nil { + return nil, false, err + } + match, err := scanUntil(r, prefix, cmp) + if !match { + panic("Expected match to be true") + } + if err != nil { + r.Close() + return nil, false, err + } + return r, true, err + } else { + // We passed it + maxIndex = curIndex - 1 + } + } + +} + +// Scans and returns the first line that starts with 'prefix' +// Consumes line and returns it. +func scanNext(r *GroupReader, prefix string) (int, string, error) { + for { + line, err := r.ReadLine() + if err != nil { + return 0, "", err + } + if !strings.HasPrefix(line, prefix) { + continue + } + index := r.CurIndex() + return index, line, nil + } +} + +// Returns true iff an exact match was found. +// Pushes line, does not consume it. +func scanUntil(r *GroupReader, prefix string, cmp SearchFunc) (bool, error) { + for { + line, err := r.ReadLine() + if err != nil { + return false, err + } + if !strings.HasPrefix(line, prefix) { + continue + } + val, err := cmp(line) + if err != nil { + return false, err + } + if val < 0 { + continue + } else if val == 0 { + r.PushLine(line) + return true, nil + } else { + r.PushLine(line) + return false, nil + } + } +} + +// Searches backwards for the last line in Group with prefix. +// Scans each file forward until the end to find the last match. +func (g *Group) FindLast(prefix string) (match string, found bool, err error) { + g.mtx.Lock() + minIndex, maxIndex := g.minIndex, g.maxIndex + g.mtx.Unlock() + + r, err := g.NewReader(maxIndex) + if err != nil { + return "", false, err + } + defer r.Close() + + // Open files from the back and read +GROUP_LOOP: + for i := maxIndex; i >= minIndex; i-- { + err := r.SetIndex(i) + if err != nil { + return "", false, err + } + // Scan each line and test whether line matches + for { + line, err := r.ReadLine() + if err == io.EOF { + if found { + return match, found, nil + } + continue GROUP_LOOP + } else if err != nil { + return "", false, err + } + if strings.HasPrefix(line, prefix) { + match = line + found = true + } + if r.CurIndex() > i { + if found { + return match, found, nil + } + continue GROUP_LOOP + } + } + } + + return +} + +// GroupInfo holds information about the group. +type GroupInfo struct { + MinIndex int // index of the first file in the group, including head + MaxIndex int // index of the last file in the group, including head + TotalSize int64 // total size of the group + HeadSize int64 // size of the head +} + +// Returns info after scanning all files in g.Head's dir. +func (g *Group) ReadGroupInfo() GroupInfo { + g.mtx.Lock() + defer g.mtx.Unlock() + return g.readGroupInfo() +} + +// Index includes the head. +// CONTRACT: caller should have called g.mtx.Lock +func (g *Group) readGroupInfo() GroupInfo { + groupDir := filepath.Dir(g.Head.Path) + headBase := filepath.Base(g.Head.Path) + var minIndex, maxIndex int = -1, -1 + var totalSize, headSize int64 = 0, 0 + + dir, err := os.Open(groupDir) + if err != nil { + panic(err) + } + defer dir.Close() + fiz, err := dir.Readdir(0) + if err != nil { + panic(err) + } + + // For each file in the directory, filter by pattern + for _, fileInfo := range fiz { + if fileInfo.Name() == headBase { + fileSize := fileInfo.Size() + totalSize += fileSize + headSize = fileSize + continue + } else if strings.HasPrefix(fileInfo.Name(), headBase) { + fileSize := fileInfo.Size() + totalSize += fileSize + indexedFilePattern := regexp.MustCompile(`^.+\.([0-9]{3,})$`) + submatch := indexedFilePattern.FindSubmatch([]byte(fileInfo.Name())) + if len(submatch) != 0 { + // Matches + fileIndex, err := strconv.Atoi(string(submatch[1])) + if err != nil { + panic(err) + } + if maxIndex < fileIndex { + maxIndex = fileIndex + } + if minIndex == -1 || fileIndex < minIndex { + minIndex = fileIndex + } + } + } + } + + // Now account for the head. + if minIndex == -1 { + // If there were no numbered files, + // then the head is index 0. + minIndex, maxIndex = 0, 0 + } else { + // Otherwise, the head file is 1 greater + maxIndex++ + } + return GroupInfo{minIndex, maxIndex, totalSize, headSize} +} + +func filePathForIndex(headPath string, index int, maxIndex int) string { + if index == maxIndex { + return headPath + } + return fmt.Sprintf("%v.%03d", headPath, index) +} + +//-------------------------------------------------------------------------------- + +// GroupReader provides an interface for reading from a Group. +type GroupReader struct { + *Group + mtx sync.Mutex + curIndex int + curFile *os.File + curReader *bufio.Reader + curLine []byte +} + +func newGroupReader(g *Group) *GroupReader { + return &GroupReader{ + Group: g, + curIndex: 0, + curFile: nil, + curReader: nil, + curLine: nil, + } +} + +// Close closes the GroupReader by closing the cursor file. +func (gr *GroupReader) Close() error { + gr.mtx.Lock() + defer gr.mtx.Unlock() + + if gr.curReader != nil { + err := gr.curFile.Close() + gr.curIndex = 0 + gr.curReader = nil + gr.curFile = nil + gr.curLine = nil + return err + } + return nil +} + +// Read implements io.Reader, reading bytes from the current Reader +// incrementing index until enough bytes are read. +func (gr *GroupReader) Read(p []byte) (n int, err error) { + lenP := len(p) + if lenP == 0 { + return 0, errors.New("given empty slice") + } + + gr.mtx.Lock() + defer gr.mtx.Unlock() + + // Open file if not open yet + if gr.curReader == nil { + if err = gr.openFile(gr.curIndex); err != nil { + return 0, err + } + } + + // Iterate over files until enough bytes are read + var nn int + for { + nn, err = gr.curReader.Read(p[n:]) + n += nn + if err == io.EOF { + if n >= lenP { + return n, nil + } + // Open the next file + if err1 := gr.openFile(gr.curIndex + 1); err1 != nil { + return n, err1 + } + } else if err != nil { + return n, err + } else if nn == 0 { // empty file + return n, err + } + } +} + +// ReadLine reads a line (without delimiter). +// just return io.EOF if no new lines found. +func (gr *GroupReader) ReadLine() (string, error) { + gr.mtx.Lock() + defer gr.mtx.Unlock() + + // From PushLine + if gr.curLine != nil { + line := string(gr.curLine) + gr.curLine = nil + return line, nil + } + + // Open file if not open yet + if gr.curReader == nil { + err := gr.openFile(gr.curIndex) + if err != nil { + return "", err + } + } + + // Iterate over files until line is found + var linePrefix string + for { + bytesRead, err := gr.curReader.ReadBytes('\n') + if err == io.EOF { + // Open the next file + if err1 := gr.openFile(gr.curIndex + 1); err1 != nil { + return "", err1 + } + if len(bytesRead) > 0 && bytesRead[len(bytesRead)-1] == byte('\n') { + return linePrefix + string(bytesRead[:len(bytesRead)-1]), nil + } + linePrefix += string(bytesRead) + continue + } else if err != nil { + return "", err + } + return linePrefix + string(bytesRead[:len(bytesRead)-1]), nil + } +} + +// IF index > gr.Group.maxIndex, returns io.EOF +// CONTRACT: caller should hold gr.mtx +func (gr *GroupReader) openFile(index int) error { + + // Lock on Group to ensure that head doesn't move in the meanwhile. + gr.Group.mtx.Lock() + defer gr.Group.mtx.Unlock() + + if index > gr.Group.maxIndex { + return io.EOF + } + + curFilePath := filePathForIndex(gr.Head.Path, index, gr.Group.maxIndex) + curFile, err := os.Open(curFilePath) + if err != nil { + return err + } + curReader := bufio.NewReader(curFile) + + // Update gr.cur* + if gr.curFile != nil { + gr.curFile.Close() // TODO return error? + } + gr.curIndex = index + gr.curFile = curFile + gr.curReader = curReader + gr.curLine = nil + return nil +} + +// PushLine makes the given line the current one, so the next time somebody +// calls ReadLine, this line will be returned. +// panics if called twice without calling ReadLine. +func (gr *GroupReader) PushLine(line string) { + gr.mtx.Lock() + defer gr.mtx.Unlock() + + if gr.curLine == nil { + gr.curLine = []byte(line) + } else { + panic("PushLine failed, already have line") + } +} + +// CurIndex returns cursor's file index. +func (gr *GroupReader) CurIndex() int { + gr.mtx.Lock() + defer gr.mtx.Unlock() + return gr.curIndex +} + +// SetIndex sets the cursor's file index to index by opening a file at this +// position. +func (gr *GroupReader) SetIndex(index int) error { + gr.mtx.Lock() + defer gr.mtx.Unlock() + return gr.openFile(index) +} + +//-------------------------------------------------------------------------------- + +// A simple SearchFunc that assumes that the marker is of form +// . +// For example, if prefix is '#HEIGHT:', the markers of expected to be of the form: +// +// #HEIGHT:1 +// ... +// #HEIGHT:2 +// ... +func MakeSimpleSearchFunc(prefix string, target int) SearchFunc { + return func(line string) (int, error) { + if !strings.HasPrefix(line, prefix) { + return -1, errors.New(cmn.Fmt("Marker line did not have prefix: %v", prefix)) + } + i, err := strconv.Atoi(line[len(prefix):]) + if err != nil { + return -1, errors.New(cmn.Fmt("Failed to parse marker line: %v", err.Error())) + } + if target < i { + return 1, nil + } else if target == i { + return 0, nil + } else { + return -1, nil + } + } +} diff --git a/libs/autofile/group_test.go b/libs/autofile/group_test.go new file mode 100644 index 000000000..72581f9e2 --- /dev/null +++ b/libs/autofile/group_test.go @@ -0,0 +1,438 @@ +package autofile + +import ( + "errors" + "fmt" + "io" + "io/ioutil" + "os" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + cmn "github.com/tendermint/tendermint/libs/common" +) + +// NOTE: Returned group has ticker stopped +func createTestGroup(t *testing.T, headSizeLimit int64) *Group { + testID := cmn.RandStr(12) + testDir := "_test_" + testID + err := cmn.EnsureDir(testDir, 0700) + require.NoError(t, err, "Error creating dir") + headPath := testDir + "/myfile" + g, err := OpenGroup(headPath) + require.NoError(t, err, "Error opening Group") + g.SetHeadSizeLimit(headSizeLimit) + g.stopTicker() + require.NotEqual(t, nil, g, "Failed to create Group") + return g +} + +func destroyTestGroup(t *testing.T, g *Group) { + g.Close() + err := os.RemoveAll(g.Dir) + require.NoError(t, err, "Error removing test Group directory") +} + +func assertGroupInfo(t *testing.T, gInfo GroupInfo, minIndex, maxIndex int, totalSize, headSize int64) { + assert.Equal(t, minIndex, gInfo.MinIndex) + assert.Equal(t, maxIndex, gInfo.MaxIndex) + assert.Equal(t, totalSize, gInfo.TotalSize) + assert.Equal(t, headSize, gInfo.HeadSize) +} + +func TestCheckHeadSizeLimit(t *testing.T) { + g := createTestGroup(t, 1000*1000) + + // At first, there are no files. + assertGroupInfo(t, g.ReadGroupInfo(), 0, 0, 0, 0) + + // Write 1000 bytes 999 times. + for i := 0; i < 999; i++ { + err := g.WriteLine(cmn.RandStr(999)) + require.NoError(t, err, "Error appending to head") + } + g.Flush() + assertGroupInfo(t, g.ReadGroupInfo(), 0, 0, 999000, 999000) + + // Even calling checkHeadSizeLimit manually won't rotate it. + g.checkHeadSizeLimit() + assertGroupInfo(t, g.ReadGroupInfo(), 0, 0, 999000, 999000) + + // Write 1000 more bytes. + err := g.WriteLine(cmn.RandStr(999)) + require.NoError(t, err, "Error appending to head") + g.Flush() + + // Calling checkHeadSizeLimit this time rolls it. + g.checkHeadSizeLimit() + assertGroupInfo(t, g.ReadGroupInfo(), 0, 1, 1000000, 0) + + // Write 1000 more bytes. + err = g.WriteLine(cmn.RandStr(999)) + require.NoError(t, err, "Error appending to head") + g.Flush() + + // Calling checkHeadSizeLimit does nothing. + g.checkHeadSizeLimit() + assertGroupInfo(t, g.ReadGroupInfo(), 0, 1, 1001000, 1000) + + // Write 1000 bytes 999 times. + for i := 0; i < 999; i++ { + err = g.WriteLine(cmn.RandStr(999)) + require.NoError(t, err, "Error appending to head") + } + g.Flush() + assertGroupInfo(t, g.ReadGroupInfo(), 0, 1, 2000000, 1000000) + + // Calling checkHeadSizeLimit rolls it again. + g.checkHeadSizeLimit() + assertGroupInfo(t, g.ReadGroupInfo(), 0, 2, 2000000, 0) + + // Write 1000 more bytes. + _, err = g.Head.Write([]byte(cmn.RandStr(999) + "\n")) + require.NoError(t, err, "Error appending to head") + g.Flush() + assertGroupInfo(t, g.ReadGroupInfo(), 0, 2, 2001000, 1000) + + // Calling checkHeadSizeLimit does nothing. + g.checkHeadSizeLimit() + assertGroupInfo(t, g.ReadGroupInfo(), 0, 2, 2001000, 1000) + + // Cleanup + destroyTestGroup(t, g) +} + +func TestSearch(t *testing.T) { + g := createTestGroup(t, 10*1000) + + // Create some files in the group that have several INFO lines in them. + // Try to put the INFO lines in various spots. + for i := 0; i < 100; i++ { + // The random junk at the end ensures that this INFO linen + // is equally likely to show up at the end. + _, err := g.Head.Write([]byte(fmt.Sprintf("INFO %v %v\n", i, cmn.RandStr(123)))) + require.NoError(t, err, "Failed to write to head") + g.checkHeadSizeLimit() + for j := 0; j < 10; j++ { + _, err1 := g.Head.Write([]byte(cmn.RandStr(123) + "\n")) + require.NoError(t, err1, "Failed to write to head") + g.checkHeadSizeLimit() + } + } + + // Create a search func that searches for line + makeSearchFunc := func(target int) SearchFunc { + return func(line string) (int, error) { + parts := strings.Split(line, " ") + if len(parts) != 3 { + return -1, errors.New("Line did not have 3 parts") + } + i, err := strconv.Atoi(parts[1]) + if err != nil { + return -1, errors.New("Failed to parse INFO: " + err.Error()) + } + if target < i { + return 1, nil + } else if target == i { + return 0, nil + } else { + return -1, nil + } + } + } + + // Now search for each number + for i := 0; i < 100; i++ { + t.Log("Testing for i", i) + gr, match, err := g.Search("INFO", makeSearchFunc(i)) + require.NoError(t, err, "Failed to search for line") + assert.True(t, match, "Expected Search to return exact match") + line, err := gr.ReadLine() + require.NoError(t, err, "Failed to read line after search") + if !strings.HasPrefix(line, fmt.Sprintf("INFO %v ", i)) { + t.Fatal("Failed to get correct line") + } + // Make sure we can continue to read from there. + cur := i + 1 + for { + line, err := gr.ReadLine() + if err == io.EOF { + if cur == 99+1 { + // OK! + break + } else { + t.Fatal("Got EOF after the wrong INFO #") + } + } else if err != nil { + t.Fatal("Error reading line", err) + } + if !strings.HasPrefix(line, "INFO ") { + continue + } + if !strings.HasPrefix(line, fmt.Sprintf("INFO %v ", cur)) { + t.Fatalf("Unexpected INFO #. Expected %v got:\n%v", cur, line) + } + cur++ + } + gr.Close() + } + + // Now search for something that is too small. + // We should get the first available line. + { + gr, match, err := g.Search("INFO", makeSearchFunc(-999)) + require.NoError(t, err, "Failed to search for line") + assert.False(t, match, "Expected Search to not return exact match") + line, err := gr.ReadLine() + require.NoError(t, err, "Failed to read line after search") + if !strings.HasPrefix(line, "INFO 0 ") { + t.Error("Failed to fetch correct line, which is the earliest INFO") + } + err = gr.Close() + require.NoError(t, err, "Failed to close GroupReader") + } + + // Now search for something that is too large. + // We should get an EOF error. + { + gr, _, err := g.Search("INFO", makeSearchFunc(999)) + assert.Equal(t, io.EOF, err) + assert.Nil(t, gr) + } + + // Cleanup + destroyTestGroup(t, g) +} + +func TestRotateFile(t *testing.T) { + g := createTestGroup(t, 0) + g.WriteLine("Line 1") + g.WriteLine("Line 2") + g.WriteLine("Line 3") + g.Flush() + g.RotateFile() + g.WriteLine("Line 4") + g.WriteLine("Line 5") + g.WriteLine("Line 6") + g.Flush() + + // Read g.Head.Path+"000" + body1, err := ioutil.ReadFile(g.Head.Path + ".000") + assert.NoError(t, err, "Failed to read first rolled file") + if string(body1) != "Line 1\nLine 2\nLine 3\n" { + t.Errorf("Got unexpected contents: [%v]", string(body1)) + } + + // Read g.Head.Path + body2, err := ioutil.ReadFile(g.Head.Path) + assert.NoError(t, err, "Failed to read first rolled file") + if string(body2) != "Line 4\nLine 5\nLine 6\n" { + t.Errorf("Got unexpected contents: [%v]", string(body2)) + } + + // Cleanup + destroyTestGroup(t, g) +} + +func TestFindLast1(t *testing.T) { + g := createTestGroup(t, 0) + + g.WriteLine("Line 1") + g.WriteLine("Line 2") + g.WriteLine("# a") + g.WriteLine("Line 3") + g.Flush() + g.RotateFile() + g.WriteLine("Line 4") + g.WriteLine("Line 5") + g.WriteLine("Line 6") + g.WriteLine("# b") + g.Flush() + + match, found, err := g.FindLast("#") + assert.NoError(t, err) + assert.True(t, found) + assert.Equal(t, "# b", match) + + // Cleanup + destroyTestGroup(t, g) +} + +func TestFindLast2(t *testing.T) { + g := createTestGroup(t, 0) + + g.WriteLine("Line 1") + g.WriteLine("Line 2") + g.WriteLine("Line 3") + g.Flush() + g.RotateFile() + g.WriteLine("# a") + g.WriteLine("Line 4") + g.WriteLine("Line 5") + g.WriteLine("# b") + g.WriteLine("Line 6") + g.Flush() + + match, found, err := g.FindLast("#") + assert.NoError(t, err) + assert.True(t, found) + assert.Equal(t, "# b", match) + + // Cleanup + destroyTestGroup(t, g) +} + +func TestFindLast3(t *testing.T) { + g := createTestGroup(t, 0) + + g.WriteLine("Line 1") + g.WriteLine("# a") + g.WriteLine("Line 2") + g.WriteLine("# b") + g.WriteLine("Line 3") + g.Flush() + g.RotateFile() + g.WriteLine("Line 4") + g.WriteLine("Line 5") + g.WriteLine("Line 6") + g.Flush() + + match, found, err := g.FindLast("#") + assert.NoError(t, err) + assert.True(t, found) + assert.Equal(t, "# b", match) + + // Cleanup + destroyTestGroup(t, g) +} + +func TestFindLast4(t *testing.T) { + g := createTestGroup(t, 0) + + g.WriteLine("Line 1") + g.WriteLine("Line 2") + g.WriteLine("Line 3") + g.Flush() + g.RotateFile() + g.WriteLine("Line 4") + g.WriteLine("Line 5") + g.WriteLine("Line 6") + g.Flush() + + match, found, err := g.FindLast("#") + assert.NoError(t, err) + assert.False(t, found) + assert.Empty(t, match) + + // Cleanup + destroyTestGroup(t, g) +} + +func TestWrite(t *testing.T) { + g := createTestGroup(t, 0) + + written := []byte("Medusa") + g.Write(written) + g.Flush() + + read := make([]byte, len(written)) + gr, err := g.NewReader(0) + require.NoError(t, err, "failed to create reader") + + _, err = gr.Read(read) + assert.NoError(t, err, "failed to read data") + assert.Equal(t, written, read) + + // Cleanup + destroyTestGroup(t, g) +} + +// test that Read reads the required amount of bytes from all the files in the +// group and returns no error if n == size of the given slice. +func TestGroupReaderRead(t *testing.T) { + g := createTestGroup(t, 0) + + professor := []byte("Professor Monster") + g.Write(professor) + g.Flush() + g.RotateFile() + frankenstein := []byte("Frankenstein's Monster") + g.Write(frankenstein) + g.Flush() + + totalWrittenLength := len(professor) + len(frankenstein) + read := make([]byte, totalWrittenLength) + gr, err := g.NewReader(0) + require.NoError(t, err, "failed to create reader") + + n, err := gr.Read(read) + assert.NoError(t, err, "failed to read data") + assert.Equal(t, totalWrittenLength, n, "not enough bytes read") + professorPlusFrankenstein := professor + professorPlusFrankenstein = append(professorPlusFrankenstein, frankenstein...) + assert.Equal(t, professorPlusFrankenstein, read) + + // Cleanup + destroyTestGroup(t, g) +} + +// test that Read returns an error if number of bytes read < size of +// the given slice. Subsequent call should return 0, io.EOF. +func TestGroupReaderRead2(t *testing.T) { + g := createTestGroup(t, 0) + + professor := []byte("Professor Monster") + g.Write(professor) + g.Flush() + g.RotateFile() + frankenstein := []byte("Frankenstein's Monster") + frankensteinPart := []byte("Frankenstein") + g.Write(frankensteinPart) // note writing only a part + g.Flush() + + totalLength := len(professor) + len(frankenstein) + read := make([]byte, totalLength) + gr, err := g.NewReader(0) + require.NoError(t, err, "failed to create reader") + + // 1) n < (size of the given slice), io.EOF + n, err := gr.Read(read) + assert.Equal(t, io.EOF, err) + assert.Equal(t, len(professor)+len(frankensteinPart), n, "Read more/less bytes than it is in the group") + + // 2) 0, io.EOF + n, err = gr.Read([]byte("0")) + assert.Equal(t, io.EOF, err) + assert.Equal(t, 0, n) + + // Cleanup + destroyTestGroup(t, g) +} + +func TestMinIndex(t *testing.T) { + g := createTestGroup(t, 0) + + assert.Zero(t, g.MinIndex(), "MinIndex should be zero at the beginning") + + // Cleanup + destroyTestGroup(t, g) +} + +func TestMaxIndex(t *testing.T) { + g := createTestGroup(t, 0) + + assert.Zero(t, g.MaxIndex(), "MaxIndex should be zero at the beginning") + + g.WriteLine("Line 1") + g.Flush() + g.RotateFile() + + assert.Equal(t, 1, g.MaxIndex(), "MaxIndex should point to the last file") + + // Cleanup + destroyTestGroup(t, g) +} diff --git a/libs/autofile/sighup_watcher.go b/libs/autofile/sighup_watcher.go new file mode 100644 index 000000000..56fbd4d86 --- /dev/null +++ b/libs/autofile/sighup_watcher.go @@ -0,0 +1,63 @@ +package autofile + +import ( + "os" + "os/signal" + "sync" + "sync/atomic" + "syscall" +) + +func init() { + initSighupWatcher() +} + +var sighupWatchers *SighupWatcher +var sighupCounter int32 // For testing + +func initSighupWatcher() { + sighupWatchers = newSighupWatcher() + + c := make(chan os.Signal, 1) + signal.Notify(c, syscall.SIGHUP) + + go func() { + for range c { + sighupWatchers.closeAll() + atomic.AddInt32(&sighupCounter, 1) + } + }() +} + +// Watchces for SIGHUP events and notifies registered AutoFiles +type SighupWatcher struct { + mtx sync.Mutex + autoFiles map[string]*AutoFile +} + +func newSighupWatcher() *SighupWatcher { + return &SighupWatcher{ + autoFiles: make(map[string]*AutoFile, 10), + } +} + +func (w *SighupWatcher) addAutoFile(af *AutoFile) { + w.mtx.Lock() + w.autoFiles[af.ID] = af + w.mtx.Unlock() +} + +// If AutoFile isn't registered or was already removed, does nothing. +func (w *SighupWatcher) removeAutoFile(af *AutoFile) { + w.mtx.Lock() + delete(w.autoFiles, af.ID) + w.mtx.Unlock() +} + +func (w *SighupWatcher) closeAll() { + w.mtx.Lock() + for _, af := range w.autoFiles { + af.closeFile() + } + w.mtx.Unlock() +} diff --git a/libs/bech32/bech32.go b/libs/bech32/bech32.go new file mode 100644 index 000000000..a4db86d5f --- /dev/null +++ b/libs/bech32/bech32.go @@ -0,0 +1,29 @@ +package bech32 + +import ( + "github.com/btcsuite/btcutil/bech32" + "github.com/pkg/errors" +) + +//ConvertAndEncode converts from a base64 encoded byte string to base32 encoded byte string and then to bech32 +func ConvertAndEncode(hrp string, data []byte) (string, error) { + converted, err := bech32.ConvertBits(data, 8, 5, true) + if err != nil { + return "", errors.Wrap(err, "encoding bech32 failed") + } + return bech32.Encode(hrp, converted) + +} + +//DecodeAndConvert decodes a bech32 encoded string and converts to base64 encoded bytes +func DecodeAndConvert(bech string) (string, []byte, error) { + hrp, data, err := bech32.Decode(bech) + if err != nil { + return "", nil, errors.Wrap(err, "decoding bech32 failed") + } + converted, err := bech32.ConvertBits(data, 5, 8, false) + if err != nil { + return "", nil, errors.Wrap(err, "decoding bech32 failed") + } + return hrp, converted, nil +} diff --git a/libs/bech32/bech32_test.go b/libs/bech32/bech32_test.go new file mode 100644 index 000000000..830942061 --- /dev/null +++ b/libs/bech32/bech32_test.go @@ -0,0 +1,31 @@ +package bech32_test + +import ( + "bytes" + "crypto/sha256" + "testing" + + "github.com/tendermint/tendermint/libs/bech32" +) + +func TestEncodeAndDecode(t *testing.T) { + + sum := sha256.Sum256([]byte("hello world\n")) + + bech, err := bech32.ConvertAndEncode("shasum", sum[:]) + + if err != nil { + t.Error(err) + } + hrp, data, err := bech32.DecodeAndConvert(bech) + + if err != nil { + t.Error(err) + } + if hrp != "shasum" { + t.Error("Invalid hrp") + } + if !bytes.Equal(data, sum[:]) { + t.Error("Invalid decode") + } +} diff --git a/libs/circle.yml b/libs/circle.yml new file mode 100644 index 000000000..390ffb039 --- /dev/null +++ b/libs/circle.yml @@ -0,0 +1,21 @@ +machine: + environment: + GOPATH: "${HOME}/.go_workspace" + PROJECT_PARENT_PATH: "$GOPATH/src/github.com/$CIRCLE_PROJECT_USERNAME" + PROJECT_PATH: $GOPATH/src/github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME + hosts: + localhost: 127.0.0.1 + +dependencies: + override: + - mkdir -p "$PROJECT_PARENT_PATH" + - ln -sf "$HOME/$CIRCLE_PROJECT_REPONAME/" "$PROJECT_PATH" + post: + - go version + +test: + override: + - cd $PROJECT_PATH && make get_tools && make get_vendor_deps && bash ./test.sh + post: + - cd "$PROJECT_PATH" && bash <(curl -s https://codecov.io/bash) -f coverage.txt + - cd "$PROJECT_PATH" && mv coverage.txt "${CIRCLE_ARTIFACTS}" diff --git a/libs/cli/flags/log_level.go b/libs/cli/flags/log_level.go new file mode 100644 index 000000000..156106a5a --- /dev/null +++ b/libs/cli/flags/log_level.go @@ -0,0 +1,86 @@ +package flags + +import ( + "fmt" + "strings" + + "github.com/pkg/errors" + + "github.com/tendermint/tendermint/libs/log" +) + +const ( + defaultLogLevelKey = "*" +) + +// ParseLogLevel parses complex log level - comma-separated +// list of module:level pairs with an optional *:level pair (* means +// all other modules). +// +// Example: +// ParseLogLevel("consensus:debug,mempool:debug,*:error", log.NewTMLogger(os.Stdout), "info") +func ParseLogLevel(lvl string, logger log.Logger, defaultLogLevelValue string) (log.Logger, error) { + if lvl == "" { + return nil, errors.New("Empty log level") + } + + l := lvl + + // prefix simple one word levels (e.g. "info") with "*" + if !strings.Contains(l, ":") { + l = defaultLogLevelKey + ":" + l + } + + options := make([]log.Option, 0) + + isDefaultLogLevelSet := false + var option log.Option + var err error + + list := strings.Split(l, ",") + for _, item := range list { + moduleAndLevel := strings.Split(item, ":") + + if len(moduleAndLevel) != 2 { + return nil, fmt.Errorf("Expected list in a form of \"module:level\" pairs, given pair %s, list %s", item, list) + } + + module := moduleAndLevel[0] + level := moduleAndLevel[1] + + if module == defaultLogLevelKey { + option, err = log.AllowLevel(level) + if err != nil { + return nil, errors.Wrap(err, fmt.Sprintf("Failed to parse default log level (pair %s, list %s)", item, l)) + } + options = append(options, option) + isDefaultLogLevelSet = true + } else { + switch level { + case "debug": + option = log.AllowDebugWith("module", module) + case "info": + option = log.AllowInfoWith("module", module) + case "error": + option = log.AllowErrorWith("module", module) + case "none": + option = log.AllowNoneWith("module", module) + default: + return nil, fmt.Errorf("Expected either \"info\", \"debug\", \"error\" or \"none\" log level, given %s (pair %s, list %s)", level, item, list) + } + options = append(options, option) + + } + } + + // if "*" is not provided, set default global level + if !isDefaultLogLevelSet { + option, err = log.AllowLevel(defaultLogLevelValue) + if err != nil { + return nil, err + } + options = append(options, option) + } + + return log.NewFilter(logger, options...), nil +} diff --git a/libs/cli/flags/log_level_test.go b/libs/cli/flags/log_level_test.go new file mode 100644 index 000000000..1503ec281 --- /dev/null +++ b/libs/cli/flags/log_level_test.go @@ -0,0 +1,94 @@ +package flags_test + +import ( + "bytes" + "strings" + "testing" + + tmflags "github.com/tendermint/tendermint/libs/cli/flags" + "github.com/tendermint/tendermint/libs/log" +) + +const ( + defaultLogLevelValue = "info" +) + +func TestParseLogLevel(t *testing.T) { + var buf bytes.Buffer + jsonLogger := log.NewTMJSONLogger(&buf) + + correctLogLevels := []struct { + lvl string + expectedLogLines []string + }{ + {"mempool:error", []string{ + ``, // if no default is given, assume info + ``, + `{"_msg":"Mesmero","level":"error","module":"mempool"}`, + `{"_msg":"Mind","level":"info","module":"state"}`, // if no default is given, assume info + ``}}, + + {"mempool:error,*:debug", []string{ + `{"_msg":"Kingpin","level":"debug","module":"wire"}`, + ``, + `{"_msg":"Mesmero","level":"error","module":"mempool"}`, + `{"_msg":"Mind","level":"info","module":"state"}`, + `{"_msg":"Gideon","level":"debug"}`}}, + + {"*:debug,wire:none", []string{ + ``, + `{"_msg":"Kitty Pryde","level":"info","module":"mempool"}`, + `{"_msg":"Mesmero","level":"error","module":"mempool"}`, + `{"_msg":"Mind","level":"info","module":"state"}`, + `{"_msg":"Gideon","level":"debug"}`}}, + } + + for _, c := range correctLogLevels { + logger, err := tmflags.ParseLogLevel(c.lvl, jsonLogger, defaultLogLevelValue) + if err != nil { + t.Fatal(err) + } + + buf.Reset() + + logger.With("module", "wire").Debug("Kingpin") + if have := strings.TrimSpace(buf.String()); c.expectedLogLines[0] != have { + t.Errorf("\nwant '%s'\nhave '%s'\nlevel '%s'", c.expectedLogLines[0], have, c.lvl) + } + + buf.Reset() + + logger.With("module", "mempool").Info("Kitty Pryde") + if have := strings.TrimSpace(buf.String()); c.expectedLogLines[1] != have { + t.Errorf("\nwant '%s'\nhave '%s'\nlevel '%s'", c.expectedLogLines[1], have, c.lvl) + } + + buf.Reset() + + logger.With("module", "mempool").Error("Mesmero") + if have := strings.TrimSpace(buf.String()); c.expectedLogLines[2] != have { + t.Errorf("\nwant '%s'\nhave '%s'\nlevel '%s'", c.expectedLogLines[2], have, c.lvl) + } + + buf.Reset() + + logger.With("module", "state").Info("Mind") + if have := strings.TrimSpace(buf.String()); c.expectedLogLines[3] != have { + t.Errorf("\nwant '%s'\nhave '%s'\nlevel '%s'", c.expectedLogLines[3], have, c.lvl) + } + + buf.Reset() + + logger.Debug("Gideon") + if have := strings.TrimSpace(buf.String()); c.expectedLogLines[4] != have { + t.Errorf("\nwant '%s'\nhave '%s'\nlevel '%s'", c.expectedLogLines[4], have, c.lvl) + } + } + + incorrectLogLevel := []string{"some", "mempool:some", "*:some,mempool:error"} + for _, lvl := range incorrectLogLevel { + if _, err := tmflags.ParseLogLevel(lvl, jsonLogger, defaultLogLevelValue); err == nil { + t.Fatalf("Expected %s to produce error", lvl) + } + } +} diff --git a/libs/cli/helper.go b/libs/cli/helper.go new file mode 100644 index 000000000..878cf26e5 --- /dev/null +++ b/libs/cli/helper.go @@ -0,0 +1,87 @@ +package cli + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" +) + +// WriteConfigVals writes a toml file with the given values. +// It returns an error if writing was impossible. +func WriteConfigVals(dir string, vals map[string]string) error { + data := "" + for k, v := range vals { + data = data + fmt.Sprintf("%s = \"%s\"\n", k, v) + } + cfile := filepath.Join(dir, "config.toml") + return ioutil.WriteFile(cfile, []byte(data), 0666) +} + +// RunWithArgs executes the given command with the specified command line args +// and environmental variables set. It returns any error returned from cmd.Execute() +func RunWithArgs(cmd Executable, args []string, env map[string]string) error { + oargs := os.Args + oenv := map[string]string{} + // defer returns the environment back to normal + defer func() { + os.Args = oargs + for k, v := range oenv { + os.Setenv(k, v) + } + }() + + // set the args and env how we want them + os.Args = args + for k, v := range env { + // backup old value if there, to restore at end + oenv[k] = os.Getenv(k) + err := os.Setenv(k, v) + if err != nil { + return err + } + } + + // and finally run the command + return cmd.Execute() +} + +// RunCaptureWithArgs executes the given command with the specified command +// line args and environmental variables set. It returns string fields +// representing output written to stdout and stderr, additionally any error +// from cmd.Execute() is also returned +func RunCaptureWithArgs(cmd Executable, args []string, env map[string]string) (stdout, stderr string, err error) { + oldout, olderr := os.Stdout, os.Stderr // keep backup of the real stdout + rOut, wOut, _ := os.Pipe() + rErr, wErr, _ := os.Pipe() + os.Stdout, os.Stderr = wOut, wErr + defer func() { + os.Stdout, os.Stderr = oldout, olderr // restoring the real stdout + }() + + // copy the output in a separate goroutine so printing can't block indefinitely + copyStd := func(reader *os.File) *(chan string) { + stdC := make(chan string) + go func() { + var buf bytes.Buffer + // io.Copy will end when we call reader.Close() below + io.Copy(&buf, reader) + stdC <- buf.String() + }() + return &stdC + } + outC := copyStd(rOut) + errC := copyStd(rErr) + + // now run the command + err = RunWithArgs(cmd, args, env) + + // and grab the stdout to return + wOut.Close() + wErr.Close() + stdout = <-*outC + stderr = <-*errC + return stdout, stderr, err +} diff --git a/libs/cli/setup.go b/libs/cli/setup.go new file mode 100644 index 000000000..06cf1cd1f --- /dev/null +++ b/libs/cli/setup.go @@ -0,0 +1,157 @@ +package cli + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/pkg/errors" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +const ( + HomeFlag = "home" + TraceFlag = "trace" + OutputFlag = "output" + EncodingFlag = "encoding" +) + +// Executable is the minimal interface to *corba.Command, so we can +// wrap if desired before the test +type Executable interface { + Execute() error +} + +// PrepareBaseCmd is meant for tendermint and other servers +func PrepareBaseCmd(cmd *cobra.Command, envPrefix, defaultHome string) Executor { + cobra.OnInitialize(func() { initEnv(envPrefix) }) + cmd.PersistentFlags().StringP(HomeFlag, "", defaultHome, "directory for config and data") + cmd.PersistentFlags().Bool(TraceFlag, false, "print out full stack trace on errors") + cmd.PersistentPreRunE = concatCobraCmdFuncs(bindFlagsLoadViper, cmd.PersistentPreRunE) + return Executor{cmd, os.Exit} +} + +// PrepareMainCmd is meant for client side libs that want some more flags +// +// This adds --encoding (hex, btc, base64) and --output (text, json) to +// the command. These only really make sense in interactive commands. +func PrepareMainCmd(cmd *cobra.Command, envPrefix, defaultHome string) Executor { + cmd.PersistentFlags().StringP(EncodingFlag, "e", "hex", "Binary encoding (hex|b64|btc)") + cmd.PersistentFlags().StringP(OutputFlag, "o", "text", "Output format (text|json)") + cmd.PersistentPreRunE = concatCobraCmdFuncs(validateOutput, cmd.PersistentPreRunE) + return PrepareBaseCmd(cmd, envPrefix, defaultHome) +} + +// initEnv sets to use ENV variables if set. +func initEnv(prefix string) { + copyEnvVars(prefix) + + // env variables with TM prefix (eg. TM_ROOT) + viper.SetEnvPrefix(prefix) + viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_", "-", "_")) + viper.AutomaticEnv() +} + +// This copies all variables like TMROOT to TM_ROOT, +// so we can support both formats for the user +func copyEnvVars(prefix string) { + prefix = strings.ToUpper(prefix) + ps := prefix + "_" + for _, e := range os.Environ() { + kv := strings.SplitN(e, "=", 2) + if len(kv) == 2 { + k, v := kv[0], kv[1] + if strings.HasPrefix(k, prefix) && !strings.HasPrefix(k, ps) { + k2 := strings.Replace(k, prefix, ps, 1) + os.Setenv(k2, v) + } + } + } +} + +// Executor wraps the cobra Command with a nicer Execute method +type Executor struct { + *cobra.Command + Exit func(int) // this is os.Exit by default, override in tests +} + +type ExitCoder interface { + ExitCode() int +} + +// execute adds all child commands to the root command sets flags appropriately. +// This is called by main.main(). It only needs to happen once to the rootCmd. +func (e Executor) Execute() error { + e.SilenceUsage = true + e.SilenceErrors = true + err := e.Command.Execute() + if err != nil { + if viper.GetBool(TraceFlag) { + fmt.Fprintf(os.Stderr, "ERROR: %+v\n", err) + } else { + fmt.Fprintf(os.Stderr, "ERROR: %v\n", err) + } + + // return error code 1 by default, can override it with a special error type + exitCode := 1 + if ec, ok := err.(ExitCoder); ok { + exitCode = ec.ExitCode() + } + e.Exit(exitCode) + } + return err +} + +type cobraCmdFunc func(cmd *cobra.Command, args []string) error + +// Returns a single function that calls each argument function in sequence +// RunE, PreRunE, PersistentPreRunE, etc. all have this same signature +func concatCobraCmdFuncs(fs ...cobraCmdFunc) cobraCmdFunc { + return func(cmd *cobra.Command, args []string) error { + for _, f := range fs { + if f != nil { + if err := f(cmd, args); err != nil { + return err + } + } + } + return nil + } +} + +// Bind all flags and read the config into viper +func bindFlagsLoadViper(cmd *cobra.Command, args []string) error { + // cmd.Flags() includes flags from this command and all persistent flags from the parent + if err := viper.BindPFlags(cmd.Flags()); err != nil { + return err + } + + homeDir := viper.GetString(HomeFlag) + viper.Set(HomeFlag, homeDir) + viper.SetConfigName("config") // name of config file (without extension) + viper.AddConfigPath(homeDir) // search root directory + viper.AddConfigPath(filepath.Join(homeDir, "config")) // search root directory /config + + // If a config file is found, read it in. + if err := viper.ReadInConfig(); err == nil { + // stderr, so if we redirect output to json file, this doesn't appear + // fmt.Fprintln(os.Stderr, "Using config file:", viper.ConfigFileUsed()) + } else if _, ok := err.(viper.ConfigFileNotFoundError); !ok { + // ignore not found error, return other errors + return err + } + return nil +} + +func validateOutput(cmd *cobra.Command, args []string) error { + // validate output format + output := viper.GetString(OutputFlag) + switch output { + case "text", "json": + default: + return errors.Errorf("Unsupported output format: %s", output) + } + return nil +} diff --git a/libs/cli/setup_test.go b/libs/cli/setup_test.go new file mode 100644 index 000000000..04209e493 --- /dev/null +++ b/libs/cli/setup_test.go @@ -0,0 +1,237 @@ +package cli + +import ( + "fmt" + "io/ioutil" + "strconv" + "strings" + "testing" + + "github.com/pkg/errors" + "github.com/spf13/cobra" + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSetupEnv(t *testing.T) { + cases := []struct { + args []string + env map[string]string + expected string + }{ + {nil, nil, ""}, + {[]string{"--foobar", "bang!"}, nil, "bang!"}, + // make sure reset is good + {nil, nil, ""}, + // test both variants of the prefix + {nil, map[string]string{"DEMO_FOOBAR": "good"}, "good"}, + {nil, map[string]string{"DEMOFOOBAR": "silly"}, "silly"}, + // and that cli overrides env... + {[]string{"--foobar", "important"}, + map[string]string{"DEMO_FOOBAR": "ignored"}, "important"}, + } + + for idx, tc := range cases { + i := strconv.Itoa(idx) + // test command that store value of foobar in local variable + var foo string + demo := &cobra.Command{ + Use: "demo", + RunE: func(cmd *cobra.Command, args []string) error { + foo = viper.GetString("foobar") + return nil + }, + } + demo.Flags().String("foobar", "", "Some test value from config") + cmd := PrepareBaseCmd(demo, "DEMO", "/qwerty/asdfgh") // some missing dir.. + cmd.Exit = func(int) {} + + viper.Reset() + args := append([]string{cmd.Use}, tc.args...) + err := RunWithArgs(cmd, args, tc.env) + require.Nil(t, err, i) + assert.Equal(t, tc.expected, foo, i) + } +} + +func tempDir() string { + cdir, err := ioutil.TempDir("", "test-cli") + if err != nil { + panic(err) + } + return cdir +} + +func TestSetupConfig(t *testing.T) { + // we pre-create two config files we can refer to in the rest of + // the test cases. + cval1 := "fubble" + conf1 := tempDir() + err := WriteConfigVals(conf1, map[string]string{"boo": cval1}) + require.Nil(t, err) + + cases := []struct { + args []string + env map[string]string + expected string + expectedTwo string + }{ + {nil, nil, "", ""}, + // setting on the command line + {[]string{"--boo", "haha"}, nil, "haha", ""}, + {[]string{"--two-words", "rocks"}, nil, "", "rocks"}, + {[]string{"--home", conf1}, nil, cval1, ""}, + // test both variants of the prefix + {nil, map[string]string{"RD_BOO": "bang"}, "bang", ""}, + {nil, map[string]string{"RD_TWO_WORDS": "fly"}, "", "fly"}, + {nil, map[string]string{"RDTWO_WORDS": "fly"}, "", "fly"}, + {nil, map[string]string{"RD_HOME": conf1}, cval1, ""}, + {nil, map[string]string{"RDHOME": conf1}, cval1, ""}, + } + + for idx, tc := range cases { + i := strconv.Itoa(idx) + // test command that store value of foobar in local variable + var foo, two string + boo := &cobra.Command{ + Use: "reader", + RunE: func(cmd *cobra.Command, args []string) error { + foo = viper.GetString("boo") + two = viper.GetString("two-words") + return nil + }, + } + boo.Flags().String("boo", "", "Some test value from config") + boo.Flags().String("two-words", "", "Check out env handling -") + cmd := PrepareBaseCmd(boo, "RD", "/qwerty/asdfgh") // some missing dir... + cmd.Exit = func(int) {} + + viper.Reset() + args := append([]string{cmd.Use}, tc.args...) + err := RunWithArgs(cmd, args, tc.env) + require.Nil(t, err, i) + assert.Equal(t, tc.expected, foo, i) + assert.Equal(t, tc.expectedTwo, two, i) + } +} + +type DemoConfig struct { + Name string `mapstructure:"name"` + Age int `mapstructure:"age"` + Unused int `mapstructure:"unused"` +} + +func TestSetupUnmarshal(t *testing.T) { + // we pre-create two config files we can refer to in the rest of + // the test cases. + cval1, cval2 := "someone", "else" + conf1 := tempDir() + err := WriteConfigVals(conf1, map[string]string{"name": cval1}) + require.Nil(t, err) + // even with some ignored fields, should be no problem + conf2 := tempDir() + err = WriteConfigVals(conf2, map[string]string{"name": cval2, "foo": "bar"}) + require.Nil(t, err) + + // unused is not declared on a flag and remains from base + base := DemoConfig{ + Name: "default", + Age: 42, + Unused: -7, + } + c := func(name string, age int) DemoConfig { + r := base + // anything set on the flags as a default is used over + // the default config object + r.Name = "from-flag" + if name != "" { + r.Name = name + } + if age != 0 { + r.Age = age + } + return r + } + + cases := []struct { + args []string + env map[string]string + expected DemoConfig + }{ + {nil, nil, c("", 0)}, + // setting on the command line + {[]string{"--name", "haha"}, nil, c("haha", 0)}, + {[]string{"--home", conf1}, nil, c(cval1, 0)}, + // test both variants of the prefix + {nil, map[string]string{"MR_AGE": "56"}, c("", 56)}, + {nil, map[string]string{"MR_HOME": conf1}, c(cval1, 0)}, + {[]string{"--age", "17"}, map[string]string{"MRHOME": conf2}, c(cval2, 17)}, + } + + for idx, tc := range cases { + i := strconv.Itoa(idx) + // test command that store value of foobar in local variable + cfg := base + marsh := &cobra.Command{ + Use: "marsh", + RunE: func(cmd *cobra.Command, args []string) error { + return viper.Unmarshal(&cfg) + }, + } + marsh.Flags().String("name", "from-flag", "Some test value from config") + // if we want a flag to use the proper default, then copy it + // from the default config here + marsh.Flags().Int("age", base.Age, "Some test value from config") + cmd := PrepareBaseCmd(marsh, "MR", "/qwerty/asdfgh") // some missing dir... + cmd.Exit = func(int) {} + + viper.Reset() + args := append([]string{cmd.Use}, tc.args...) + err := RunWithArgs(cmd, args, tc.env) + require.Nil(t, err, i) + assert.Equal(t, tc.expected, cfg, i) + } +} + +func TestSetupTrace(t *testing.T) { + cases := []struct { + args []string + env map[string]string + long bool + expected string + }{ + {nil, nil, false, "Trace flag = false"}, + {[]string{"--trace"}, nil, true, "Trace flag = true"}, + {[]string{"--no-such-flag"}, nil, false, "unknown flag: --no-such-flag"}, + {nil, map[string]string{"DBG_TRACE": "true"}, true, "Trace flag = true"}, + } + + for idx, tc := range cases { + i := strconv.Itoa(idx) + // test command that store value of foobar in local variable + trace := &cobra.Command{ + Use: "trace", + RunE: func(cmd *cobra.Command, args []string) error { + return errors.Errorf("Trace flag = %t", viper.GetBool(TraceFlag)) + }, + } + cmd := PrepareBaseCmd(trace, "DBG", "/qwerty/asdfgh") // some missing dir.. + cmd.Exit = func(int) {} + + viper.Reset() + args := append([]string{cmd.Use}, tc.args...) + stdout, stderr, err := RunCaptureWithArgs(cmd, args, tc.env) + require.NotNil(t, err, i) + require.Equal(t, "", stdout, i) + require.NotEqual(t, "", stderr, i) + msg := strings.Split(stderr, "\n") + desired := fmt.Sprintf("ERROR: %s", tc.expected) + assert.Equal(t, desired, msg[0], i) + if tc.long && assert.True(t, len(msg) > 2, i) { + // the next line starts the stack trace... + assert.Contains(t, msg[1], "TestSetupTrace", i) + assert.Contains(t, msg[2], "setup_test.go", i) + } + } +} diff --git a/libs/clist/clist.go b/libs/clist/clist.go new file mode 100644 index 000000000..ccb1f5777 --- /dev/null +++ b/libs/clist/clist.go @@ -0,0 +1,384 @@ +package clist + +/* + +The purpose of CList is to provide a goroutine-safe linked-list. +This list can be traversed concurrently by any number of goroutines. +However, removed CElements cannot be added back. +NOTE: Not all methods of container/list are (yet) implemented. +NOTE: Removed elements need to DetachPrev or DetachNext consistently +to ensure garbage collection of removed elements. + +*/ + +import ( + "sync" +) + +/* + +CElement is an element of a linked-list +Traversal from a CElement is goroutine-safe. + +We can't avoid using WaitGroups or for-loops given the documentation +spec without re-implementing the primitives that already exist in +golang/sync. Notice that WaitGroup allows many go-routines to be +simultaneously released, which is what we want. Mutex doesn't do +this. RWMutex does this, but it's clumsy to use in the way that a +WaitGroup would be used -- and we'd end up having two RWMutex's for +prev/next each, which is doubly confusing. + +sync.Cond would be sort-of useful, but we don't need a write-lock in +the for-loop. Use sync.Cond when you need serial access to the +"condition". In our case our condition is if `next != nil || removed`, +and there's no reason to serialize that condition for goroutines +waiting on NextWait() (since it's just a read operation). + +*/ +type CElement struct { + mtx sync.RWMutex + prev *CElement + prevWg *sync.WaitGroup + prevWaitCh chan struct{} + next *CElement + nextWg *sync.WaitGroup + nextWaitCh chan struct{} + removed bool + + Value interface{} // immutable +} + +// Blocking implementation of Next(). +// May return nil iff CElement was tail and got removed. +func (e *CElement) NextWait() *CElement { + for { + e.mtx.RLock() + next := e.next + nextWg := e.nextWg + removed := e.removed + e.mtx.RUnlock() + + if next != nil || removed { + return next + } + + nextWg.Wait() + // e.next doesn't necessarily exist here. + // That's why we need to continue a for-loop. + } +} + +// Blocking implementation of Prev(). +// May return nil iff CElement was head and got removed. +func (e *CElement) PrevWait() *CElement { + for { + e.mtx.RLock() + prev := e.prev + prevWg := e.prevWg + removed := e.removed + e.mtx.RUnlock() + + if prev != nil || removed { + return prev + } + + prevWg.Wait() + } +} + +// PrevWaitChan can be used to wait until Prev becomes not nil. Once it does, +// channel will be closed. +func (e *CElement) PrevWaitChan() <-chan struct{} { + e.mtx.RLock() + defer e.mtx.RUnlock() + + return e.prevWaitCh +} + +// NextWaitChan can be used to wait until Next becomes not nil. Once it does, +// channel will be closed. +func (e *CElement) NextWaitChan() <-chan struct{} { + e.mtx.RLock() + defer e.mtx.RUnlock() + + return e.nextWaitCh +} + +// Nonblocking, may return nil if at the end. +func (e *CElement) Next() *CElement { + e.mtx.RLock() + defer e.mtx.RUnlock() + + return e.next +} + +// Nonblocking, may return nil if at the end. +func (e *CElement) Prev() *CElement { + e.mtx.RLock() + defer e.mtx.RUnlock() + + return e.prev +} + +func (e *CElement) Removed() bool { + e.mtx.RLock() + defer e.mtx.RUnlock() + + return e.removed +} + +func (e *CElement) DetachNext() { + if !e.Removed() { + panic("DetachNext() must be called after Remove(e)") + } + e.mtx.Lock() + defer e.mtx.Unlock() + + e.next = nil +} + +func (e *CElement) DetachPrev() { + if !e.Removed() { + panic("DetachPrev() must be called after Remove(e)") + } + e.mtx.Lock() + defer e.mtx.Unlock() + + e.prev = nil +} + +// NOTE: This function needs to be safe for +// concurrent goroutines waiting on nextWg. +func (e *CElement) SetNext(newNext *CElement) { + e.mtx.Lock() + defer e.mtx.Unlock() + + oldNext := e.next + e.next = newNext + if oldNext != nil && newNext == nil { + // See https://golang.org/pkg/sync/: + // + // If a WaitGroup is reused to wait for several independent sets of + // events, new Add calls must happen after all previous Wait calls have + // returned. + e.nextWg = waitGroup1() // WaitGroups are difficult to re-use. + e.nextWaitCh = make(chan struct{}) + } + if oldNext == nil && newNext != nil { + e.nextWg.Done() + close(e.nextWaitCh) + } +} + +// NOTE: This function needs to be safe for +// concurrent goroutines waiting on prevWg +func (e *CElement) SetPrev(newPrev *CElement) { + e.mtx.Lock() + defer e.mtx.Unlock() + + oldPrev := e.prev + e.prev = newPrev + if oldPrev != nil && newPrev == nil { + e.prevWg = waitGroup1() // WaitGroups are difficult to re-use. + e.prevWaitCh = make(chan struct{}) + } + if oldPrev == nil && newPrev != nil { + e.prevWg.Done() + close(e.prevWaitCh) + } +} + +func (e *CElement) SetRemoved() { + e.mtx.Lock() + defer e.mtx.Unlock() + + e.removed = true + + // This wakes up anyone waiting in either direction. + if e.prev == nil { + e.prevWg.Done() + close(e.prevWaitCh) + } + if e.next == nil { + e.nextWg.Done() + close(e.nextWaitCh) + } +} + +//-------------------------------------------------------------------------------- + +// CList represents a linked list. +// The zero value for CList is an empty list ready to use. +// Operations are goroutine-safe. +type CList struct { + mtx sync.RWMutex + wg *sync.WaitGroup + waitCh chan struct{} + head *CElement // first element + tail *CElement // last element + len int // list length +} + +func (l *CList) Init() *CList { + l.mtx.Lock() + defer l.mtx.Unlock() + + l.wg = waitGroup1() + l.waitCh = make(chan struct{}) + l.head = nil + l.tail = nil + l.len = 0 + return l +} + +func New() *CList { return new(CList).Init() } + +func (l *CList) Len() int { + l.mtx.RLock() + defer l.mtx.RUnlock() + + return l.len +} + +func (l *CList) Front() *CElement { + l.mtx.RLock() + defer l.mtx.RUnlock() + + return l.head +} + +func (l *CList) FrontWait() *CElement { + // Loop until the head is non-nil else wait and try again + for { + l.mtx.RLock() + head := l.head + wg := l.wg + l.mtx.RUnlock() + + if head != nil { + return head + } + wg.Wait() + // NOTE: If you think l.head exists here, think harder. + } +} + +func (l *CList) Back() *CElement { + l.mtx.RLock() + defer l.mtx.RUnlock() + + return l.tail +} + +func (l *CList) BackWait() *CElement { + for { + l.mtx.RLock() + tail := l.tail + wg := l.wg + l.mtx.RUnlock() + + if tail != nil { + return tail + } + wg.Wait() + // l.tail doesn't necessarily exist here. + // That's why we need to continue a for-loop. + } +} + +// WaitChan can be used to wait until Front or Back becomes not nil. Once it +// does, channel will be closed. +func (l *CList) WaitChan() <-chan struct{} { + l.mtx.Lock() + defer l.mtx.Unlock() + + return l.waitCh +} + +func (l *CList) PushBack(v interface{}) *CElement { + l.mtx.Lock() + defer l.mtx.Unlock() + + // Construct a new element + e := &CElement{ + prev: nil, + prevWg: waitGroup1(), + prevWaitCh: make(chan struct{}), + next: nil, + nextWg: waitGroup1(), + nextWaitCh: make(chan struct{}), + removed: false, + Value: v, + } + + // Release waiters on FrontWait/BackWait maybe + if l.len == 0 { + l.wg.Done() + close(l.waitCh) + } + l.len++ + + // Modify the tail + if l.tail == nil { + l.head = e + l.tail = e + } else { + e.SetPrev(l.tail) // We must init e first. + l.tail.SetNext(e) // This will make e accessible. + l.tail = e // Update the list. + } + + return e +} + +// CONTRACT: Caller must call e.DetachPrev() and/or e.DetachNext() to avoid memory leaks. +// NOTE: As per the contract of CList, removed elements cannot be added back. +func (l *CList) Remove(e *CElement) interface{} { + l.mtx.Lock() + defer l.mtx.Unlock() + + prev := e.Prev() + next := e.Next() + + if l.head == nil || l.tail == nil { + panic("Remove(e) on empty CList") + } + if prev == nil && l.head != e { + panic("Remove(e) with false head") + } + if next == nil && l.tail != e { + panic("Remove(e) with false tail") + } + + // If we're removing the only item, make CList FrontWait/BackWait wait. + if l.len == 1 { + l.wg = waitGroup1() // WaitGroups are difficult to re-use. + l.waitCh = make(chan struct{}) + } + + // Update l.len + l.len-- + + // Connect next/prev and set head/tail + if prev == nil { + l.head = next + } else { + prev.SetNext(next) + } + if next == nil { + l.tail = prev + } else { + next.SetPrev(prev) + } + + // Set .Done() on e, otherwise waiters will wait forever. + e.SetRemoved() + + return e.Value +} + +func waitGroup1() (wg *sync.WaitGroup) { + wg = &sync.WaitGroup{} + wg.Add(1) + return +} diff --git a/libs/clist/clist_test.go b/libs/clist/clist_test.go new file mode 100644 index 000000000..6171f1a39 --- /dev/null +++ b/libs/clist/clist_test.go @@ -0,0 +1,293 @@ +package clist + +import ( + "fmt" + "math/rand" + "runtime" + "sync/atomic" + "testing" + "time" +) + +func TestSmall(t *testing.T) { + l := New() + el1 := l.PushBack(1) + el2 := l.PushBack(2) + el3 := l.PushBack(3) + if l.Len() != 3 { + t.Error("Expected len 3, got ", l.Len()) + } + + //fmt.Printf("%p %v\n", el1, el1) + //fmt.Printf("%p %v\n", el2, el2) + //fmt.Printf("%p %v\n", el3, el3) + + r1 := l.Remove(el1) + + //fmt.Printf("%p %v\n", el1, el1) + //fmt.Printf("%p %v\n", el2, el2) + //fmt.Printf("%p %v\n", el3, el3) + + r2 := l.Remove(el2) + + //fmt.Printf("%p %v\n", el1, el1) + //fmt.Printf("%p %v\n", el2, el2) + //fmt.Printf("%p %v\n", el3, el3) + + r3 := l.Remove(el3) + + if r1 != 1 { + t.Error("Expected 1, got ", r1) + } + if r2 != 2 { + t.Error("Expected 2, got ", r2) + } + if r3 != 3 { + t.Error("Expected 3, got ", r3) + } + if l.Len() != 0 { + t.Error("Expected len 0, got ", l.Len()) + } + +} + +/* +This test is quite hacky because it relies on SetFinalizer +which isn't guaranteed to run at all. +*/ +// nolint: megacheck +func _TestGCFifo(t *testing.T) { + + const numElements = 1000000 + l := New() + gcCount := new(uint64) + + // SetFinalizer doesn't work well with circular structures, + // so we construct a trivial non-circular structure to + // track. + type value struct { + Int int + } + done := make(chan struct{}) + + for i := 0; i < numElements; i++ { + v := new(value) + v.Int = i + l.PushBack(v) + runtime.SetFinalizer(v, func(v *value) { + atomic.AddUint64(gcCount, 1) + }) + } + + for el := l.Front(); el != nil; { + l.Remove(el) + //oldEl := el + el = el.Next() + //oldEl.DetachPrev() + //oldEl.DetachNext() + } + + runtime.GC() + time.Sleep(time.Second * 3) + runtime.GC() + time.Sleep(time.Second * 3) + _ = done + + if *gcCount != numElements { + t.Errorf("Expected gcCount to be %v, got %v", numElements, + *gcCount) + } +} + +/* +This test is quite hacky because it relies on SetFinalizer +which isn't guaranteed to run at all. +*/ +// nolint: megacheck +func _TestGCRandom(t *testing.T) { + + const numElements = 1000000 + l := New() + gcCount := 0 + + // SetFinalizer doesn't work well with circular structures, + // so we construct a trivial non-circular structure to + // track. + type value struct { + Int int + } + + for i := 0; i < numElements; i++ { + v := new(value) + v.Int = i + l.PushBack(v) + runtime.SetFinalizer(v, func(v *value) { + gcCount++ + }) + } + + els := make([]*CElement, 0, numElements) + for el := l.Front(); el != nil; el = el.Next() { + els = append(els, el) + } + + for _, i := range rand.Perm(numElements) { + el := els[i] + l.Remove(el) + _ = el.Next() + } + + runtime.GC() + time.Sleep(time.Second * 3) + + if gcCount != numElements { + t.Errorf("Expected gcCount to be %v, got %v", numElements, + gcCount) + } +} + +func TestScanRightDeleteRandom(t *testing.T) { + + const numElements = 10000 + const numTimes = 1000 + const numScanners = 10 + + l := New() + stop := make(chan struct{}) + + els := make([]*CElement, numElements) + for i := 0; i < numElements; i++ { + el := l.PushBack(i) + els[i] = el + } + + // Launch scanner routines that will rapidly iterate over elements. + for i := 0; i < numScanners; i++ { + go func(scannerID int) { + var el *CElement + restartCounter := 0 + counter := 0 + FOR_LOOP: + for { + select { + case <-stop: + fmt.Println("stopped") + break FOR_LOOP + default: + } + if el == nil { + el = l.FrontWait() + restartCounter++ + } + el = el.Next() + counter++ + } + fmt.Printf("Scanner %v restartCounter: %v counter: %v\n", scannerID, restartCounter, counter) + }(i) + } + + // Remove an element, push back an element. + for i := 0; i < numTimes; i++ { + // Pick an element to remove + rmElIdx := rand.Intn(len(els)) + rmEl := els[rmElIdx] + + // Remove it + l.Remove(rmEl) + //fmt.Print(".") + + // Insert a new element + newEl := l.PushBack(-1*i - 1) + els[rmElIdx] = newEl + + if i%100000 == 0 { + fmt.Printf("Pushed %vK elements so far...\n", i/1000) + } + + } + + // Stop scanners + close(stop) + time.Sleep(time.Second * 1) + + // And remove all the elements. + for el := l.Front(); el != nil; el = el.Next() { + l.Remove(el) + } + if l.Len() != 0 { + t.Fatal("Failed to remove all elements from CList") + } +} + +func TestWaitChan(t *testing.T) { + l := New() + ch := l.WaitChan() + + // 1) add one element to an empty list + go l.PushBack(1) + <-ch + + // 2) and remove it + el := l.Front() + v := l.Remove(el) + if v != 1 { + t.Fatal("where is 1 coming from?") + } + + // 3) test iterating forward and waiting for Next (NextWaitChan and Next) + el = l.PushBack(0) + + done := make(chan struct{}) + pushed := 0 + go func() { + for i := 1; i < 100; i++ { + l.PushBack(i) + pushed++ + time.Sleep(time.Duration(rand.Intn(100)) * time.Millisecond) + } + close(done) + }() + + next := el + seen := 0 +FOR_LOOP: + for { + select { + case <-next.NextWaitChan(): + next = next.Next() + seen++ + if next == nil { + continue + } + case <-done: + break FOR_LOOP + case <-time.After(10 * time.Second): + t.Fatal("max execution time") + } + } + + if pushed != seen { + t.Fatalf("number of pushed items (%d) not equal to number of seen items (%d)", pushed, seen) + } + + // 4) test iterating backwards (PrevWaitChan and Prev) + prev := next + seen = 0 +FOR_LOOP2: + for { + select { + case <-prev.PrevWaitChan(): + prev = prev.Prev() + seen++ + if prev == nil { + t.Fatal("expected PrevWaitChan to block forever on nil when reached first elem") + } + case <-time.After(5 * time.Second): + break FOR_LOOP2 + } + } + + if pushed != seen { + t.Fatalf("number of pushed items (%d) not equal to number of seen items (%d)", pushed, seen) + } +} diff --git a/libs/common/LICENSE b/libs/common/LICENSE new file mode 100644 index 000000000..8a142a71b --- /dev/null +++ b/libs/common/LICENSE @@ -0,0 +1,193 @@ +Tendermint Go-Common +Copyright (C) 2015 Tendermint + + + + Apache License + Version 2.0, January 2004 + https://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/libs/common/array.go b/libs/common/array.go new file mode 100644 index 000000000..adedc42be --- /dev/null +++ b/libs/common/array.go @@ -0,0 +1,5 @@ +package common + +func Arr(items ...interface{}) []interface{} { + return items +} diff --git a/libs/common/async.go b/libs/common/async.go new file mode 100644 index 000000000..e3293ab4c --- /dev/null +++ b/libs/common/async.go @@ -0,0 +1,175 @@ +package common + +import ( + "sync/atomic" +) + +//---------------------------------------- +// Task + +// val: the value returned after task execution. +// err: the error returned during task completion. +// abort: tells Parallel to return, whether or not all tasks have completed. +type Task func(i int) (val interface{}, err error, abort bool) + +type TaskResult struct { + Value interface{} + Error error +} + +type TaskResultCh <-chan TaskResult + +type taskResultOK struct { + TaskResult + OK bool +} + +type TaskResultSet struct { + chz []TaskResultCh + results []taskResultOK +} + +func newTaskResultSet(chz []TaskResultCh) *TaskResultSet { + return &TaskResultSet{ + chz: chz, + results: make([]taskResultOK, len(chz)), + } +} + +func (trs *TaskResultSet) Channels() []TaskResultCh { + return trs.chz +} + +func (trs *TaskResultSet) LatestResult(index int) (TaskResult, bool) { + if len(trs.results) <= index { + return TaskResult{}, false + } + resultOK := trs.results[index] + return resultOK.TaskResult, resultOK.OK +} + +// NOTE: Not concurrency safe. +// Writes results to trs.results without waiting for all tasks to complete. +func (trs *TaskResultSet) Reap() *TaskResultSet { + for i := 0; i < len(trs.results); i++ { + var trch = trs.chz[i] + select { + case result, ok := <-trch: + if ok { + // Write result. + trs.results[i] = taskResultOK{ + TaskResult: result, + OK: true, + } + } else { + // We already wrote it. + } + default: + // Do nothing. + } + } + return trs +} + +// NOTE: Not concurrency safe. +// Like Reap() but waits until all tasks have returned or panic'd. +func (trs *TaskResultSet) Wait() *TaskResultSet { + for i := 0; i < len(trs.results); i++ { + var trch = trs.chz[i] + result, ok := <-trch + if ok { + // Write result. + trs.results[i] = taskResultOK{ + TaskResult: result, + OK: true, + } + } else { + // We already wrote it. + } + } + return trs +} + +// Returns the firstmost (by task index) error as +// discovered by all previous Reap() calls. +func (trs *TaskResultSet) FirstValue() interface{} { + for _, result := range trs.results { + if result.Value != nil { + return result.Value + } + } + return nil +} + +// Returns the firstmost (by task index) error as +// discovered by all previous Reap() calls. +func (trs *TaskResultSet) FirstError() error { + for _, result := range trs.results { + if result.Error != nil { + return result.Error + } + } + return nil +} + +//---------------------------------------- +// Parallel + +// Run tasks in parallel, with ability to abort early. +// Returns ok=false iff any of the tasks returned abort=true. +// NOTE: Do not implement quit features here. Instead, provide convenient +// concurrent quit-like primitives, passed implicitly via Task closures. (e.g. +// it's not Parallel's concern how you quit/abort your tasks). +func Parallel(tasks ...Task) (trs *TaskResultSet, ok bool) { + var taskResultChz = make([]TaskResultCh, len(tasks)) // To return. + var taskDoneCh = make(chan bool, len(tasks)) // A "wait group" channel, early abort if any true received. + var numPanics = new(int32) // Keep track of panics to set ok=false later. + ok = true // We will set it to false iff any tasks panic'd or returned abort. + + // Start all tasks in parallel in separate goroutines. + // When the task is complete, it will appear in the + // respective taskResultCh (associated by task index). + for i, task := range tasks { + var taskResultCh = make(chan TaskResult, 1) // Capacity for 1 result. + taskResultChz[i] = taskResultCh + go func(i int, task Task, taskResultCh chan TaskResult) { + // Recovery + defer func() { + if pnk := recover(); pnk != nil { + atomic.AddInt32(numPanics, 1) + // Send panic to taskResultCh. + taskResultCh <- TaskResult{nil, ErrorWrap(pnk, "Panic in task")} + // Closing taskResultCh lets trs.Wait() work. + close(taskResultCh) + // Decrement waitgroup. + taskDoneCh <- false + } + }() + // Run the task. + var val, err, abort = task(i) + // Send val/err to taskResultCh. + // NOTE: Below this line, nothing must panic/ + taskResultCh <- TaskResult{val, err} + // Closing taskResultCh lets trs.Wait() work. + close(taskResultCh) + // Decrement waitgroup. + taskDoneCh <- abort + }(i, task, taskResultCh) + } + + // Wait until all tasks are done, or until abort. + // DONE_LOOP: + for i := 0; i < len(tasks); i++ { + abort := <-taskDoneCh + if abort { + ok = false + break + } + } + + // Ok is also false if there were any panics. + // We must do this check here (after DONE_LOOP). + ok = ok && (atomic.LoadInt32(numPanics) == 0) + + return newTaskResultSet(taskResultChz).Reap(), ok +} diff --git a/libs/common/async_test.go b/libs/common/async_test.go new file mode 100644 index 000000000..f565b4bd3 --- /dev/null +++ b/libs/common/async_test.go @@ -0,0 +1,156 @@ +package common + +import ( + "errors" + "fmt" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestParallel(t *testing.T) { + + // Create tasks. + var counter = new(int32) + var tasks = make([]Task, 100*1000) + for i := 0; i < len(tasks); i++ { + tasks[i] = func(i int) (res interface{}, err error, abort bool) { + atomic.AddInt32(counter, 1) + return -1 * i, nil, false + } + } + + // Run in parallel. + var trs, ok = Parallel(tasks...) + assert.True(t, ok) + + // Verify. + assert.Equal(t, int(*counter), len(tasks), "Each task should have incremented the counter already") + var failedTasks int + for i := 0; i < len(tasks); i++ { + taskResult, ok := trs.LatestResult(i) + if !ok { + assert.Fail(t, "Task #%v did not complete.", i) + failedTasks++ + } else if taskResult.Error != nil { + assert.Fail(t, "Task should not have errored but got %v", taskResult.Error) + failedTasks++ + } else if !assert.Equal(t, -1*i, taskResult.Value.(int)) { + assert.Fail(t, "Task should have returned %v but got %v", -1*i, taskResult.Value.(int)) + failedTasks++ + } else { + // Good! + } + } + assert.Equal(t, failedTasks, 0, "No task should have failed") + assert.Nil(t, trs.FirstError(), "There should be no errors") + assert.Equal(t, 0, trs.FirstValue(), "First value should be 0") +} + +func TestParallelAbort(t *testing.T) { + + var flow1 = make(chan struct{}, 1) + var flow2 = make(chan struct{}, 1) + var flow3 = make(chan struct{}, 1) // Cap must be > 0 to prevent blocking. + var flow4 = make(chan struct{}, 1) + + // Create tasks. + var tasks = []Task{ + func(i int) (res interface{}, err error, abort bool) { + assert.Equal(t, i, 0) + flow1 <- struct{}{} + return 0, nil, false + }, + func(i int) (res interface{}, err error, abort bool) { + assert.Equal(t, i, 1) + flow2 <- <-flow1 + return 1, errors.New("some error"), false + }, + func(i int) (res interface{}, err error, abort bool) { + assert.Equal(t, i, 2) + flow3 <- <-flow2 + return 2, nil, true + }, + func(i int) (res interface{}, err error, abort bool) { + assert.Equal(t, i, 3) + <-flow4 + return 3, nil, false + }, + } + + // Run in parallel. + var taskResultSet, ok = Parallel(tasks...) + assert.False(t, ok, "ok should be false since we aborted task #2.") + + // Verify task #3. + // Initially taskResultSet.chz[3] sends nothing since flow4 didn't send. + waitTimeout(t, taskResultSet.chz[3], "Task #3") + + // Now let the last task (#3) complete after abort. + flow4 <- <-flow3 + + // Wait until all tasks have returned or panic'd. + taskResultSet.Wait() + + // Verify task #0, #1, #2. + checkResult(t, taskResultSet, 0, 0, nil, nil) + checkResult(t, taskResultSet, 1, 1, errors.New("some error"), nil) + checkResult(t, taskResultSet, 2, 2, nil, nil) + checkResult(t, taskResultSet, 3, 3, nil, nil) +} + +func TestParallelRecover(t *testing.T) { + + // Create tasks. + var tasks = []Task{ + func(i int) (res interface{}, err error, abort bool) { + return 0, nil, false + }, + func(i int) (res interface{}, err error, abort bool) { + return 1, errors.New("some error"), false + }, + func(i int) (res interface{}, err error, abort bool) { + panic(2) + }, + } + + // Run in parallel. + var taskResultSet, ok = Parallel(tasks...) + assert.False(t, ok, "ok should be false since we panic'd in task #2.") + + // Verify task #0, #1, #2. + checkResult(t, taskResultSet, 0, 0, nil, nil) + checkResult(t, taskResultSet, 1, 1, errors.New("some error"), nil) + checkResult(t, taskResultSet, 2, nil, nil, 2) +} + +// Wait for result +func checkResult(t *testing.T, taskResultSet *TaskResultSet, index int, val interface{}, err error, pnk interface{}) { + taskResult, ok := taskResultSet.LatestResult(index) + taskName := fmt.Sprintf("Task #%v", index) + assert.True(t, ok, "TaskResultCh unexpectedly closed for %v", taskName) + assert.Equal(t, val, taskResult.Value, taskName) + if err != nil { + assert.Equal(t, err, taskResult.Error, taskName) + } else if pnk != nil { + assert.Equal(t, pnk, taskResult.Error.(Error).Data(), taskName) + } else { + assert.Nil(t, taskResult.Error, taskName) + } +} + +// Wait for timeout (no result) +func waitTimeout(t *testing.T, taskResultCh TaskResultCh, taskName string) { + select { + case _, ok := <-taskResultCh: + if !ok { + assert.Fail(t, "TaskResultCh unexpectedly closed (%v)", taskName) + } else { + assert.Fail(t, "TaskResultCh unexpectedly returned for %v", taskName) + } + case <-time.After(1 * time.Second): // TODO use deterministic time? + // Good! + } +} diff --git a/libs/common/bit_array.go b/libs/common/bit_array.go new file mode 100644 index 000000000..0290921a6 --- /dev/null +++ b/libs/common/bit_array.go @@ -0,0 +1,378 @@ +package common + +import ( + "encoding/binary" + "fmt" + "regexp" + "strings" + "sync" +) + +type BitArray struct { + mtx sync.Mutex + Bits int `json:"bits"` // NOTE: persisted via reflect, must be exported + Elems []uint64 `json:"elems"` // NOTE: persisted via reflect, must be exported +} + +// There is no BitArray whose Size is 0. Use nil instead. +func NewBitArray(bits int) *BitArray { + if bits <= 0 { + return nil + } + return &BitArray{ + Bits: bits, + Elems: make([]uint64, (bits+63)/64), + } +} + +func (bA *BitArray) Size() int { + if bA == nil { + return 0 + } + return bA.Bits +} + +// NOTE: behavior is undefined if i >= bA.Bits +func (bA *BitArray) GetIndex(i int) bool { + if bA == nil { + return false + } + bA.mtx.Lock() + defer bA.mtx.Unlock() + return bA.getIndex(i) +} + +func (bA *BitArray) getIndex(i int) bool { + if i >= bA.Bits { + return false + } + return bA.Elems[i/64]&(uint64(1)< 0 +} + +// NOTE: behavior is undefined if i >= bA.Bits +func (bA *BitArray) SetIndex(i int, v bool) bool { + if bA == nil { + return false + } + bA.mtx.Lock() + defer bA.mtx.Unlock() + return bA.setIndex(i, v) +} + +func (bA *BitArray) setIndex(i int, v bool) bool { + if i >= bA.Bits { + return false + } + if v { + bA.Elems[i/64] |= (uint64(1) << uint(i%64)) + } else { + bA.Elems[i/64] &= ^(uint64(1) << uint(i%64)) + } + return true +} + +func (bA *BitArray) Copy() *BitArray { + if bA == nil { + return nil + } + bA.mtx.Lock() + defer bA.mtx.Unlock() + return bA.copy() +} + +func (bA *BitArray) copy() *BitArray { + c := make([]uint64, len(bA.Elems)) + copy(c, bA.Elems) + return &BitArray{ + Bits: bA.Bits, + Elems: c, + } +} + +func (bA *BitArray) copyBits(bits int) *BitArray { + c := make([]uint64, (bits+63)/64) + copy(c, bA.Elems) + return &BitArray{ + Bits: bits, + Elems: c, + } +} + +// Returns a BitArray of larger bits size. +func (bA *BitArray) Or(o *BitArray) *BitArray { + if bA == nil && o == nil { + return nil + } + if bA == nil && o != nil { + return o.Copy() + } + if o == nil { + return bA.Copy() + } + bA.mtx.Lock() + defer bA.mtx.Unlock() + c := bA.copyBits(MaxInt(bA.Bits, o.Bits)) + for i := 0; i < len(c.Elems); i++ { + c.Elems[i] |= o.Elems[i] + } + return c +} + +// Returns a BitArray of smaller bit size. +func (bA *BitArray) And(o *BitArray) *BitArray { + if bA == nil || o == nil { + return nil + } + bA.mtx.Lock() + defer bA.mtx.Unlock() + return bA.and(o) +} + +func (bA *BitArray) and(o *BitArray) *BitArray { + c := bA.copyBits(MinInt(bA.Bits, o.Bits)) + for i := 0; i < len(c.Elems); i++ { + c.Elems[i] &= o.Elems[i] + } + return c +} + +func (bA *BitArray) Not() *BitArray { + if bA == nil { + return nil // Degenerate + } + bA.mtx.Lock() + defer bA.mtx.Unlock() + c := bA.copy() + for i := 0; i < len(c.Elems); i++ { + c.Elems[i] = ^c.Elems[i] + } + return c +} + +func (bA *BitArray) Sub(o *BitArray) *BitArray { + if bA == nil || o == nil { + // TODO: Decide if we should do 1's complement here? + return nil + } + bA.mtx.Lock() + defer bA.mtx.Unlock() + if bA.Bits > o.Bits { + c := bA.copy() + for i := 0; i < len(o.Elems)-1; i++ { + c.Elems[i] &= ^c.Elems[i] + } + i := len(o.Elems) - 1 + if i >= 0 { + for idx := i * 64; idx < o.Bits; idx++ { + // NOTE: each individual GetIndex() call to o is safe. + c.setIndex(idx, c.getIndex(idx) && !o.GetIndex(idx)) + } + } + return c + } + return bA.and(o.Not()) // Note degenerate case where o == nil +} + +func (bA *BitArray) IsEmpty() bool { + if bA == nil { + return true // should this be opposite? + } + bA.mtx.Lock() + defer bA.mtx.Unlock() + for _, e := range bA.Elems { + if e > 0 { + return false + } + } + return true +} + +func (bA *BitArray) IsFull() bool { + if bA == nil { + return true + } + bA.mtx.Lock() + defer bA.mtx.Unlock() + + // Check all elements except the last + for _, elem := range bA.Elems[:len(bA.Elems)-1] { + if (^elem) != 0 { + return false + } + } + + // Check that the last element has (lastElemBits) 1's + lastElemBits := (bA.Bits+63)%64 + 1 + lastElem := bA.Elems[len(bA.Elems)-1] + return (lastElem+1)&((uint64(1)< 0 { + randBitStart := RandIntn(64) + for j := 0; j < 64; j++ { + bitIdx := ((j + randBitStart) % 64) + if (bA.Elems[elemIdx] & (uint64(1) << uint(bitIdx))) > 0 { + return 64*elemIdx + bitIdx, true + } + } + PanicSanity("should not happen") + } + } else { + // Special case for last elem, to ignore straggler bits + elemBits := bA.Bits % 64 + if elemBits == 0 { + elemBits = 64 + } + randBitStart := RandIntn(elemBits) + for j := 0; j < elemBits; j++ { + bitIdx := ((j + randBitStart) % elemBits) + if (bA.Elems[elemIdx] & (uint64(1) << uint(bitIdx))) > 0 { + return 64*elemIdx + bitIdx, true + } + } + } + } + return 0, false +} + +// String returns a string representation of BitArray: BA{}, +// where is a sequence of 'x' (1) and '_' (0). +// The includes spaces and newlines to help people. +// For a simple sequence of 'x' and '_' characters with no spaces or newlines, +// see the MarshalJSON() method. +// Example: "BA{_x_}" or "nil-BitArray" for nil. +func (bA *BitArray) String() string { + return bA.StringIndented("") +} + +func (bA *BitArray) StringIndented(indent string) string { + if bA == nil { + return "nil-BitArray" + } + bA.mtx.Lock() + defer bA.mtx.Unlock() + return bA.stringIndented(indent) +} + +func (bA *BitArray) stringIndented(indent string) string { + lines := []string{} + bits := "" + for i := 0; i < bA.Bits; i++ { + if bA.getIndex(i) { + bits += "x" + } else { + bits += "_" + } + if i%100 == 99 { + lines = append(lines, bits) + bits = "" + } + if i%10 == 9 { + bits += indent + } + if i%50 == 49 { + bits += indent + } + } + if len(bits) > 0 { + lines = append(lines, bits) + } + return fmt.Sprintf("BA{%v:%v}", bA.Bits, strings.Join(lines, indent)) +} + +func (bA *BitArray) Bytes() []byte { + bA.mtx.Lock() + defer bA.mtx.Unlock() + + numBytes := (bA.Bits + 7) / 8 + bytes := make([]byte, numBytes) + for i := 0; i < len(bA.Elems); i++ { + elemBytes := [8]byte{} + binary.LittleEndian.PutUint64(elemBytes[:], bA.Elems[i]) + copy(bytes[i*8:], elemBytes[:]) + } + return bytes +} + +// NOTE: other bitarray o is not locked when reading, +// so if necessary, caller must copy or lock o prior to calling Update. +// If bA is nil, does nothing. +func (bA *BitArray) Update(o *BitArray) { + if bA == nil || o == nil { + return + } + bA.mtx.Lock() + defer bA.mtx.Unlock() + + copy(bA.Elems, o.Elems) +} + +// MarshalJSON implements json.Marshaler interface by marshaling bit array +// using a custom format: a string of '-' or 'x' where 'x' denotes the 1 bit. +func (bA *BitArray) MarshalJSON() ([]byte, error) { + if bA == nil { + return []byte("null"), nil + } + + bA.mtx.Lock() + defer bA.mtx.Unlock() + + bits := `"` + for i := 0; i < bA.Bits; i++ { + if bA.getIndex(i) { + bits += `x` + } else { + bits += `_` + } + } + bits += `"` + return []byte(bits), nil +} + +var bitArrayJSONRegexp = regexp.MustCompile(`\A"([_x]*)"\z`) + +// UnmarshalJSON implements json.Unmarshaler interface by unmarshaling a custom +// JSON description. +func (bA *BitArray) UnmarshalJSON(bz []byte) error { + b := string(bz) + if b == "null" { + // This is required e.g. for encoding/json when decoding + // into a pointer with pre-allocated BitArray. + bA.Bits = 0 + bA.Elems = nil + return nil + } + + // Validate 'b'. + match := bitArrayJSONRegexp.FindStringSubmatch(b) + if match == nil { + return fmt.Errorf("BitArray in JSON should be a string of format %q but got %s", bitArrayJSONRegexp.String(), b) + } + bits := match[1] + + // Construct new BitArray and copy over. + numBits := len(bits) + bA2 := NewBitArray(numBits) + for i := 0; i < numBits; i++ { + if bits[i] == 'x' { + bA2.SetIndex(i, true) + } + } + *bA = *bA2 + return nil +} diff --git a/libs/common/bit_array_test.go b/libs/common/bit_array_test.go new file mode 100644 index 000000000..c697ba5de --- /dev/null +++ b/libs/common/bit_array_test.go @@ -0,0 +1,267 @@ +package common + +import ( + "bytes" + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func randBitArray(bits int) (*BitArray, []byte) { + src := RandBytes((bits + 7) / 8) + bA := NewBitArray(bits) + for i := 0; i < len(src); i++ { + for j := 0; j < 8; j++ { + if i*8+j >= bits { + return bA, src + } + setBit := src[i]&(1< 0 + bA.SetIndex(i*8+j, setBit) + } + } + return bA, src +} + +func TestAnd(t *testing.T) { + + bA1, _ := randBitArray(51) + bA2, _ := randBitArray(31) + bA3 := bA1.And(bA2) + + var bNil *BitArray + require.Equal(t, bNil.And(bA1), (*BitArray)(nil)) + require.Equal(t, bA1.And(nil), (*BitArray)(nil)) + require.Equal(t, bNil.And(nil), (*BitArray)(nil)) + + if bA3.Bits != 31 { + t.Error("Expected min bits", bA3.Bits) + } + if len(bA3.Elems) != len(bA2.Elems) { + t.Error("Expected min elems length") + } + for i := 0; i < bA3.Bits; i++ { + expected := bA1.GetIndex(i) && bA2.GetIndex(i) + if bA3.GetIndex(i) != expected { + t.Error("Wrong bit from bA3", i, bA1.GetIndex(i), bA2.GetIndex(i), bA3.GetIndex(i)) + } + } +} + +func TestOr(t *testing.T) { + + bA1, _ := randBitArray(51) + bA2, _ := randBitArray(31) + bA3 := bA1.Or(bA2) + + bNil := (*BitArray)(nil) + require.Equal(t, bNil.Or(bA1), bA1) + require.Equal(t, bA1.Or(nil), bA1) + require.Equal(t, bNil.Or(nil), (*BitArray)(nil)) + + if bA3.Bits != 51 { + t.Error("Expected max bits") + } + if len(bA3.Elems) != len(bA1.Elems) { + t.Error("Expected max elems length") + } + for i := 0; i < bA3.Bits; i++ { + expected := bA1.GetIndex(i) || bA2.GetIndex(i) + if bA3.GetIndex(i) != expected { + t.Error("Wrong bit from bA3", i, bA1.GetIndex(i), bA2.GetIndex(i), bA3.GetIndex(i)) + } + } +} + +func TestSub1(t *testing.T) { + + bA1, _ := randBitArray(31) + bA2, _ := randBitArray(51) + bA3 := bA1.Sub(bA2) + + bNil := (*BitArray)(nil) + require.Equal(t, bNil.Sub(bA1), (*BitArray)(nil)) + require.Equal(t, bA1.Sub(nil), (*BitArray)(nil)) + require.Equal(t, bNil.Sub(nil), (*BitArray)(nil)) + + if bA3.Bits != bA1.Bits { + t.Error("Expected bA1 bits") + } + if len(bA3.Elems) != len(bA1.Elems) { + t.Error("Expected bA1 elems length") + } + for i := 0; i < bA3.Bits; i++ { + expected := bA1.GetIndex(i) + if bA2.GetIndex(i) { + expected = false + } + if bA3.GetIndex(i) != expected { + t.Error("Wrong bit from bA3", i, bA1.GetIndex(i), bA2.GetIndex(i), bA3.GetIndex(i)) + } + } +} + +func TestSub2(t *testing.T) { + + bA1, _ := randBitArray(51) + bA2, _ := randBitArray(31) + bA3 := bA1.Sub(bA2) + + bNil := (*BitArray)(nil) + require.Equal(t, bNil.Sub(bA1), (*BitArray)(nil)) + require.Equal(t, bA1.Sub(nil), (*BitArray)(nil)) + require.Equal(t, bNil.Sub(nil), (*BitArray)(nil)) + + if bA3.Bits != bA1.Bits { + t.Error("Expected bA1 bits") + } + if len(bA3.Elems) != len(bA1.Elems) { + t.Error("Expected bA1 elems length") + } + for i := 0; i < bA3.Bits; i++ { + expected := bA1.GetIndex(i) + if i < bA2.Bits && bA2.GetIndex(i) { + expected = false + } + if bA3.GetIndex(i) != expected { + t.Error("Wrong bit from bA3") + } + } +} + +func TestPickRandom(t *testing.T) { + for idx := 0; idx < 123; idx++ { + bA1 := NewBitArray(123) + bA1.SetIndex(idx, true) + index, ok := bA1.PickRandom() + if !ok { + t.Fatal("Expected to pick element but got none") + } + if index != idx { + t.Fatalf("Expected to pick element at %v but got wrong index", idx) + } + } +} + +func TestBytes(t *testing.T) { + bA := NewBitArray(4) + bA.SetIndex(0, true) + check := func(bA *BitArray, bz []byte) { + if !bytes.Equal(bA.Bytes(), bz) { + panic(Fmt("Expected %X but got %X", bz, bA.Bytes())) + } + } + check(bA, []byte{0x01}) + bA.SetIndex(3, true) + check(bA, []byte{0x09}) + + bA = NewBitArray(9) + check(bA, []byte{0x00, 0x00}) + bA.SetIndex(7, true) + check(bA, []byte{0x80, 0x00}) + bA.SetIndex(8, true) + check(bA, []byte{0x80, 0x01}) + + bA = NewBitArray(16) + check(bA, []byte{0x00, 0x00}) + bA.SetIndex(7, true) + check(bA, []byte{0x80, 0x00}) + bA.SetIndex(8, true) + check(bA, []byte{0x80, 0x01}) + bA.SetIndex(9, true) + check(bA, []byte{0x80, 0x03}) +} + +func TestEmptyFull(t *testing.T) { + ns := []int{47, 123} + for _, n := range ns { + bA := NewBitArray(n) + if !bA.IsEmpty() { + t.Fatal("Expected bit array to be empty") + } + for i := 0; i < n; i++ { + bA.SetIndex(i, true) + } + if !bA.IsFull() { + t.Fatal("Expected bit array to be full") + } + } +} + +func TestUpdateNeverPanics(t *testing.T) { + newRandBitArray := func(n int) *BitArray { + ba, _ := randBitArray(n) + return ba + } + pairs := []struct { + a, b *BitArray + }{ + {nil, nil}, + {newRandBitArray(10), newRandBitArray(12)}, + {newRandBitArray(23), newRandBitArray(23)}, + {newRandBitArray(37), nil}, + {nil, NewBitArray(10)}, + } + + for _, pair := range pairs { + a, b := pair.a, pair.b + a.Update(b) + b.Update(a) + } +} + +func TestNewBitArrayNeverCrashesOnNegatives(t *testing.T) { + bitList := []int{-127, -128, -1 << 31} + for _, bits := range bitList { + _ = NewBitArray(bits) + } +} + +func TestJSONMarshalUnmarshal(t *testing.T) { + + bA1 := NewBitArray(0) + + bA2 := NewBitArray(1) + + bA3 := NewBitArray(1) + bA3.SetIndex(0, true) + + bA4 := NewBitArray(5) + bA4.SetIndex(0, true) + bA4.SetIndex(1, true) + + testCases := []struct { + bA *BitArray + marshalledBA string + }{ + {nil, `null`}, + {bA1, `null`}, + {bA2, `"_"`}, + {bA3, `"x"`}, + {bA4, `"xx___"`}, + } + + for _, tc := range testCases { + t.Run(tc.bA.String(), func(t *testing.T) { + bz, err := json.Marshal(tc.bA) + require.NoError(t, err) + + assert.Equal(t, tc.marshalledBA, string(bz)) + + var unmarshalledBA *BitArray + err = json.Unmarshal(bz, &unmarshalledBA) + require.NoError(t, err) + + if tc.bA == nil { + require.Nil(t, unmarshalledBA) + } else { + require.NotNil(t, unmarshalledBA) + assert.EqualValues(t, tc.bA.Bits, unmarshalledBA.Bits) + if assert.EqualValues(t, tc.bA.String(), unmarshalledBA.String()) { + assert.EqualValues(t, tc.bA.Elems, unmarshalledBA.Elems) + } + } + }) + } +} diff --git a/libs/common/bytes.go b/libs/common/bytes.go new file mode 100644 index 000000000..711720aa7 --- /dev/null +++ b/libs/common/bytes.go @@ -0,0 +1,62 @@ +package common + +import ( + "encoding/hex" + "fmt" + "strings" +) + +// The main purpose of HexBytes is to enable HEX-encoding for json/encoding. +type HexBytes []byte + +// Marshal needed for protobuf compatibility +func (bz HexBytes) Marshal() ([]byte, error) { + return bz, nil +} + +// Unmarshal needed for protobuf compatibility +func (bz *HexBytes) Unmarshal(data []byte) error { + *bz = data + return nil +} + +// This is the point of Bytes. +func (bz HexBytes) MarshalJSON() ([]byte, error) { + s := strings.ToUpper(hex.EncodeToString(bz)) + jbz := make([]byte, len(s)+2) + jbz[0] = '"' + copy(jbz[1:], []byte(s)) + jbz[len(jbz)-1] = '"' + return jbz, nil +} + +// This is the point of Bytes. +func (bz *HexBytes) UnmarshalJSON(data []byte) error { + if len(data) < 2 || data[0] != '"' || data[len(data)-1] != '"' { + return fmt.Errorf("Invalid hex string: %s", data) + } + bz2, err := hex.DecodeString(string(data[1 : len(data)-1])) + if err != nil { + return err + } + *bz = bz2 + return nil +} + +// Allow it to fulfill various interfaces in light-client, etc... +func (bz HexBytes) Bytes() []byte { + return bz +} + +func (bz HexBytes) String() string { + return strings.ToUpper(hex.EncodeToString(bz)) +} + +func (bz HexBytes) Format(s fmt.State, verb rune) { + switch verb { + case 'p': + s.Write([]byte(fmt.Sprintf("%p", bz))) + default: + s.Write([]byte(fmt.Sprintf("%X", []byte(bz)))) + } +} diff --git a/libs/common/bytes_test.go b/libs/common/bytes_test.go new file mode 100644 index 000000000..9e11988f2 --- /dev/null +++ b/libs/common/bytes_test.go @@ -0,0 +1,65 @@ +package common + +import ( + "encoding/json" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +// This is a trivial test for protobuf compatibility. +func TestMarshal(t *testing.T) { + bz := []byte("hello world") + dataB := HexBytes(bz) + bz2, err := dataB.Marshal() + assert.Nil(t, err) + assert.Equal(t, bz, bz2) + + var dataB2 HexBytes + err = (&dataB2).Unmarshal(bz) + assert.Nil(t, err) + assert.Equal(t, dataB, dataB2) +} + +// Test that the hex encoding works. +func TestJSONMarshal(t *testing.T) { + + type TestStruct struct { + B1 []byte + B2 HexBytes + } + + cases := []struct { + input []byte + expected string + }{ + {[]byte(``), `{"B1":"","B2":""}`}, + {[]byte(`a`), `{"B1":"YQ==","B2":"61"}`}, + {[]byte(`abc`), `{"B1":"YWJj","B2":"616263"}`}, + } + + for i, tc := range cases { + t.Run(fmt.Sprintf("Case %d", i), func(t *testing.T) { + ts := TestStruct{B1: tc.input, B2: tc.input} + + // Test that it marshals correctly to JSON. + jsonBytes, err := json.Marshal(ts) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, string(jsonBytes), tc.expected) + + // TODO do fuzz testing to ensure that unmarshal fails + + // Test that unmarshaling works correctly. + ts2 := TestStruct{} + err = json.Unmarshal(jsonBytes, &ts2) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, ts2.B1, tc.input) + assert.Equal(t, ts2.B2, HexBytes(tc.input)) + }) + } +} diff --git a/libs/common/byteslice.go b/libs/common/byteslice.go new file mode 100644 index 000000000..57b3a8a2b --- /dev/null +++ b/libs/common/byteslice.go @@ -0,0 +1,73 @@ +package common + +import ( + "bytes" +) + +// Fingerprint returns the first 6 bytes of a byte slice. +// If the slice is less than 6 bytes, the fingerprint +// contains trailing zeroes. +func Fingerprint(slice []byte) []byte { + fingerprint := make([]byte, 6) + copy(fingerprint, slice) + return fingerprint +} + +func IsZeros(slice []byte) bool { + for _, byt := range slice { + if byt != byte(0) { + return false + } + } + return true +} + +func RightPadBytes(slice []byte, l int) []byte { + if l < len(slice) { + return slice + } + padded := make([]byte, l) + copy(padded[0:len(slice)], slice) + return padded +} + +func LeftPadBytes(slice []byte, l int) []byte { + if l < len(slice) { + return slice + } + padded := make([]byte, l) + copy(padded[l-len(slice):], slice) + return padded +} + +func TrimmedString(b []byte) string { + trimSet := string([]byte{0}) + return string(bytes.TrimLeft(b, trimSet)) + +} + +// PrefixEndBytes returns the end byteslice for a noninclusive range +// that would include all byte slices for which the input is the prefix +func PrefixEndBytes(prefix []byte) []byte { + if prefix == nil { + return nil + } + + end := make([]byte, len(prefix)) + copy(end, prefix) + finished := false + + for !finished { + if end[len(end)-1] != byte(255) { + end[len(end)-1]++ + finished = true + } else { + end = end[:len(end)-1] + if len(end) == 0 { + end = nil + finished = true + } + } + } + return end +} diff --git a/libs/common/byteslice_test.go b/libs/common/byteslice_test.go new file mode 100644 index 000000000..98085d125 --- /dev/null +++ b/libs/common/byteslice_test.go @@ -0,0 +1,28 @@ +package common + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestPrefixEndBytes(t *testing.T) { + assert := assert.New(t) + + var testCases = []struct { + prefix []byte + expected []byte + }{ + {[]byte{byte(55), byte(255), byte(255), byte(0)}, []byte{byte(55), byte(255), byte(255), byte(1)}}, + {[]byte{byte(55), byte(255), byte(255), byte(15)}, []byte{byte(55), byte(255), byte(255), byte(16)}}, + {[]byte{byte(55), byte(200), byte(255)}, []byte{byte(55), byte(201)}}, + {[]byte{byte(55), byte(255), byte(255)}, []byte{byte(56)}}, + {[]byte{byte(255), byte(255), byte(255)}, nil}, + {nil, nil}, + } + + for _, test := range testCases { + end := PrefixEndBytes(test.prefix) + assert.Equal(test.expected, end) + } +} diff --git a/libs/common/cmap.go b/libs/common/cmap.go new file mode 100644 index 000000000..c65c27d4c --- /dev/null +++ b/libs/common/cmap.go @@ -0,0 +1,73 @@ +package common + +import "sync" + +// CMap is a goroutine-safe map +type CMap struct { + m map[string]interface{} + l sync.Mutex +} + +func NewCMap() *CMap { + return &CMap{ + m: make(map[string]interface{}), + } +} + +func (cm *CMap) Set(key string, value interface{}) { + cm.l.Lock() + defer cm.l.Unlock() + cm.m[key] = value +} + +func (cm *CMap) Get(key string) interface{} { + cm.l.Lock() + defer cm.l.Unlock() + return cm.m[key] +} + +func (cm *CMap) Has(key string) bool { + cm.l.Lock() + defer cm.l.Unlock() + _, ok := cm.m[key] + return ok +} + +func (cm *CMap) Delete(key string) { + cm.l.Lock() + defer cm.l.Unlock() + delete(cm.m, key) +} + +func (cm *CMap) Size() int { + cm.l.Lock() + defer cm.l.Unlock() + return len(cm.m) +} + +func (cm *CMap) Clear() { + cm.l.Lock() + defer cm.l.Unlock() + cm.m = make(map[string]interface{}) +} + +func (cm *CMap) Keys() []string { + cm.l.Lock() + defer cm.l.Unlock() + + keys := []string{} + for k := range cm.m { + keys = append(keys, k) + } + return keys +} + +func (cm *CMap) Values() []interface{} { + cm.l.Lock() + defer cm.l.Unlock() + items := []interface{}{} + for _, v := range cm.m { + items = append(items, v) + } + return items +} diff --git a/libs/common/cmap_test.go b/libs/common/cmap_test.go new file mode 100644 index 000000000..c665a7f3e --- /dev/null +++ b/libs/common/cmap_test.go @@ -0,0 +1,53 @@ +package common + +import ( + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIterateKeysWithValues(t *testing.T) { + cmap := NewCMap() + + for i := 1; i <= 10; i++ { + cmap.Set(fmt.Sprintf("key%d", i), fmt.Sprintf("value%d", i)) + } + + // Testing size + assert.Equal(t, 10, cmap.Size()) + assert.Equal(t, 10, len(cmap.Keys())) + assert.Equal(t, 10, len(cmap.Values())) + + // Iterating Keys, checking for matching Value + for _, key := range cmap.Keys() { + val := strings.Replace(key, "key", "value", -1) + assert.Equal(t, val, cmap.Get(key)) + } + + // Test if all keys are within []Keys() + keys := cmap.Keys() + for i := 1; i <= 10; i++ { + assert.Contains(t, keys, fmt.Sprintf("key%d", i), "cmap.Keys() should contain key") + } + + // Delete 1 Key + cmap.Delete("key1") + + assert.NotEqual(t, len(keys), len(cmap.Keys()), "[]keys and []Keys() should not be equal, they are copies, one item was removed") +} + +func TestContains(t *testing.T) { + cmap := NewCMap() + + cmap.Set("key1", "value1") + + // Test for known values + assert.True(t, cmap.Has("key1")) + assert.Equal(t, "value1", cmap.Get("key1")) + + // Test for unknown values + assert.False(t, cmap.Has("key2")) + assert.Nil(t, cmap.Get("key2")) +} diff --git a/libs/common/colors.go b/libs/common/colors.go new file mode 100644 index 000000000..049ce7a50 --- /dev/null +++ b/libs/common/colors.go @@ -0,0 +1,95 @@ +package common + +import ( + "fmt" + "strings" +) + +const ( + ANSIReset = "\x1b[0m" + ANSIBright = "\x1b[1m" + ANSIDim = "\x1b[2m" + ANSIUnderscore = "\x1b[4m" + ANSIBlink = "\x1b[5m" + ANSIReverse = "\x1b[7m" + ANSIHidden = "\x1b[8m" + + ANSIFgBlack = "\x1b[30m" + ANSIFgRed = "\x1b[31m" + ANSIFgGreen = "\x1b[32m" + ANSIFgYellow = "\x1b[33m" + ANSIFgBlue = "\x1b[34m" + ANSIFgMagenta = "\x1b[35m" + ANSIFgCyan = "\x1b[36m" + ANSIFgWhite = "\x1b[37m" + + ANSIBgBlack = "\x1b[40m" + ANSIBgRed = "\x1b[41m" + ANSIBgGreen = "\x1b[42m" + ANSIBgYellow = "\x1b[43m" + ANSIBgBlue = "\x1b[44m" + ANSIBgMagenta = "\x1b[45m" + ANSIBgCyan = "\x1b[46m" + ANSIBgWhite = "\x1b[47m" +) + +// color the string s with color 'color' +// unless s is already colored +func treat(s string, color string) string { + if len(s) > 2 && s[:2] == "\x1b[" { + return s + } + return color + s + ANSIReset +} + +func treatAll(color string, args ...interface{}) string { + var parts []string + for _, arg := range args { + parts = append(parts, treat(fmt.Sprintf("%v", arg), color)) + } + return strings.Join(parts, "") +} + +func Black(args ...interface{}) string { + return treatAll(ANSIFgBlack, args...) +} + +func Red(args ...interface{}) string { + return treatAll(ANSIFgRed, args...) +} + +func Green(args ...interface{}) string { + return treatAll(ANSIFgGreen, args...) +} + +func Yellow(args ...interface{}) string { + return treatAll(ANSIFgYellow, args...) +} + +func Blue(args ...interface{}) string { + return treatAll(ANSIFgBlue, args...) +} + +func Magenta(args ...interface{}) string { + return treatAll(ANSIFgMagenta, args...) +} + +func Cyan(args ...interface{}) string { + return treatAll(ANSIFgCyan, args...) +} + +func White(args ...interface{}) string { + return treatAll(ANSIFgWhite, args...) +} + +func ColoredBytes(data []byte, textColor, bytesColor func(...interface{}) string) string { + s := "" + for _, b := range data { + if 0x21 <= b && b < 0x7F { + s += textColor(string(b)) + } else { + s += bytesColor(Fmt("%02X", b)) + } + } + return s +} diff --git a/libs/common/date.go b/libs/common/date.go new file mode 100644 index 000000000..e017a4b41 --- /dev/null +++ b/libs/common/date.go @@ -0,0 +1,43 @@ +package common + +import ( + "strings" + "time" + + "github.com/pkg/errors" +) + +// TimeLayout helps to parse a date string of the format YYYY-MM-DD +// Intended to be used with the following function: +// time.Parse(TimeLayout, date) +var TimeLayout = "2006-01-02" //this represents YYYY-MM-DD + +// ParseDateRange parses a date range string of the format start:end +// where the start and end date are of the format YYYY-MM-DD. +// The parsed dates are time.Time and will return the zero time for +// unbounded dates, ex: +// unbounded start: :2000-12-31 +// unbounded end: 2000-12-31: +func ParseDateRange(dateRange string) (startDate, endDate time.Time, err error) { + dates := strings.Split(dateRange, ":") + if len(dates) != 2 { + err = errors.New("bad date range, must be in format date:date") + return + } + parseDate := func(date string) (out time.Time, err error) { + if len(date) == 0 { + return + } + out, err = time.Parse(TimeLayout, date) + return + } + startDate, err = parseDate(dates[0]) + if err != nil { + return + } + endDate, err = parseDate(dates[1]) + if err != nil { + return + } + return +} diff --git a/libs/common/date_test.go b/libs/common/date_test.go new file mode 100644 index 000000000..2c0632477 --- /dev/null +++ b/libs/common/date_test.go @@ -0,0 +1,46 @@ +package common + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +var ( + date = time.Date(2015, time.Month(12), 31, 0, 0, 0, 0, time.UTC) + date2 = time.Date(2016, time.Month(12), 31, 0, 0, 0, 0, time.UTC) + zero time.Time +) + +func TestParseDateRange(t *testing.T) { + assert := assert.New(t) + + var testDates = []struct { + dateStr string + start time.Time + end time.Time + errNil bool + }{ + {"2015-12-31:2016-12-31", date, date2, true}, + {"2015-12-31:", date, zero, true}, + {":2016-12-31", zero, date2, true}, + {"2016-12-31", zero, zero, false}, + {"2016-31-12:", zero, zero, false}, + {":2016-31-12", zero, zero, false}, + } + + for _, test := range testDates { + start, end, err := ParseDateRange(test.dateStr) + if test.errNil { + assert.Nil(err) + testPtr := func(want, have time.Time) { + assert.True(have.Equal(want)) + } + testPtr(test.start, start) + testPtr(test.end, end) + } else { + assert.NotNil(err) + } + } +} diff --git a/libs/common/errors.go b/libs/common/errors.go new file mode 100644 index 000000000..5c31b8968 --- /dev/null +++ b/libs/common/errors.go @@ -0,0 +1,246 @@ +package common + +import ( + "fmt" + "runtime" +) + +//---------------------------------------- +// Convenience method. + +func ErrorWrap(cause interface{}, format string, args ...interface{}) Error { + if causeCmnError, ok := cause.(*cmnError); ok { + msg := Fmt(format, args...) + return causeCmnError.Stacktrace().Trace(1, msg) + } else if cause == nil { + return newCmnError(FmtError{format, args}).Stacktrace() + } else { + // NOTE: causeCmnError is a typed nil here. + msg := Fmt(format, args...) + return newCmnError(cause).Stacktrace().Trace(1, msg) + } +} + +//---------------------------------------- +// Error & cmnError + +/* + +Usage with arbitrary error data: + +```go + // Error construction + type MyError struct{} + var err1 error = NewErrorWithData(MyError{}, "my message") + ... + // Wrapping + var err2 error = ErrorWrap(err1, "another message") + if (err1 != err2) { panic("should be the same") + ... + // Error handling + switch err2.Data().(type){ + case MyError: ... + default: ... + } +``` +*/ +type Error interface { + Error() string + Stacktrace() Error + Trace(offset int, format string, args ...interface{}) Error + Data() interface{} +} + +// New Error with formatted message. +// The Error's Data will be a FmtError type. +func NewError(format string, args ...interface{}) Error { + err := FmtError{format, args} + return newCmnError(err) +} + +// New Error with specified data. +func NewErrorWithData(data interface{}) Error { + return newCmnError(data) +} + +type cmnError struct { + data interface{} // associated data + msgtraces []msgtraceItem // all messages traced + stacktrace []uintptr // first stack trace +} + +var _ Error = &cmnError{} + +// NOTE: do not expose. +func newCmnError(data interface{}) *cmnError { + return &cmnError{ + data: data, + msgtraces: nil, + stacktrace: nil, + } +} + +// Implements error. +func (err *cmnError) Error() string { + return fmt.Sprintf("%v", err) +} + +// Captures a stacktrace if one was not already captured. +func (err *cmnError) Stacktrace() Error { + if err.stacktrace == nil { + var offset = 3 + var depth = 32 + err.stacktrace = captureStacktrace(offset, depth) + } + return err +} + +// Add tracing information with msg. +// Set n=0 unless wrapped with some function, then n > 0. +func (err *cmnError) Trace(offset int, format string, args ...interface{}) Error { + msg := Fmt(format, args...) + return err.doTrace(msg, offset) +} + +// Return the "data" of this error. +// Data could be used for error handling/switching, +// or for holding general error/debug information. +func (err *cmnError) Data() interface{} { + return err.data +} + +func (err *cmnError) doTrace(msg string, n int) Error { + pc, _, _, _ := runtime.Caller(n + 2) // +1 for doTrace(). +1 for the caller. + // Include file & line number & msg. + // Do not include the whole stack trace. + err.msgtraces = append(err.msgtraces, msgtraceItem{ + pc: pc, + msg: msg, + }) + return err +} + +func (err *cmnError) Format(s fmt.State, verb rune) { + switch verb { + case 'p': + s.Write([]byte(fmt.Sprintf("%p", &err))) + default: + if s.Flag('#') { + s.Write([]byte("--= Error =--\n")) + // Write data. + s.Write([]byte(fmt.Sprintf("Data: %#v\n", err.data))) + // Write msg trace items. + s.Write([]byte(fmt.Sprintf("Msg Traces:\n"))) + for i, msgtrace := range err.msgtraces { + s.Write([]byte(fmt.Sprintf(" %4d %s\n", i, msgtrace.String()))) + } + // Write stack trace. + if err.stacktrace != nil { + s.Write([]byte(fmt.Sprintf("Stack Trace:\n"))) + for i, pc := range err.stacktrace { + fnc := runtime.FuncForPC(pc) + file, line := fnc.FileLine(pc) + s.Write([]byte(fmt.Sprintf(" %4d %s:%d\n", i, file, line))) + } + } + s.Write([]byte("--= /Error =--\n")) + } else { + // Write msg. + s.Write([]byte(fmt.Sprintf("Error{%v}", err.data))) // TODO tick-esc? + } + } +} + +//---------------------------------------- +// stacktrace & msgtraceItem + +func captureStacktrace(offset int, depth int) []uintptr { + var pcs = make([]uintptr, depth) + n := runtime.Callers(offset, pcs) + return pcs[0:n] +} + +type msgtraceItem struct { + pc uintptr + msg string +} + +func (mti msgtraceItem) String() string { + fnc := runtime.FuncForPC(mti.pc) + file, line := fnc.FileLine(mti.pc) + return fmt.Sprintf("%s:%d - %s", + file, line, + mti.msg, + ) +} + +//---------------------------------------- +// fmt error + +/* + +FmtError is the data type for NewError() (e.g. NewError().Data().(FmtError)) +Theoretically it could be used to switch on the format string. + +```go + // Error construction + var err1 error = NewError("invalid username %v", "BOB") + var err2 error = NewError("another kind of error") + ... + // Error handling + switch err1.Data().(cmn.FmtError).Format() { + case "invalid username %v": ... + case "another kind of error": ... + default: ... + } +``` +*/ +type FmtError struct { + format string + args []interface{} +} + +func (fe FmtError) Error() string { + return fmt.Sprintf(fe.format, fe.args...) +} + +func (fe FmtError) String() string { + return fmt.Sprintf("FmtError{format:%v,args:%v}", + fe.format, fe.args) +} + +func (fe FmtError) Format() string { + return fe.format +} + +//---------------------------------------- +// Panic wrappers +// XXX DEPRECATED + +// A panic resulting from a sanity check means there is a programmer error +// and some guarantee is not satisfied. +// XXX DEPRECATED +func PanicSanity(v interface{}) { + panic(Fmt("Panicked on a Sanity Check: %v", v)) +} + +// A panic here means something has gone horribly wrong, in the form of data corruption or +// failure of the operating system. In a correct/healthy system, these should never fire. +// If they do, it's indicative of a much more serious problem. +// XXX DEPRECATED +func PanicCrisis(v interface{}) { + panic(Fmt("Panicked on a Crisis: %v", v)) +} + +// Indicates a failure of consensus. Someone was malicious or something has +// gone horribly wrong. These should really boot us into an "emergency-recover" mode +// XXX DEPRECATED +func PanicConsensus(v interface{}) { + panic(Fmt("Panicked on a Consensus Failure: %v", v)) +} + +// For those times when we're not sure if we should panic +// XXX DEPRECATED +func PanicQ(v interface{}) { + panic(Fmt("Panicked questionably: %v", v)) +} diff --git a/libs/common/errors_test.go b/libs/common/errors_test.go new file mode 100644 index 000000000..52c78a765 --- /dev/null +++ b/libs/common/errors_test.go @@ -0,0 +1,101 @@ +package common + +import ( + fmt "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestErrorPanic(t *testing.T) { + type pnk struct { + msg string + } + + capturePanic := func() (err Error) { + defer func() { + if r := recover(); r != nil { + err = ErrorWrap(r, "This is the message in ErrorWrap(r, message).") + } + }() + panic(pnk{"something"}) + } + + var err = capturePanic() + + assert.Equal(t, pnk{"something"}, err.Data()) + assert.Equal(t, "Error{{something}}", fmt.Sprintf("%v", err)) + assert.Contains(t, fmt.Sprintf("%#v", err), "This is the message in ErrorWrap(r, message).") + assert.Contains(t, fmt.Sprintf("%#v", err), "Stack Trace:\n 0") +} + +func TestErrorWrapSomething(t *testing.T) { + + var err = ErrorWrap("something", "formatter%v%v", 0, 1) + + assert.Equal(t, "something", err.Data()) + assert.Equal(t, "Error{something}", fmt.Sprintf("%v", err)) + assert.Regexp(t, `formatter01\n`, fmt.Sprintf("%#v", err)) + assert.Contains(t, fmt.Sprintf("%#v", err), "Stack Trace:\n 0") +} + +func TestErrorWrapNothing(t *testing.T) { + + var err = ErrorWrap(nil, "formatter%v%v", 0, 1) + + assert.Equal(t, + FmtError{"formatter%v%v", []interface{}{0, 1}}, + err.Data()) + assert.Equal(t, "Error{formatter01}", fmt.Sprintf("%v", err)) + assert.Contains(t, fmt.Sprintf("%#v", err), `Data: common.FmtError{format:"formatter%v%v", args:[]interface {}{0, 1}}`) + assert.Contains(t, fmt.Sprintf("%#v", err), "Stack Trace:\n 0") +} + +func TestErrorNewError(t *testing.T) { + + var err = NewError("formatter%v%v", 0, 1) + + assert.Equal(t, + FmtError{"formatter%v%v", []interface{}{0, 1}}, + err.Data()) + assert.Equal(t, "Error{formatter01}", fmt.Sprintf("%v", err)) + assert.Contains(t, fmt.Sprintf("%#v", err), `Data: common.FmtError{format:"formatter%v%v", args:[]interface {}{0, 1}}`) + assert.NotContains(t, fmt.Sprintf("%#v", err), "Stack Trace") +} + +func TestErrorNewErrorWithStacktrace(t *testing.T) { + + var err = NewError("formatter%v%v", 0, 1).Stacktrace() + + assert.Equal(t, + FmtError{"formatter%v%v", []interface{}{0, 1}}, + err.Data()) + assert.Equal(t, "Error{formatter01}", fmt.Sprintf("%v", err)) + assert.Contains(t, fmt.Sprintf("%#v", err), `Data: common.FmtError{format:"formatter%v%v", args:[]interface {}{0, 1}}`) + assert.Contains(t, fmt.Sprintf("%#v", err), "Stack Trace:\n 0") +} + +func TestErrorNewErrorWithTrace(t *testing.T) { + + var err = NewError("formatter%v%v", 0, 1) + err.Trace(0, "trace %v", 1) + err.Trace(0, "trace %v", 2) + err.Trace(0, "trace %v", 3) + + assert.Equal(t, + FmtError{"formatter%v%v", []interface{}{0, 1}}, + err.Data()) + assert.Equal(t, "Error{formatter01}", fmt.Sprintf("%v", err)) + assert.Contains(t, fmt.Sprintf("%#v", err), `Data: common.FmtError{format:"formatter%v%v", args:[]interface {}{0, 1}}`) + dump := fmt.Sprintf("%#v", err) + assert.NotContains(t, dump, "Stack Trace") + assert.Regexp(t, `common/errors_test\.go:[0-9]+ - trace 1`, dump) + assert.Regexp(t, `common/errors_test\.go:[0-9]+ - trace 2`, dump) + assert.Regexp(t, `common/errors_test\.go:[0-9]+ - trace 3`, dump) +} + +func TestErrorWrapError(t *testing.T) { + var err1 error = NewError("my message") + var err2 error = ErrorWrap(err1, "another message") + assert.Equal(t, err1, err2) +} diff --git a/libs/common/heap.go b/libs/common/heap.go new file mode 100644 index 000000000..b3bcb9db8 --- /dev/null +++ b/libs/common/heap.go @@ -0,0 +1,125 @@ +package common + +import ( + "bytes" + "container/heap" +) + +/* + Example usage: + + ``` + h := NewHeap() + + h.Push("msg1", 1) + h.Push("msg3", 3) + h.Push("msg2", 2) + + fmt.Println(h.Pop()) // msg1 + fmt.Println(h.Pop()) // msg2 + fmt.Println(h.Pop()) // msg3 + ``` +*/ +type Heap struct { + pq priorityQueue +} + +func NewHeap() *Heap { + return &Heap{pq: make([]*pqItem, 0)} +} + +func (h *Heap) Len() int64 { + return int64(len(h.pq)) +} + +func (h *Heap) Push(value interface{}, priority int) { + heap.Push(&h.pq, &pqItem{value: value, priority: cmpInt(priority)}) +} + +func (h *Heap) PushBytes(value interface{}, priority []byte) { + heap.Push(&h.pq, &pqItem{value: value, priority: cmpBytes(priority)}) +} + +func (h *Heap) PushComparable(value interface{}, priority Comparable) { + heap.Push(&h.pq, &pqItem{value: value, priority: priority}) +} + +func (h *Heap) Peek() interface{} { + if len(h.pq) == 0 { + return nil + } + return h.pq[0].value +} + +func (h *Heap) Update(value interface{}, priority Comparable) { + h.pq.Update(h.pq[0], value, priority) +} + +func (h *Heap) Pop() interface{} { + item := heap.Pop(&h.pq).(*pqItem) + return item.value +} + +//----------------------------------------------------------------------------- +// From: http://golang.org/pkg/container/heap/#example__priorityQueue + +type pqItem struct { + value interface{} + priority Comparable + index int +} + +type priorityQueue []*pqItem + +func (pq priorityQueue) Len() int { return len(pq) } + +func (pq priorityQueue) Less(i, j int) bool { + return pq[i].priority.Less(pq[j].priority) +} + +func (pq priorityQueue) Swap(i, j int) { + pq[i], pq[j] = pq[j], pq[i] + pq[i].index = i + pq[j].index = j +} + +func (pq *priorityQueue) Push(x interface{}) { + n := len(*pq) + item := x.(*pqItem) + item.index = n + *pq = append(*pq, item) +} + +func (pq *priorityQueue) Pop() interface{} { + old := *pq + n := len(old) + item := old[n-1] + item.index = -1 // for safety + *pq = old[0 : n-1] + return item +} + +func (pq *priorityQueue) Update(item *pqItem, value interface{}, priority Comparable) { + item.value = value + item.priority = priority + heap.Fix(pq, item.index) +} + +//-------------------------------------------------------------------------------- +// Comparable + +type Comparable interface { + Less(o interface{}) bool +} + +type cmpInt int + +func (i cmpInt) Less(o interface{}) bool { + return int(i) < int(o.(cmpInt)) +} + +type cmpBytes []byte + +func (bz cmpBytes) Less(o interface{}) bool { + return bytes.Compare([]byte(bz), []byte(o.(cmpBytes))) < 0 +} diff --git a/libs/common/int.go b/libs/common/int.go new file mode 100644 index 000000000..a8a5f1e00 --- /dev/null +++ b/libs/common/int.go @@ -0,0 +1,65 @@ +package common + +import ( + "encoding/binary" + "sort" +) + +// Sort for []uint64 + +type Uint64Slice []uint64 + +func (p Uint64Slice) Len() int { return len(p) } +func (p Uint64Slice) Less(i, j int) bool { return p[i] < p[j] } +func (p Uint64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p Uint64Slice) Sort() { sort.Sort(p) } + +func SearchUint64s(a []uint64, x uint64) int { + return sort.Search(len(a), func(i int) bool { return a[i] >= x }) +} + +func (p Uint64Slice) Search(x uint64) int { return SearchUint64s(p, x) } + +//-------------------------------------------------------------------------------- + +func PutUint64LE(dest []byte, i uint64) { + binary.LittleEndian.PutUint64(dest, i) +} + +func GetUint64LE(src []byte) uint64 { + return binary.LittleEndian.Uint64(src) +} + +func PutUint64BE(dest []byte, i uint64) { + binary.BigEndian.PutUint64(dest, i) +} + +func GetUint64BE(src []byte) uint64 { + return binary.BigEndian.Uint64(src) +} + +func PutInt64LE(dest []byte, i int64) { + binary.LittleEndian.PutUint64(dest, uint64(i)) +} + +func GetInt64LE(src []byte) int64 { + return int64(binary.LittleEndian.Uint64(src)) +} + +func PutInt64BE(dest []byte, i int64) { + binary.BigEndian.PutUint64(dest, uint64(i)) +} + +func GetInt64BE(src []byte) int64 { + return int64(binary.BigEndian.Uint64(src)) +} + +// IntInSlice returns true if a is found in the list. +func IntInSlice(a int, list []int) bool { + for _, b := range list { + if b == a { + return true + } + } + return false +} diff --git a/libs/common/int_test.go b/libs/common/int_test.go new file mode 100644 index 000000000..1ecc7844c --- /dev/null +++ b/libs/common/int_test.go @@ -0,0 +1,14 @@ +package common + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIntInSlice(t *testing.T) { + assert.True(t, IntInSlice(1, []int{1, 2, 3})) + assert.False(t, IntInSlice(4, []int{1, 2, 3})) + assert.True(t, IntInSlice(0, []int{0})) + assert.False(t, IntInSlice(0, []int{})) +} diff --git a/libs/common/io.go b/libs/common/io.go new file mode 100644 index 000000000..fa0443e09 --- /dev/null +++ b/libs/common/io.go @@ -0,0 +1,74 @@ +package common + +import ( + "bytes" + "errors" + "io" +) + +type PrefixedReader struct { + Prefix []byte + reader io.Reader +} + +func NewPrefixedReader(prefix []byte, reader io.Reader) *PrefixedReader { + return &PrefixedReader{prefix, reader} +} + +func (pr *PrefixedReader) Read(p []byte) (n int, err error) { + if len(pr.Prefix) > 0 { + read := copy(p, pr.Prefix) + pr.Prefix = pr.Prefix[read:] + return read, nil + } + return pr.reader.Read(p) +} + +// NOTE: Not goroutine safe +type BufferCloser struct { + bytes.Buffer + Closed bool +} + +func NewBufferCloser(buf []byte) *BufferCloser { + return &BufferCloser{ + *bytes.NewBuffer(buf), + false, + } +} + +func (bc *BufferCloser) Close() error { + if bc.Closed { + return errors.New("BufferCloser already closed") + } + bc.Closed = true + return nil +} + +func (bc *BufferCloser) Write(p []byte) (n int, err error) { + if bc.Closed { + return 0, errors.New("Cannot write to closed BufferCloser") + } + return bc.Buffer.Write(p) +} + +func (bc *BufferCloser) WriteByte(c byte) error { + if bc.Closed { + return errors.New("Cannot write to closed BufferCloser") + } + return bc.Buffer.WriteByte(c) +} + +func (bc *BufferCloser) WriteRune(r rune) (n int, err error) { + if bc.Closed { + return 0, errors.New("Cannot write to closed BufferCloser") + } + return bc.Buffer.WriteRune(r) +} + +func (bc *BufferCloser) WriteString(s string) (n int, err error) { + if bc.Closed { + return 0, errors.New("Cannot write to closed BufferCloser") + } + return bc.Buffer.WriteString(s) +} diff --git a/libs/common/kvpair.go b/libs/common/kvpair.go new file mode 100644 index 000000000..54c3a58c0 --- /dev/null +++ b/libs/common/kvpair.go @@ -0,0 +1,67 @@ +package common + +import ( + "bytes" + "sort" +) + +//---------------------------------------- +// KVPair + +/* +Defined in types.proto + +type KVPair struct { + Key []byte + Value []byte +} +*/ + +type KVPairs []KVPair + +// Sorting +func (kvs KVPairs) Len() int { return len(kvs) } +func (kvs KVPairs) Less(i, j int) bool { + switch bytes.Compare(kvs[i].Key, kvs[j].Key) { + case -1: + return true + case 0: + return bytes.Compare(kvs[i].Value, kvs[j].Value) < 0 + case 1: + return false + default: + panic("invalid comparison result") + } +} +func (kvs KVPairs) Swap(i, j int) { kvs[i], kvs[j] = kvs[j], kvs[i] } +func (kvs KVPairs) Sort() { sort.Sort(kvs) } + +//---------------------------------------- +// KI64Pair + +/* +Defined in types.proto +type KI64Pair struct { + Key []byte + Value int64 +} +*/ + +type KI64Pairs []KI64Pair + +// Sorting +func (kvs KI64Pairs) Len() int { return len(kvs) } +func (kvs KI64Pairs) Less(i, j int) bool { + switch bytes.Compare(kvs[i].Key, kvs[j].Key) { + case -1: + return true + case 0: + return kvs[i].Value < kvs[j].Value + case 1: + return false + default: + panic("invalid comparison result") + } +} +func (kvs KI64Pairs) Swap(i, j int) { kvs[i], kvs[j] = kvs[j], kvs[i] } +func (kvs KI64Pairs) Sort() { sort.Sort(kvs) } diff --git a/libs/common/math.go b/libs/common/math.go new file mode 100644 index 000000000..b037d1a71 --- /dev/null +++ b/libs/common/math.go @@ -0,0 +1,157 @@ +package common + +func MaxInt8(a, b int8) int8 { + if a > b { + return a + } + return b +} + +func MaxUint8(a, b uint8) uint8 { + if a > b { + return a + } + return b +} + +func MaxInt16(a, b int16) int16 { + if a > b { + return a + } + return b +} + +func MaxUint16(a, b uint16) uint16 { + if a > b { + return a + } + return b +} + +func MaxInt32(a, b int32) int32 { + if a > b { + return a + } + return b +} + +func MaxUint32(a, b uint32) uint32 { + if a > b { + return a + } + return b +} + +func MaxInt64(a, b int64) int64 { + if a > b { + return a + } + return b +} + +func MaxUint64(a, b uint64) uint64 { + if a > b { + return a + } + return b +} + +func MaxInt(a, b int) int { + if a > b { + return a + } + return b +} + +func MaxUint(a, b uint) uint { + if a > b { + return a + } + return b +} + +//----------------------------------------------------------------------------- + +func MinInt8(a, b int8) int8 { + if a < b { + return a + } + return b +} + +func MinUint8(a, b uint8) uint8 { + if a < b { + return a + } + return b +} + +func MinInt16(a, b int16) int16 { + if a < b { + return a + } + return b +} + +func MinUint16(a, b uint16) uint16 { + if a < b { + return a + } + return b +} + +func MinInt32(a, b int32) int32 { + if a < b { + return a + } + return b +} + +func MinUint32(a, b uint32) uint32 { + if a < b { + return a + } + return b +} + +func MinInt64(a, b int64) int64 { + if a < b { + return a + } + return b +} + +func MinUint64(a, b uint64) uint64 { + if a < b { + return a + } + return b +} + +func MinInt(a, b int) int { + if a < b { + return a + } + return b +} + +func MinUint(a, b uint) uint { + if a < b { + return a + } + return b +} + +//----------------------------------------------------------------------------- + +func ExpUint64(a, b uint64) uint64 { + accum := uint64(1) + for b > 0 { + if b&1 == 1 { + accum *= a + } + a *= a + b >>= 1 + } + return accum +} diff --git a/libs/common/net.go b/libs/common/net.go new file mode 100644 index 000000000..bdbe38f79 --- /dev/null +++ b/libs/common/net.go @@ -0,0 +1,26 @@ +package common + +import ( + "net" + "strings" +) + +// Connect dials the given address and returns a net.Conn. The protoAddr argument should be prefixed with the protocol, +// eg. "tcp://127.0.0.1:8080" or "unix:///tmp/test.sock" +func Connect(protoAddr string) (net.Conn, error) { + proto, address := ProtocolAndAddress(protoAddr) + conn, err := net.Dial(proto, address) + return conn, err +} + +// ProtocolAndAddress splits an address into the protocol and address components. +// For instance, "tcp://127.0.0.1:8080" will be split into "tcp" and "127.0.0.1:8080". +// If the address has no protocol prefix, the default is "tcp". +func ProtocolAndAddress(listenAddr string) (string, string) { + protocol, address := "tcp", listenAddr + parts := strings.SplitN(address, "://", 2) + if len(parts) == 2 { + protocol, address = parts[0], parts[1] + } + return protocol, address +} diff --git a/libs/common/net_test.go b/libs/common/net_test.go new file mode 100644 index 000000000..38d2ae82d --- /dev/null +++ b/libs/common/net_test.go @@ -0,0 +1,38 @@ +package common + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestProtocolAndAddress(t *testing.T) { + + cases := []struct { + fullAddr string + proto string + addr string + }{ + { + "tcp://mydomain:80", + "tcp", + "mydomain:80", + }, + { + "mydomain:80", + "tcp", + "mydomain:80", + }, + { + "unix://mydomain:80", + "unix", + "mydomain:80", + }, + } + + for _, c := range cases { + proto, addr := ProtocolAndAddress(c.fullAddr) + assert.Equal(t, proto, c.proto) + assert.Equal(t, addr, c.addr) + } +} diff --git a/libs/common/nil.go b/libs/common/nil.go new file mode 100644 index 000000000..31f75f008 --- /dev/null +++ b/libs/common/nil.go @@ -0,0 +1,29 @@ +package common + +import "reflect" + +// Go lacks a simple and safe way to see if something is a typed nil. +// See: +// - https://dave.cheney.net/2017/08/09/typed-nils-in-go-2 +// - https://groups.google.com/forum/#!topic/golang-nuts/wnH302gBa4I/discussion +// - https://github.com/golang/go/issues/21538 +func IsTypedNil(o interface{}) bool { + rv := reflect.ValueOf(o) + switch rv.Kind() { + case reflect.Chan, reflect.Func, reflect.Map, reflect.Ptr, reflect.Slice: + return rv.IsNil() + default: + return false + } +} + +// Returns true if it has zero length. +func IsEmpty(o interface{}) bool { + rv := reflect.ValueOf(o) + switch rv.Kind() { + case reflect.Array, reflect.Chan, reflect.Map, reflect.Slice, reflect.String: + return rv.Len() == 0 + default: + return false + } +} diff --git a/libs/common/os.go b/libs/common/os.go new file mode 100644 index 000000000..00f4da57b --- /dev/null +++ b/libs/common/os.go @@ -0,0 +1,195 @@ +package common + +import ( + "bufio" + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "os/signal" + "path/filepath" + "strings" + "syscall" +) + +var gopath string + +// GoPath returns GOPATH env variable value. If it is not set, this function +// will try to call `go env GOPATH` subcommand. +func GoPath() string { + if gopath != "" { + return gopath + } + + path := os.Getenv("GOPATH") + if len(path) == 0 { + goCmd := exec.Command("go", "env", "GOPATH") + out, err := goCmd.Output() + if err != nil { + panic(fmt.Sprintf("failed to determine gopath: %v", err)) + } + path = string(out) + } + gopath = path + return path +} + +// TrapSignal catches the SIGTERM and executes cb function. After that it exits +// with code 1. +func TrapSignal(cb func()) { + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt, syscall.SIGTERM) + go func() { + for sig := range c { + fmt.Printf("captured %v, exiting...\n", sig) + if cb != nil { + cb() + } + os.Exit(1) + } + }() + select {} +} + +// Kill the running process by sending itself SIGTERM. +func Kill() error { + p, err := os.FindProcess(os.Getpid()) + if err != nil { + return err + } + return p.Signal(syscall.SIGTERM) +} + +func Exit(s string) { + fmt.Printf(s + "\n") + os.Exit(1) +} + +func EnsureDir(dir string, mode os.FileMode) error { + if _, err := os.Stat(dir); os.IsNotExist(err) { + err := os.MkdirAll(dir, mode) + if err != nil { + return fmt.Errorf("Could not create directory %v. %v", dir, err) + } + } + return nil +} + +func IsDirEmpty(name string) (bool, error) { + f, err := os.Open(name) + if err != nil { + if os.IsNotExist(err) { + return true, err + } + // Otherwise perhaps a permission + // error or some other error. + return false, err + } + defer f.Close() + + _, err = f.Readdirnames(1) // Or f.Readdir(1) + if err == io.EOF { + return true, nil + } + return false, err // Either not empty or error, suits both cases +} + +func FileExists(filePath string) bool { + _, err := os.Stat(filePath) + return !os.IsNotExist(err) +} + +func ReadFile(filePath string) ([]byte, error) { + return ioutil.ReadFile(filePath) +} + +func MustReadFile(filePath string) []byte { + fileBytes, err := ioutil.ReadFile(filePath) + if err != nil { + Exit(Fmt("MustReadFile failed: %v", err)) + return nil + } + return fileBytes +} + +func WriteFile(filePath string, contents []byte, mode os.FileMode) error { + return ioutil.WriteFile(filePath, contents, mode) +} + +func MustWriteFile(filePath string, contents []byte, mode os.FileMode) { + err := WriteFile(filePath, contents, mode) + if err != nil { + Exit(Fmt("MustWriteFile failed: %v", err)) + } +} + +// WriteFileAtomic creates a temporary file with data and the perm given and +// swaps it atomically with filename if successful. +func WriteFileAtomic(filename string, data []byte, perm os.FileMode) error { + var ( + dir = filepath.Dir(filename) + tempFile = filepath.Join(dir, "write-file-atomic-"+RandStr(32)) + // Override in case it does exist, create in case it doesn't and force kernel + // flush, which still leaves the potential of lingering disk cache. + flag = os.O_WRONLY | os.O_CREATE | os.O_SYNC | os.O_TRUNC + ) + + f, err := os.OpenFile(tempFile, flag, perm) + if err != nil { + return err + } + // Clean up in any case. Defer stacking order is last-in-first-out. + defer os.Remove(f.Name()) + defer f.Close() + + if n, err := f.Write(data); err != nil { + return err + } else if n < len(data) { + return io.ErrShortWrite + } + // Close the file before renaming it, otherwise it will cause "The process + // cannot access the file because it is being used by another process." on windows. + f.Close() + + return os.Rename(f.Name(), filename) +} + +//-------------------------------------------------------------------------------- + +func Tempfile(prefix string) (*os.File, string) { + file, err := ioutil.TempFile("", prefix) + if err != nil { + PanicCrisis(err) + } + return file, file.Name() +} + +func Tempdir(prefix string) (*os.File, string) { + tempDir := os.TempDir() + "/" + prefix + RandStr(12) + err := EnsureDir(tempDir, 0700) + if err != nil { + panic(Fmt("Error creating temp dir: %v", err)) + } + dir, err := os.Open(tempDir) + if err != nil { + panic(Fmt("Error opening temp dir: %v", err)) + } + return dir, tempDir +} + +//-------------------------------------------------------------------------------- + +func Prompt(prompt string, defaultValue string) (string, error) { + fmt.Print(prompt) + reader := bufio.NewReader(os.Stdin) + line, err := reader.ReadString('\n') + if err != nil { + return defaultValue, err + } + line = strings.TrimSpace(line) + if line == "" { + return defaultValue, nil + } + return line, nil +} diff --git a/libs/common/os_test.go b/libs/common/os_test.go new file mode 100644 index 000000000..973d68901 --- /dev/null +++ b/libs/common/os_test.go @@ -0,0 +1,91 @@ +package common + +import ( + "bytes" + "io/ioutil" + "math/rand" + "os" + "testing" + "time" +) + +func TestWriteFileAtomic(t *testing.T) { + var ( + seed = rand.New(rand.NewSource(time.Now().UnixNano())) + data = []byte(RandStr(seed.Intn(2048))) + old = RandBytes(seed.Intn(2048)) + perm os.FileMode = 0600 + ) + + f, err := ioutil.TempFile("/tmp", "write-atomic-test-") + if err != nil { + t.Fatal(err) + } + defer os.Remove(f.Name()) + + if err = ioutil.WriteFile(f.Name(), old, 0664); err != nil { + t.Fatal(err) + } + + if err = WriteFileAtomic(f.Name(), data, perm); err != nil { + t.Fatal(err) + } + + rData, err := ioutil.ReadFile(f.Name()) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(data, rData) { + t.Fatalf("data mismatch: %v != %v", data, rData) + } + + stat, err := os.Stat(f.Name()) + if err != nil { + t.Fatal(err) + } + + if have, want := stat.Mode().Perm(), perm; have != want { + t.Errorf("have %v, want %v", have, want) + } +} + +func TestGoPath(t *testing.T) { + // restore original gopath upon exit + path := os.Getenv("GOPATH") + defer func() { + _ = os.Setenv("GOPATH", path) + }() + + err := os.Setenv("GOPATH", "~/testgopath") + if err != nil { + t.Fatal(err) + } + path = GoPath() + if path != "~/testgopath" { + t.Fatalf("should get GOPATH env var value, got %v", path) + } + os.Unsetenv("GOPATH") + + path = GoPath() + if path != "~/testgopath" { + t.Fatalf("subsequent calls should return the same value, got %v", path) + } +} + +func TestGoPathWithoutEnvVar(t *testing.T) { + // restore original gopath upon exit + path := os.Getenv("GOPATH") + defer func() { + _ = os.Setenv("GOPATH", path) + }() + + os.Unsetenv("GOPATH") + // reset cache + gopath = "" + + path = GoPath() + if path == "" || path == "~/testgopath" { + t.Fatalf("should get nonempty result of calling go env GOPATH, got %v", path) + } +} diff --git a/libs/common/random.go b/libs/common/random.go new file mode 100644 index 000000000..389a32fc2 --- /dev/null +++ b/libs/common/random.go @@ -0,0 +1,357 @@ +package common + +import ( + crand "crypto/rand" + mrand "math/rand" + "sync" + "time" +) + +const ( + strChars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" // 62 characters +) + +// pseudo random number generator. +// seeded with OS randomness (crand) + +type Rand struct { + sync.Mutex + rand *mrand.Rand +} + +var grand *Rand + +func init() { + grand = NewRand() + grand.init() +} + +func NewRand() *Rand { + rand := &Rand{} + rand.init() + return rand +} + +func (r *Rand) init() { + bz := cRandBytes(8) + var seed uint64 + for i := 0; i < 8; i++ { + seed |= uint64(bz[i]) + seed <<= 8 + } + r.reset(int64(seed)) +} + +func (r *Rand) reset(seed int64) { + r.rand = mrand.New(mrand.NewSource(seed)) +} + +//---------------------------------------- +// Global functions + +func Seed(seed int64) { + grand.Seed(seed) +} + +func RandStr(length int) string { + return grand.Str(length) +} + +func RandUint16() uint16 { + return grand.Uint16() +} + +func RandUint32() uint32 { + return grand.Uint32() +} + +func RandUint64() uint64 { + return grand.Uint64() +} + +func RandUint() uint { + return grand.Uint() +} + +func RandInt16() int16 { + return grand.Int16() +} + +func RandInt32() int32 { + return grand.Int32() +} + +func RandInt64() int64 { + return grand.Int64() +} + +func RandInt() int { + return grand.Int() +} + +func RandInt31() int32 { + return grand.Int31() +} + +func RandInt31n(n int32) int32 { + return grand.Int31n(n) +} + +func RandInt63() int64 { + return grand.Int63() +} + +func RandInt63n(n int64) int64 { + return grand.Int63n(n) +} + +func RandUint16Exp() uint16 { + return grand.Uint16Exp() +} + +func RandUint32Exp() uint32 { + return grand.Uint32Exp() +} + +func RandUint64Exp() uint64 { + return grand.Uint64Exp() +} + +func RandFloat32() float32 { + return grand.Float32() +} + +func RandFloat64() float64 { + return grand.Float64() +} + +func RandTime() time.Time { + return grand.Time() +} + +func RandBytes(n int) []byte { + return grand.Bytes(n) +} + +func RandIntn(n int) int { + return grand.Intn(n) +} + +func RandPerm(n int) []int { + return grand.Perm(n) +} + +//---------------------------------------- +// Rand methods + +func (r *Rand) Seed(seed int64) { + r.Lock() + r.reset(seed) + r.Unlock() +} + +// Constructs an alphanumeric string of given length. +// It is not safe for cryptographic usage. +func (r *Rand) Str(length int) string { + chars := []byte{} +MAIN_LOOP: + for { + val := r.Int63() + for i := 0; i < 10; i++ { + v := int(val & 0x3f) // rightmost 6 bits + if v >= 62 { // only 62 characters in strChars + val >>= 6 + continue + } else { + chars = append(chars, strChars[v]) + if len(chars) == length { + break MAIN_LOOP + } + val >>= 6 + } + } + } + + return string(chars) +} + +// It is not safe for cryptographic usage. +func (r *Rand) Uint16() uint16 { + return uint16(r.Uint32() & (1<<16 - 1)) +} + +// It is not safe for cryptographic usage. +func (r *Rand) Uint32() uint32 { + r.Lock() + u32 := r.rand.Uint32() + r.Unlock() + return u32 +} + +// It is not safe for cryptographic usage. +func (r *Rand) Uint64() uint64 { + return uint64(r.Uint32())<<32 + uint64(r.Uint32()) +} + +// It is not safe for cryptographic usage. +func (r *Rand) Uint() uint { + r.Lock() + i := r.rand.Int() + r.Unlock() + return uint(i) +} + +// It is not safe for cryptographic usage. +func (r *Rand) Int16() int16 { + return int16(r.Uint32() & (1<<16 - 1)) +} + +// It is not safe for cryptographic usage. +func (r *Rand) Int32() int32 { + return int32(r.Uint32()) +} + +// It is not safe for cryptographic usage. +func (r *Rand) Int64() int64 { + return int64(r.Uint64()) +} + +// It is not safe for cryptographic usage. +func (r *Rand) Int() int { + r.Lock() + i := r.rand.Int() + r.Unlock() + return i +} + +// It is not safe for cryptographic usage. +func (r *Rand) Int31() int32 { + r.Lock() + i31 := r.rand.Int31() + r.Unlock() + return i31 +} + +// It is not safe for cryptographic usage. +func (r *Rand) Int31n(n int32) int32 { + r.Lock() + i31n := r.rand.Int31n(n) + r.Unlock() + return i31n +} + +// It is not safe for cryptographic usage. +func (r *Rand) Int63() int64 { + r.Lock() + i63 := r.rand.Int63() + r.Unlock() + return i63 +} + +// It is not safe for cryptographic usage. +func (r *Rand) Int63n(n int64) int64 { + r.Lock() + i63n := r.rand.Int63n(n) + r.Unlock() + return i63n +} + +// Distributed pseudo-exponentially to test for various cases +// It is not safe for cryptographic usage. +func (r *Rand) Uint16Exp() uint16 { + bits := r.Uint32() % 16 + if bits == 0 { + return 0 + } + n := uint16(1 << (bits - 1)) + n += uint16(r.Int31()) & ((1 << (bits - 1)) - 1) + return n +} + +// Distributed pseudo-exponentially to test for various cases +// It is not safe for cryptographic usage. +func (r *Rand) Uint32Exp() uint32 { + bits := r.Uint32() % 32 + if bits == 0 { + return 0 + } + n := uint32(1 << (bits - 1)) + n += uint32(r.Int31()) & ((1 << (bits - 1)) - 1) + return n +} + +// Distributed pseudo-exponentially to test for various cases +// It is not safe for cryptographic usage. +func (r *Rand) Uint64Exp() uint64 { + bits := r.Uint32() % 64 + if bits == 0 { + return 0 + } + n := uint64(1 << (bits - 1)) + n += uint64(r.Int63()) & ((1 << (bits - 1)) - 1) + return n +} + +// It is not safe for cryptographic usage. +func (r *Rand) Float32() float32 { + r.Lock() + f32 := r.rand.Float32() + r.Unlock() + return f32 +} + +// It is not safe for cryptographic usage. +func (r *Rand) Float64() float64 { + r.Lock() + f64 := r.rand.Float64() + r.Unlock() + return f64 +} + +// It is not safe for cryptographic usage. +func (r *Rand) Time() time.Time { + return time.Unix(int64(r.Uint64Exp()), 0) +} + +// RandBytes returns n random bytes from the OS's source of entropy ie. via crypto/rand. +// It is not safe for cryptographic usage. +func (r *Rand) Bytes(n int) []byte { + // cRandBytes isn't guaranteed to be fast so instead + // use random bytes generated from the internal PRNG + bs := make([]byte, n) + for i := 0; i < len(bs); i++ { + bs[i] = byte(r.Int() & 0xFF) + } + return bs +} + +// RandIntn returns, as an int, a non-negative pseudo-random number in [0, n). +// It panics if n <= 0. +// It is not safe for cryptographic usage. +func (r *Rand) Intn(n int) int { + r.Lock() + i := r.rand.Intn(n) + r.Unlock() + return i +} + +// RandPerm returns a pseudo-random permutation of n integers in [0, n). +// It is not safe for cryptographic usage. +func (r *Rand) Perm(n int) []int { + r.Lock() + perm := r.rand.Perm(n) + r.Unlock() + return perm +} + +// NOTE: This relies on the os's random number generator. +// For real security, we should salt that with some seed. +// See github.com/tendermint/go-crypto for a more secure reader. +func cRandBytes(numBytes int) []byte { + b := make([]byte, numBytes) + _, err := crand.Read(b) + if err != nil { + PanicCrisis(err) + } + return b +} diff --git a/libs/common/random_test.go b/libs/common/random_test.go new file mode 100644 index 000000000..b58b4a13a --- /dev/null +++ b/libs/common/random_test.go @@ -0,0 +1,121 @@ +package common + +import ( + "bytes" + "encoding/json" + "fmt" + mrand "math/rand" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestRandStr(t *testing.T) { + l := 243 + s := RandStr(l) + assert.Equal(t, l, len(s)) +} + +func TestRandBytes(t *testing.T) { + l := 243 + b := RandBytes(l) + assert.Equal(t, l, len(b)) +} + +func TestRandIntn(t *testing.T) { + n := 243 + for i := 0; i < 100; i++ { + x := RandIntn(n) + assert.True(t, x < n) + } +} + +// Test to make sure that we never call math.rand(). +// We do this by ensuring that outputs are deterministic. +func TestDeterminism(t *testing.T) { + var firstOutput string + + // Set math/rand's seed for the sake of debugging this test. + // (It isn't strictly necessary). + mrand.Seed(1) + + for i := 0; i < 100; i++ { + output := testThemAll() + if i == 0 { + firstOutput = output + } else { + if firstOutput != output { + t.Errorf("Run #%d's output was different from first run.\nfirst: %v\nlast: %v", + i, firstOutput, output) + } + } + } +} + +func testThemAll() string { + + // Such determinism. + grand.reset(1) + + // Use it. + out := new(bytes.Buffer) + perm := RandPerm(10) + blob, _ := json.Marshal(perm) + fmt.Fprintf(out, "perm: %s\n", blob) + fmt.Fprintf(out, "randInt: %d\n", RandInt()) + fmt.Fprintf(out, "randUint: %d\n", RandUint()) + fmt.Fprintf(out, "randIntn: %d\n", RandIntn(97)) + fmt.Fprintf(out, "randInt31: %d\n", RandInt31()) + fmt.Fprintf(out, "randInt32: %d\n", RandInt32()) + fmt.Fprintf(out, "randInt63: %d\n", RandInt63()) + fmt.Fprintf(out, "randInt64: %d\n", RandInt64()) + fmt.Fprintf(out, "randUint32: %d\n", RandUint32()) + fmt.Fprintf(out, "randUint64: %d\n", RandUint64()) + fmt.Fprintf(out, "randUint16Exp: %d\n", RandUint16Exp()) + fmt.Fprintf(out, "randUint32Exp: %d\n", RandUint32Exp()) + fmt.Fprintf(out, "randUint64Exp: %d\n", RandUint64Exp()) + return out.String() +} + +func TestRngConcurrencySafety(t *testing.T) { + var wg sync.WaitGroup + for i := 0; i < 100; i++ { + wg.Add(1) + go func() { + defer wg.Done() + + _ = RandUint64() + <-time.After(time.Millisecond * time.Duration(RandIntn(100))) + _ = RandPerm(3) + }() + } + wg.Wait() +} + +func BenchmarkRandBytes10B(b *testing.B) { + benchmarkRandBytes(b, 10) +} +func BenchmarkRandBytes100B(b *testing.B) { + benchmarkRandBytes(b, 100) +} +func BenchmarkRandBytes1KiB(b *testing.B) { + benchmarkRandBytes(b, 1024) +} +func BenchmarkRandBytes10KiB(b *testing.B) { + benchmarkRandBytes(b, 10*1024) +} +func BenchmarkRandBytes100KiB(b *testing.B) { + benchmarkRandBytes(b, 100*1024) +} +func BenchmarkRandBytes1MiB(b *testing.B) { + benchmarkRandBytes(b, 1024*1024) +} + +func benchmarkRandBytes(b *testing.B, n int) { + for i := 0; i < b.N; i++ { + _ = RandBytes(n) + } + b.ReportAllocs() +} diff --git a/libs/common/repeat_timer.go b/libs/common/repeat_timer.go new file mode 100644 index 000000000..5d049738d --- /dev/null +++ b/libs/common/repeat_timer.go @@ -0,0 +1,232 @@ +package common + +import ( + "sync" + "time" +) + +// Used by RepeatTimer the first time, +// and every time it's Reset() after Stop(). +type TickerMaker func(dur time.Duration) Ticker + +// Ticker is a basic ticker interface. +type Ticker interface { + + // Never changes, never closes. + Chan() <-chan time.Time + + // Stopping a stopped Ticker will panic. + Stop() +} + +//---------------------------------------- +// defaultTicker + +var _ Ticker = (*defaultTicker)(nil) + +type defaultTicker time.Ticker + +func defaultTickerMaker(dur time.Duration) Ticker { + ticker := time.NewTicker(dur) + return (*defaultTicker)(ticker) +} + +// Implements Ticker +func (t *defaultTicker) Chan() <-chan time.Time { + return t.C +} + +// Implements Ticker +func (t *defaultTicker) Stop() { + ((*time.Ticker)(t)).Stop() +} + +//---------------------------------------- +// LogicalTickerMaker + +// Construct a TickerMaker that always uses `source`. +// It's useful for simulating a deterministic clock. +func NewLogicalTickerMaker(source chan time.Time) TickerMaker { + return func(dur time.Duration) Ticker { + return newLogicalTicker(source, dur) + } +} + +type logicalTicker struct { + source <-chan time.Time + ch chan time.Time + quit chan struct{} +} + +func newLogicalTicker(source <-chan time.Time, interval time.Duration) Ticker { + lt := &logicalTicker{ + source: source, + ch: make(chan time.Time), + quit: make(chan struct{}), + } + go lt.fireRoutine(interval) + return lt +} + +// We need a goroutine to read times from t.source +// and fire on t.Chan() when `interval` has passed. +func (t *logicalTicker) fireRoutine(interval time.Duration) { + source := t.source + + // Init `lasttime` + lasttime := time.Time{} + select { + case lasttime = <-source: + case <-t.quit: + return + } + // Init `lasttime` end + + for { + select { + case newtime := <-source: + elapsed := newtime.Sub(lasttime) + if interval <= elapsed { + // Block for determinism until the ticker is stopped. + select { + case t.ch <- newtime: + case <-t.quit: + return + } + // Reset timeleft. + // Don't try to "catch up" by sending more. + // "Ticker adjusts the intervals or drops ticks to make up for + // slow receivers" - https://golang.org/pkg/time/#Ticker + lasttime = newtime + } + case <-t.quit: + return // done + } + } +} + +// Implements Ticker +func (t *logicalTicker) Chan() <-chan time.Time { + return t.ch // immutable +} + +// Implements Ticker +func (t *logicalTicker) Stop() { + close(t.quit) // it *should* panic when stopped twice. +} + +//--------------------------------------------------------------------- + +/* + RepeatTimer repeatedly sends a struct{}{} to `.Chan()` after each `dur` + period. (It's good for keeping connections alive.) + A RepeatTimer must be stopped, or it will keep a goroutine alive. +*/ +type RepeatTimer struct { + name string + ch chan time.Time + tm TickerMaker + + mtx sync.Mutex + dur time.Duration + ticker Ticker + quit chan struct{} +} + +// NewRepeatTimer returns a RepeatTimer with a defaultTicker. +func NewRepeatTimer(name string, dur time.Duration) *RepeatTimer { + return NewRepeatTimerWithTickerMaker(name, dur, defaultTickerMaker) +} + +// NewRepeatTimerWithTicker returns a RepeatTimer with the given ticker +// maker. +func NewRepeatTimerWithTickerMaker(name string, dur time.Duration, tm TickerMaker) *RepeatTimer { + var t = &RepeatTimer{ + name: name, + ch: make(chan time.Time), + tm: tm, + dur: dur, + ticker: nil, + quit: nil, + } + t.reset() + return t +} + +// receive ticks on ch, send out on t.ch +func (t *RepeatTimer) fireRoutine(ch <-chan time.Time, quit <-chan struct{}) { + for { + select { + case tick := <-ch: + select { + case t.ch <- tick: + case <-quit: + return + } + case <-quit: // NOTE: `t.quit` races. + return + } + } +} + +func (t *RepeatTimer) Chan() <-chan time.Time { + return t.ch +} + +func (t *RepeatTimer) Stop() { + t.mtx.Lock() + defer t.mtx.Unlock() + + t.stop() +} + +// Wait the duration again before firing. +func (t *RepeatTimer) Reset() { + t.mtx.Lock() + defer t.mtx.Unlock() + + t.reset() +} + +//---------------------------------------- +// Misc. + +// CONTRACT: (non-constructor) caller should hold t.mtx. +func (t *RepeatTimer) reset() { + if t.ticker != nil { + t.stop() + } + t.ticker = t.tm(t.dur) + t.quit = make(chan struct{}) + go t.fireRoutine(t.ticker.Chan(), t.quit) +} + +// CONTRACT: caller should hold t.mtx. +func (t *RepeatTimer) stop() { + if t.ticker == nil { + /* + Similar to the case of closing channels twice: + https://groups.google.com/forum/#!topic/golang-nuts/rhxMiNmRAPk + Stopping a RepeatTimer twice implies that you do + not know whether you are done or not. + If you're calling stop on a stopped RepeatTimer, + you probably have race conditions. + */ + panic("Tried to stop a stopped RepeatTimer") + } + t.ticker.Stop() + t.ticker = nil + /* + From https://golang.org/pkg/time/#Ticker: + "Stop the ticker to release associated resources" + "After Stop, no more ticks will be sent" + So we shouldn't have to do the below. + + select { + case <-t.ch: + // read off channel if there's anything there + default: + } + */ + close(t.quit) +} diff --git a/libs/common/repeat_timer_test.go b/libs/common/repeat_timer_test.go new file mode 100644 index 000000000..b81720c85 --- /dev/null +++ b/libs/common/repeat_timer_test.go @@ -0,0 +1,137 @@ +package common + +import ( + "math/rand" + "sync" + "testing" + "time" + + "github.com/fortytw2/leaktest" + "github.com/stretchr/testify/assert" +) + +func TestDefaultTicker(t *testing.T) { + ticker := defaultTickerMaker(time.Millisecond * 10) + <-ticker.Chan() + ticker.Stop() +} + +func TestRepeatTimer(t *testing.T) { + + ch := make(chan time.Time, 100) + mtx := new(sync.Mutex) + + // tick() fires from start to end + // (exclusive) in milliseconds with incr. + // It locks on mtx, so subsequent calls + // run in series. + tick := func(startMs, endMs, incrMs time.Duration) { + mtx.Lock() + go func() { + for tMs := startMs; tMs < endMs; tMs += incrMs { + lt := time.Time{} + lt = lt.Add(tMs * time.Millisecond) + ch <- lt + } + mtx.Unlock() + }() + } + + // tock consumes Ticker.Chan() events and checks them against the ms in "timesMs". + tock := func(t *testing.T, rt *RepeatTimer, timesMs []int64) { + + // Check against timesMs. + for _, timeMs := range timesMs { + tyme := <-rt.Chan() + sinceMs := tyme.Sub(time.Time{}) / time.Millisecond + assert.Equal(t, timeMs, int64(sinceMs)) + } + + // TODO detect number of running + // goroutines to ensure that + // no other times will fire. + // See https://github.com/tendermint/tendermint/libs/issues/120. + time.Sleep(time.Millisecond * 100) + done := true + select { + case <-rt.Chan(): + done = false + default: + } + assert.True(t, done) + } + + tm := NewLogicalTickerMaker(ch) + rt := NewRepeatTimerWithTickerMaker("bar", time.Second, tm) + + /* NOTE: Useful for debugging deadlocks... + go func() { + time.Sleep(time.Second * 3) + trace := make([]byte, 102400) + count := runtime.Stack(trace, true) + fmt.Printf("Stack of %d bytes: %s\n", count, trace) + }() + */ + + tick(0, 1000, 10) + tock(t, rt, []int64{}) + tick(1000, 2000, 10) + tock(t, rt, []int64{1000}) + tick(2005, 5000, 10) + tock(t, rt, []int64{2005, 3005, 4005}) + tick(5001, 5999, 1) + // Read 5005 instead of 5001 because + // it's 1 second greater than 4005. + tock(t, rt, []int64{5005}) + tick(6000, 7005, 1) + tock(t, rt, []int64{6005}) + tick(7033, 8032, 1) + tock(t, rt, []int64{7033}) + + // After a reset, nothing happens + // until two ticks are received. + rt.Reset() + tock(t, rt, []int64{}) + tick(8040, 8041, 1) + tock(t, rt, []int64{}) + tick(9555, 9556, 1) + tock(t, rt, []int64{9555}) + + // After a stop, nothing more is sent. + rt.Stop() + tock(t, rt, []int64{}) + + // Another stop panics. + assert.Panics(t, func() { rt.Stop() }) +} + +func TestRepeatTimerReset(t *testing.T) { + // check that we are not leaking any go-routines + defer leaktest.Check(t)() + + timer := NewRepeatTimer("test", 20*time.Millisecond) + defer timer.Stop() + + // test we don't receive tick before duration ms. + select { + case <-timer.Chan(): + t.Fatal("did not expect to receive tick") + default: + } + + timer.Reset() + + // test we receive tick after Reset is called + select { + case <-timer.Chan(): + // all good + case <-time.After(40 * time.Millisecond): + t.Fatal("expected to receive tick after reset") + } + + // just random calls + for i := 0; i < 100; i++ { + time.Sleep(time.Duration(rand.Intn(40)) * time.Millisecond) + timer.Reset() + } +} diff --git a/libs/common/service.go b/libs/common/service.go new file mode 100644 index 000000000..b6f166e77 --- /dev/null +++ b/libs/common/service.go @@ -0,0 +1,205 @@ +package common + +import ( + "errors" + "fmt" + "sync/atomic" + + "github.com/tendermint/tendermint/libs/log" +) + +var ( + ErrAlreadyStarted = errors.New("already started") + ErrAlreadyStopped = errors.New("already stopped") +) + +// Service defines a service that can be started, stopped, and reset. +type Service interface { + // Start the service. + // If it's already started or stopped, will return an error. + // If OnStart() returns an error, it's returned by Start() + Start() error + OnStart() error + + // Stop the service. + // If it's already stopped, will return an error. + // OnStop must never error. + Stop() error + OnStop() + + // Reset the service. + // Panics by default - must be overwritten to enable reset. + Reset() error + OnReset() error + + // Return true if the service is running + IsRunning() bool + + // Quit returns a channel, which is closed once service is stopped. + Quit() <-chan struct{} + + // String representation of the service + String() string + + // SetLogger sets a logger. + SetLogger(log.Logger) +} + +/* +Classical-inheritance-style service declarations. Services can be started, then +stopped, then optionally restarted. + +Users can override the OnStart/OnStop methods. In the absence of errors, these +methods are guaranteed to be called at most once. If OnStart returns an error, +service won't be marked as started, so the user can call Start again. + +A call to Reset will panic, unless OnReset is overwritten, allowing +OnStart/OnStop to be called again. + +The caller must ensure that Start and Stop are not called concurrently. + +It is ok to call Stop without calling Start first. + +Typical usage: + + type FooService struct { + BaseService + // private fields + } + + func NewFooService() *FooService { + fs := &FooService{ + // init + } + fs.BaseService = *NewBaseService(log, "FooService", fs) + return fs + } + + func (fs *FooService) OnStart() error { + fs.BaseService.OnStart() // Always call the overridden method. + // initialize private fields + // start subroutines, etc. + } + + func (fs *FooService) OnStop() error { + fs.BaseService.OnStop() // Always call the overridden method. + // close/destroy private fields + // stop subroutines, etc. + } +*/ +type BaseService struct { + Logger log.Logger + name string + started uint32 // atomic + stopped uint32 // atomic + quit chan struct{} + + // The "subclass" of BaseService + impl Service +} + +// NewBaseService creates a new BaseService. +func NewBaseService(logger log.Logger, name string, impl Service) *BaseService { + if logger == nil { + logger = log.NewNopLogger() + } + + return &BaseService{ + Logger: logger, + name: name, + quit: make(chan struct{}), + impl: impl, + } +} + +// SetLogger implements Service by setting a logger. +func (bs *BaseService) SetLogger(l log.Logger) { + bs.Logger = l +} + +// Start implements Service by calling OnStart (if defined). An error will be +// returned if the service is already running or stopped. Not to start the +// stopped service, you need to call Reset. +func (bs *BaseService) Start() error { + if atomic.CompareAndSwapUint32(&bs.started, 0, 1) { + if atomic.LoadUint32(&bs.stopped) == 1 { + bs.Logger.Error(Fmt("Not starting %v -- already stopped", bs.name), "impl", bs.impl) + return ErrAlreadyStopped + } + bs.Logger.Info(Fmt("Starting %v", bs.name), "impl", bs.impl) + err := bs.impl.OnStart() + if err != nil { + // revert flag + atomic.StoreUint32(&bs.started, 0) + return err + } + return nil + } + bs.Logger.Debug(Fmt("Not starting %v -- already started", bs.name), "impl", bs.impl) + return ErrAlreadyStarted +} + +// OnStart implements Service by doing nothing. +// NOTE: Do not put anything in here, +// that way users don't need to call BaseService.OnStart() +func (bs *BaseService) OnStart() error { return nil } + +// Stop implements Service by calling OnStop (if defined) and closing quit +// channel. An error will be returned if the service is already stopped. +func (bs *BaseService) Stop() error { + if atomic.CompareAndSwapUint32(&bs.stopped, 0, 1) { + bs.Logger.Info(Fmt("Stopping %v", bs.name), "impl", bs.impl) + bs.impl.OnStop() + close(bs.quit) + return nil + } + bs.Logger.Debug(Fmt("Stopping %v (ignoring: already stopped)", bs.name), "impl", bs.impl) + return ErrAlreadyStopped +} + +// OnStop implements Service by doing nothing. +// NOTE: Do not put anything in here, +// that way users don't need to call BaseService.OnStop() +func (bs *BaseService) OnStop() {} + +// Reset implements Service by calling OnReset callback (if defined). An error +// will be returned if the service is running. +func (bs *BaseService) Reset() error { + if !atomic.CompareAndSwapUint32(&bs.stopped, 1, 0) { + bs.Logger.Debug(Fmt("Can't reset %v. Not stopped", bs.name), "impl", bs.impl) + return fmt.Errorf("can't reset running %s", bs.name) + } + + // whether or not we've started, we can reset + atomic.CompareAndSwapUint32(&bs.started, 1, 0) + + bs.quit = make(chan struct{}) + return bs.impl.OnReset() +} + +// OnReset implements Service by panicking. +func (bs *BaseService) OnReset() error { + PanicSanity("The service cannot be reset") + return nil +} + +// IsRunning implements Service by returning true or false depending on the +// service's state. +func (bs *BaseService) IsRunning() bool { + return atomic.LoadUint32(&bs.started) == 1 && atomic.LoadUint32(&bs.stopped) == 0 +} + +// Wait blocks until the service is stopped. +func (bs *BaseService) Wait() { + <-bs.quit +} + +// String implements Servce by returning a string representation of the service. +func (bs *BaseService) String() string { + return bs.name +} + +// Quit Implements Service by returning a quit channel. +func (bs *BaseService) Quit() <-chan struct{} { + return bs.quit +} diff --git a/libs/common/service_test.go b/libs/common/service_test.go new file mode 100644 index 000000000..ef360a648 --- /dev/null +++ b/libs/common/service_test.go @@ -0,0 +1,54 @@ +package common + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +type testService struct { + BaseService +} + +func (testService) OnReset() error { + return nil +} + +func TestBaseServiceWait(t *testing.T) { + ts := &testService{} + ts.BaseService = *NewBaseService(nil, "TestService", ts) + ts.Start() + + waitFinished := make(chan struct{}) + go func() { + ts.Wait() + waitFinished <- struct{}{} + }() + + go ts.Stop() + + select { + case <-waitFinished: + // all good + case <-time.After(100 * time.Millisecond): + t.Fatal("expected Wait() to finish within 100 ms.") + } +} + +func TestBaseServiceReset(t *testing.T) { + ts := &testService{} + ts.BaseService = *NewBaseService(nil, "TestService", ts) + ts.Start() + + err := ts.Reset() + require.Error(t, err, "expected cant reset service error") + + ts.Stop() + + err = ts.Reset() + require.NoError(t, err) + + err = ts.Start() + require.NoError(t, err) +} diff --git a/libs/common/string.go b/libs/common/string.go new file mode 100644 index 000000000..fac1be6c9 --- /dev/null +++ b/libs/common/string.go @@ -0,0 +1,89 @@ +package common + +import ( + "encoding/hex" + "fmt" + "strings" +) + +// Like fmt.Sprintf, but skips formatting if args are empty. +var Fmt = func(format string, a ...interface{}) string { + if len(a) == 0 { + return format + } + return fmt.Sprintf(format, a...) +} + +// IsHex returns true for non-empty hex-string prefixed with "0x" +func IsHex(s string) bool { + if len(s) > 2 && strings.EqualFold(s[:2], "0x") { + _, err := hex.DecodeString(s[2:]) + return err == nil + } + return false +} + +// StripHex returns hex string without leading "0x" +func StripHex(s string) string { + if IsHex(s) { + return s[2:] + } + return s +} + +// StringInSlice returns true if a is found the list. +func StringInSlice(a string, list []string) bool { + for _, b := range list { + if b == a { + return true + } + } + return false +} + +// SplitAndTrim slices s into all subslices separated by sep and returns a +// slice of the string s with all leading and trailing Unicode code points +// contained in cutset removed. If sep is empty, SplitAndTrim splits after each +// UTF-8 sequence. First part is equivalent to strings.SplitN with a count of +// -1. +func SplitAndTrim(s, sep, cutset string) []string { + if s == "" { + return []string{} + } + + spl := strings.Split(s, sep) + for i := 0; i < len(spl); i++ { + spl[i] = strings.Trim(spl[i], cutset) + } + return spl +} + +// Returns true if s is a non-empty printable non-tab ascii character. +func IsASCIIText(s string) bool { + if len(s) == 0 { + return false + } + for _, b := range []byte(s) { + if 32 <= b && b <= 126 { + // good + } else { + return false + } + } + return true +} + +// NOTE: Assumes that s is ASCII as per IsASCIIText(), otherwise panics. +func ASCIITrim(s string) string { + r := make([]byte, 0, len(s)) + for _, b := range []byte(s) { + if b == 32 { + continue // skip space + } else if 32 < b && b <= 126 { + r = append(r, b) + } else { + panic(fmt.Sprintf("non-ASCII (non-tab) char 0x%X", b)) + } + } + return string(r) +} diff --git a/libs/common/string_test.go b/libs/common/string_test.go new file mode 100644 index 000000000..5d1b68feb --- /dev/null +++ b/libs/common/string_test.go @@ -0,0 +1,74 @@ +package common + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestStringInSlice(t *testing.T) { + assert.True(t, StringInSlice("a", []string{"a", "b", "c"})) + assert.False(t, StringInSlice("d", []string{"a", "b", "c"})) + assert.True(t, StringInSlice("", []string{""})) + assert.False(t, StringInSlice("", []string{})) +} + +func TestIsHex(t *testing.T) { + notHex := []string{ + "", " ", "a", "x", "0", "0x", "0X", "0x ", "0X ", "0X a", + "0xf ", "0x f", "0xp", "0x-", + "0xf", "0XBED", "0xF", "0xbed", // Odd lengths + } + for _, v := range notHex { + assert.False(t, IsHex(v), "%q is not hex", v) + } + hex := []string{ + "0x00", "0x0a", "0x0F", "0xFFFFFF", "0Xdeadbeef", "0x0BED", + "0X12", "0X0A", + } + for _, v := range hex { + assert.True(t, IsHex(v), "%q is hex", v) + } +} + +func TestSplitAndTrim(t *testing.T) { + testCases := []struct { + s string + sep string + cutset string + expected []string + }{ + {"a,b,c", ",", " ", []string{"a", "b", "c"}}, + {" a , b , c ", ",", " ", []string{"a", "b", "c"}}, + {" a, b, c ", ",", " ", []string{"a", "b", "c"}}, + {" , ", ",", " ", []string{"", ""}}, + {" ", ",", " ", []string{""}}, + } + + for _, tc := range testCases { + assert.Equal(t, tc.expected, SplitAndTrim(tc.s, tc.sep, tc.cutset), "%s", tc.s) + } +} + +func TestIsASCIIText(t *testing.T) { + notASCIIText := []string{ + "", "\xC2", "\xC2\xA2", "\xFF", "\x80", "\xF0", "\n", "\t", + } + for _, v := range notASCIIText { + assert.False(t, IsHex(v), "%q is not ascii-text", v) + } + asciiText := []string{ + " ", ".", "x", "$", "_", "abcdefg;", "-", "0x00", "0", "123", + } + for _, v := range asciiText { + assert.True(t, IsASCIIText(v), "%q is ascii-text", v) + } +} + +func TestASCIITrim(t *testing.T) { + assert.Equal(t, ASCIITrim(" "), "") + assert.Equal(t, ASCIITrim(" a"), "a") + assert.Equal(t, ASCIITrim("a "), "a") + assert.Equal(t, ASCIITrim(" a "), "a") + assert.Panics(t, func() { ASCIITrim("\xC2\xA2") }) +} diff --git a/libs/common/throttle_timer.go b/libs/common/throttle_timer.go new file mode 100644 index 000000000..38ef4e9a3 --- /dev/null +++ b/libs/common/throttle_timer.go @@ -0,0 +1,75 @@ +package common + +import ( + "sync" + "time" +) + +/* +ThrottleTimer fires an event at most "dur" after each .Set() call. +If a short burst of .Set() calls happens, ThrottleTimer fires once. +If a long continuous burst of .Set() calls happens, ThrottleTimer fires +at most once every "dur". +*/ +type ThrottleTimer struct { + Name string + Ch chan struct{} + quit chan struct{} + dur time.Duration + + mtx sync.Mutex + timer *time.Timer + isSet bool +} + +func NewThrottleTimer(name string, dur time.Duration) *ThrottleTimer { + var ch = make(chan struct{}) + var quit = make(chan struct{}) + var t = &ThrottleTimer{Name: name, Ch: ch, dur: dur, quit: quit} + t.mtx.Lock() + t.timer = time.AfterFunc(dur, t.fireRoutine) + t.mtx.Unlock() + t.timer.Stop() + return t +} + +func (t *ThrottleTimer) fireRoutine() { + t.mtx.Lock() + defer t.mtx.Unlock() + select { + case t.Ch <- struct{}{}: + t.isSet = false + case <-t.quit: + // do nothing + default: + t.timer.Reset(t.dur) + } +} + +func (t *ThrottleTimer) Set() { + t.mtx.Lock() + defer t.mtx.Unlock() + if !t.isSet { + t.isSet = true + t.timer.Reset(t.dur) + } +} + +func (t *ThrottleTimer) Unset() { + t.mtx.Lock() + defer t.mtx.Unlock() + t.isSet = false + t.timer.Stop() +} + +// For ease of .Stop()'ing services before .Start()'ing them, +// we ignore .Stop()'s on nil ThrottleTimers +func (t *ThrottleTimer) Stop() bool { + if t == nil { + return false + } + close(t.quit) + t.mtx.Lock() + defer t.mtx.Unlock() + return t.timer.Stop() +} diff --git a/libs/common/throttle_timer_test.go b/libs/common/throttle_timer_test.go new file mode 100644 index 000000000..00f5abdec --- /dev/null +++ b/libs/common/throttle_timer_test.go @@ -0,0 +1,78 @@ +package common + +import ( + "sync" + "testing" + "time" + + // make govet noshadow happy... + asrt "github.com/stretchr/testify/assert" +) + +type thCounter struct { + input chan struct{} + mtx sync.Mutex + count int +} + +func (c *thCounter) Increment() { + c.mtx.Lock() + c.count++ + c.mtx.Unlock() +} + +func (c *thCounter) Count() int { + c.mtx.Lock() + val := c.count + c.mtx.Unlock() + return val +} + +// Read should run in a go-routine and +// updates count by one every time a packet comes in +func (c *thCounter) Read() { + for range c.input { + c.Increment() + } +} + +func TestThrottle(test *testing.T) { + assert := asrt.New(test) + + ms := 50 + delay := time.Duration(ms) * time.Millisecond + longwait := time.Duration(2) * delay + t := NewThrottleTimer("foo", delay) + + // start at 0 + c := &thCounter{input: t.Ch} + assert.Equal(0, c.Count()) + go c.Read() + + // waiting does nothing + time.Sleep(longwait) + assert.Equal(0, c.Count()) + + // send one event adds one + t.Set() + time.Sleep(longwait) + assert.Equal(1, c.Count()) + + // send a burst adds one + for i := 0; i < 5; i++ { + t.Set() + } + time.Sleep(longwait) + assert.Equal(2, c.Count()) + + // send 12, over 2 delay sections, adds 3 + short := time.Duration(ms/5) * time.Millisecond + for i := 0; i < 13; i++ { + t.Set() + time.Sleep(short) + } + time.Sleep(longwait) + assert.Equal(5, c.Count()) + + close(t.Ch) +} diff --git a/libs/common/types.pb.go b/libs/common/types.pb.go new file mode 100644 index 000000000..f6645602a --- /dev/null +++ b/libs/common/types.pb.go @@ -0,0 +1,98 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: common/types.proto + +/* +Package common is a generated protocol buffer package. + +It is generated from these files: + common/types.proto + +It has these top-level messages: + KVPair + KI64Pair +*/ +//nolint: gas +package common + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +// Define these here for compatibility but use tmlibs/common.KVPair. +type KVPair struct { + Key []byte `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` +} + +func (m *KVPair) Reset() { *m = KVPair{} } +func (m *KVPair) String() string { return proto.CompactTextString(m) } +func (*KVPair) ProtoMessage() {} +func (*KVPair) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } + +func (m *KVPair) GetKey() []byte { + if m != nil { + return m.Key + } + return nil +} + +func (m *KVPair) GetValue() []byte { + if m != nil { + return m.Value + } + return nil +} + +// Define these here for compatibility but use tmlibs/common.KI64Pair. +type KI64Pair struct { + Key []byte `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + Value int64 `protobuf:"varint,2,opt,name=value" json:"value,omitempty"` +} + +func (m *KI64Pair) Reset() { *m = KI64Pair{} } +func (m *KI64Pair) String() string { return proto.CompactTextString(m) } +func (*KI64Pair) ProtoMessage() {} +func (*KI64Pair) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } + +func (m *KI64Pair) GetKey() []byte { + if m != nil { + return m.Key + } + return nil +} + +func (m *KI64Pair) GetValue() int64 { + if m != nil { + return m.Value + } + return 0 +} + +func init() { + proto.RegisterType((*KVPair)(nil), "common.KVPair") + proto.RegisterType((*KI64Pair)(nil), "common.KI64Pair") +} + +func init() { proto.RegisterFile("common/types.proto", fileDescriptor0) } + +var fileDescriptor0 = []byte{ + // 107 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x12, 0x4a, 0xce, 0xcf, 0xcd, + 0xcd, 0xcf, 0xd3, 0x2f, 0xa9, 0x2c, 0x48, 0x2d, 0xd6, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, + 0x83, 0x88, 0x29, 0x19, 0x70, 0xb1, 0x79, 0x87, 0x05, 0x24, 0x66, 0x16, 0x09, 0x09, 0x70, 0x31, + 0x67, 0xa7, 0x56, 0x4a, 0x30, 0x2a, 0x30, 0x6a, 0xf0, 0x04, 0x81, 0x98, 0x42, 0x22, 0x5c, 0xac, + 0x65, 0x89, 0x39, 0xa5, 0xa9, 0x12, 0x4c, 0x60, 0x31, 0x08, 0x47, 0xc9, 0x88, 0x8b, 0xc3, 0xdb, + 0xd3, 0xcc, 0x84, 0x18, 0x3d, 0xcc, 0x50, 0x3d, 0x49, 0x6c, 0x60, 0x4b, 0x8d, 0x01, 0x01, 0x00, + 0x00, 0xff, 0xff, 0xd8, 0xf1, 0xc3, 0x8c, 0x8a, 0x00, 0x00, 0x00, +} diff --git a/libs/common/types.proto b/libs/common/types.proto new file mode 100644 index 000000000..8406fcfdd --- /dev/null +++ b/libs/common/types.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; +package common; + +//---------------------------------------- +// Abstract types + +// Define these here for compatibility but use tmlibs/common.KVPair. +message KVPair { + bytes key = 1; + bytes value = 2; +} + +// Define these here for compatibility but use tmlibs/common.KI64Pair. +message KI64Pair { + bytes key = 1; + int64 value = 2; +} diff --git a/libs/common/word.go b/libs/common/word.go new file mode 100644 index 000000000..a5b841f55 --- /dev/null +++ b/libs/common/word.go @@ -0,0 +1,90 @@ +package common + +import ( + "bytes" + "sort" +) + +var ( + Zero256 = Word256{0} + One256 = Word256{1} +) + +type Word256 [32]byte + +func (w Word256) String() string { return string(w[:]) } +func (w Word256) TrimmedString() string { return TrimmedString(w.Bytes()) } +func (w Word256) Copy() Word256 { return w } +func (w Word256) Bytes() []byte { return w[:] } // copied. +func (w Word256) Prefix(n int) []byte { return w[:n] } +func (w Word256) Postfix(n int) []byte { return w[32-n:] } +func (w Word256) IsZero() bool { + accum := byte(0) + for _, byt := range w { + accum |= byt + } + return accum == 0 +} +func (w Word256) Compare(other Word256) int { + return bytes.Compare(w[:], other[:]) +} + +func Uint64ToWord256(i uint64) Word256 { + buf := [8]byte{} + PutUint64BE(buf[:], i) + return LeftPadWord256(buf[:]) +} + +func Int64ToWord256(i int64) Word256 { + buf := [8]byte{} + PutInt64BE(buf[:], i) + return LeftPadWord256(buf[:]) +} + +func RightPadWord256(bz []byte) (word Word256) { + copy(word[:], bz) + return +} + +func LeftPadWord256(bz []byte) (word Word256) { + copy(word[32-len(bz):], bz) + return +} + +func Uint64FromWord256(word Word256) uint64 { + buf := word.Postfix(8) + return GetUint64BE(buf) +} + +func Int64FromWord256(word Word256) int64 { + buf := word.Postfix(8) + return GetInt64BE(buf) +} + +//------------------------------------- + +type Tuple256 struct { + First Word256 + Second Word256 +} + +func (tuple Tuple256) Compare(other Tuple256) int { + firstCompare := tuple.First.Compare(other.First) + if firstCompare == 0 { + return tuple.Second.Compare(other.Second) + } + return firstCompare +} + +func Tuple256Split(t Tuple256) (Word256, Word256) { + return t.First, t.Second +} + +type Tuple256Slice []Tuple256 + +func (p Tuple256Slice) Len() int { return len(p) } +func (p Tuple256Slice) Less(i, j int) bool { + return p[i].Compare(p[j]) < 0 +} +func (p Tuple256Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p Tuple256Slice) Sort() { sort.Sort(p) } diff --git a/libs/db/LICENSE.md b/libs/db/LICENSE.md new file mode 100644 index 000000000..ab8da59d8 --- /dev/null +++ b/libs/db/LICENSE.md @@ -0,0 +1,3 @@ +Tendermint Go-DB Copyright (C) 2015 All in Bits, Inc + +Released under the Apache2.0 license diff --git a/libs/db/README.md b/libs/db/README.md new file mode 100644 index 000000000..ca5ab33f9 --- /dev/null +++ b/libs/db/README.md @@ -0,0 +1 @@ +TODO: syndtr/goleveldb should be replaced with actual LevelDB instance diff --git a/libs/db/backend_test.go b/libs/db/backend_test.go new file mode 100644 index 000000000..493ed83f9 --- /dev/null +++ b/libs/db/backend_test.go @@ -0,0 +1,215 @@ +package db + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + cmn "github.com/tendermint/tendermint/libs/common" +) + +func cleanupDBDir(dir, name string) { + os.RemoveAll(filepath.Join(dir, name) + ".db") +} + +func testBackendGetSetDelete(t *testing.T, backend DBBackendType) { + // Default + dir, dirname := cmn.Tempdir(fmt.Sprintf("test_backend_%s_", backend)) + defer dir.Close() + db := NewDB("testdb", backend, dirname) + + // A nonexistent key should return nil, even if the key is empty + require.Nil(t, db.Get([]byte(""))) + + // A nonexistent key should return nil, even if the key is nil + require.Nil(t, db.Get(nil)) + + // A nonexistent key should return nil. + key := []byte("abc") + require.Nil(t, db.Get(key)) + + // Set empty value. + db.Set(key, []byte("")) + require.NotNil(t, db.Get(key)) + require.Empty(t, db.Get(key)) + + // Set nil value. + db.Set(key, nil) + require.NotNil(t, db.Get(key)) + require.Empty(t, db.Get(key)) + + // Delete. + db.Delete(key) + require.Nil(t, db.Get(key)) +} + +func TestBackendsGetSetDelete(t *testing.T) { + for dbType := range backends { + testBackendGetSetDelete(t, dbType) + } +} + +func withDB(t *testing.T, creator dbCreator, fn func(DB)) { + name := cmn.Fmt("test_%x", cmn.RandStr(12)) + db, err := creator(name, "") + defer cleanupDBDir("", name) + assert.Nil(t, err) + fn(db) + db.Close() +} + +func TestBackendsNilKeys(t *testing.T) { + + // Test all backends. + for dbType, creator := range backends { + withDB(t, creator, func(db DB) { + t.Run(fmt.Sprintf("Testing %s", dbType), func(t *testing.T) { + + // Nil keys are treated as the empty key for most operations. + expect := func(key, value []byte) { + if len(key) == 0 { // nil or empty + assert.Equal(t, db.Get(nil), db.Get([]byte(""))) + assert.Equal(t, db.Has(nil), db.Has([]byte(""))) + } + assert.Equal(t, db.Get(key), value) + assert.Equal(t, db.Has(key), value != nil) + } + + // Not set + expect(nil, nil) + + // Set nil value + db.Set(nil, nil) + expect(nil, []byte("")) + + // Set empty value + db.Set(nil, []byte("")) + expect(nil, []byte("")) + + // Set nil, Delete nil + db.Set(nil, []byte("abc")) + expect(nil, []byte("abc")) + db.Delete(nil) + expect(nil, nil) + + // Set nil, Delete empty + db.Set(nil, []byte("abc")) + expect(nil, []byte("abc")) + db.Delete([]byte("")) + expect(nil, nil) + + // Set empty, Delete nil + db.Set([]byte(""), []byte("abc")) + expect(nil, []byte("abc")) + db.Delete(nil) + expect(nil, nil) + + // Set empty, Delete empty + db.Set([]byte(""), []byte("abc")) + expect(nil, []byte("abc")) + db.Delete([]byte("")) + expect(nil, nil) + + // SetSync nil, DeleteSync nil + db.SetSync(nil, []byte("abc")) + expect(nil, []byte("abc")) + db.DeleteSync(nil) + expect(nil, nil) + + // SetSync nil, DeleteSync empty + db.SetSync(nil, []byte("abc")) + expect(nil, []byte("abc")) + db.DeleteSync([]byte("")) + expect(nil, nil) + + // SetSync empty, DeleteSync nil + db.SetSync([]byte(""), []byte("abc")) + expect(nil, []byte("abc")) + db.DeleteSync(nil) + expect(nil, nil) + + // SetSync empty, DeleteSync empty + db.SetSync([]byte(""), []byte("abc")) + expect(nil, []byte("abc")) + db.DeleteSync([]byte("")) + expect(nil, nil) + }) + }) + } +} + +func TestGoLevelDBBackend(t *testing.T) { + name := cmn.Fmt("test_%x", cmn.RandStr(12)) + db := NewDB(name, GoLevelDBBackend, "") + defer cleanupDBDir("", name) + + _, ok := db.(*GoLevelDB) + assert.True(t, ok) +} + +func TestDBIterator(t *testing.T) { + for dbType := range backends { + t.Run(fmt.Sprintf("%v", dbType), func(t *testing.T) { + testDBIterator(t, dbType) + }) + } +} + +func testDBIterator(t *testing.T, backend DBBackendType) { + name := cmn.Fmt("test_%x", cmn.RandStr(12)) + db := NewDB(name, backend, "") + defer cleanupDBDir("", name) + + for i := 0; i < 10; i++ { + if i != 6 { // but skip 6. + db.Set(int642Bytes(int64(i)), nil) + } + } + + verifyIterator(t, db.Iterator(nil, nil), []int64{0, 1, 2, 3, 4, 5, 7, 8, 9}, "forward iterator") + verifyIterator(t, db.ReverseIterator(nil, nil), []int64{9, 8, 7, 5, 4, 3, 2, 1, 0}, "reverse iterator") + + verifyIterator(t, db.Iterator(nil, int642Bytes(0)), []int64(nil), "forward iterator to 0") + verifyIterator(t, db.ReverseIterator(nil, int642Bytes(10)), []int64(nil), "reverse iterator 10") + + verifyIterator(t, db.Iterator(int642Bytes(0), nil), []int64{0, 1, 2, 3, 4, 5, 7, 8, 9}, "forward iterator from 0") + verifyIterator(t, db.Iterator(int642Bytes(1), nil), []int64{1, 2, 3, 4, 5, 7, 8, 9}, "forward iterator from 1") + verifyIterator(t, db.ReverseIterator(int642Bytes(10), nil), []int64{9, 8, 7, 5, 4, 3, 2, 1, 0}, "reverse iterator from 10") + verifyIterator(t, db.ReverseIterator(int642Bytes(9), nil), []int64{9, 8, 7, 5, 4, 3, 2, 1, 0}, "reverse iterator from 9") + verifyIterator(t, db.ReverseIterator(int642Bytes(8), nil), []int64{8, 7, 5, 4, 3, 2, 1, 0}, "reverse iterator from 8") + + verifyIterator(t, db.Iterator(int642Bytes(5), int642Bytes(6)), []int64{5}, "forward iterator from 5 to 6") + verifyIterator(t, db.Iterator(int642Bytes(5), int642Bytes(7)), []int64{5}, "forward iterator from 5 to 7") + verifyIterator(t, db.Iterator(int642Bytes(5), int642Bytes(8)), []int64{5, 7}, "forward iterator from 5 to 8") + verifyIterator(t, db.Iterator(int642Bytes(6), int642Bytes(7)), []int64(nil), "forward iterator from 6 to 7") + verifyIterator(t, db.Iterator(int642Bytes(6), int642Bytes(8)), []int64{7}, "forward iterator from 6 to 8") + verifyIterator(t, db.Iterator(int642Bytes(7), int642Bytes(8)), []int64{7}, "forward iterator from 7 to 8") + + verifyIterator(t, db.ReverseIterator(int642Bytes(5), int642Bytes(4)), []int64{5}, "reverse iterator from 5 to 4") + verifyIterator(t, db.ReverseIterator(int642Bytes(6), int642Bytes(4)), []int64{5}, "reverse iterator from 6 to 4") + verifyIterator(t, db.ReverseIterator(int642Bytes(7), int642Bytes(4)), []int64{7, 5}, "reverse iterator from 7 to 4") + verifyIterator(t, db.ReverseIterator(int642Bytes(6), int642Bytes(5)), []int64(nil), "reverse iterator from 6 to 5") + verifyIterator(t, db.ReverseIterator(int642Bytes(7), int642Bytes(5)), []int64{7}, "reverse iterator from 7 to 5") + verifyIterator(t, db.ReverseIterator(int642Bytes(7), int642Bytes(6)), []int64{7}, "reverse iterator from 7 to 6") + + verifyIterator(t, db.Iterator(int642Bytes(0), int642Bytes(1)), []int64{0}, "forward iterator from 0 to 1") + verifyIterator(t, db.ReverseIterator(int642Bytes(9), int642Bytes(8)), []int64{9}, "reverse iterator from 9 to 8") + + verifyIterator(t, db.Iterator(int642Bytes(2), int642Bytes(4)), []int64{2, 3}, "forward iterator from 2 to 4") + verifyIterator(t, db.Iterator(int642Bytes(4), int642Bytes(2)), []int64(nil), "forward iterator from 4 to 2") + verifyIterator(t, db.ReverseIterator(int642Bytes(4), int642Bytes(2)), []int64{4, 3}, "reverse iterator from 4 to 2") + verifyIterator(t, db.ReverseIterator(int642Bytes(2), int642Bytes(4)), []int64(nil), "reverse iterator from 2 to 4") + +} + +func verifyIterator(t *testing.T, itr Iterator, expected []int64, msg string) { + var list []int64 + for itr.Valid() { + list = append(list, bytes2Int64(itr.Key())) + itr.Next() + } + assert.Equal(t, expected, list, msg) +} diff --git a/libs/db/c_level_db.go b/libs/db/c_level_db.go new file mode 100644 index 000000000..307461261 --- /dev/null +++ b/libs/db/c_level_db.go @@ -0,0 +1,312 @@ +// +build gcc + +package db + +import ( + "bytes" + "fmt" + "path/filepath" + + "github.com/jmhodges/levigo" +) + +func init() { + dbCreator := func(name string, dir string) (DB, error) { + return NewCLevelDB(name, dir) + } + registerDBCreator(LevelDBBackend, dbCreator, true) + registerDBCreator(CLevelDBBackend, dbCreator, false) +} + +var _ DB = (*CLevelDB)(nil) + +type CLevelDB struct { + db *levigo.DB + ro *levigo.ReadOptions + wo *levigo.WriteOptions + woSync *levigo.WriteOptions +} + +func NewCLevelDB(name string, dir string) (*CLevelDB, error) { + dbPath := filepath.Join(dir, name+".db") + + opts := levigo.NewOptions() + opts.SetCache(levigo.NewLRUCache(1 << 30)) + opts.SetCreateIfMissing(true) + db, err := levigo.Open(dbPath, opts) + if err != nil { + return nil, err + } + ro := levigo.NewReadOptions() + wo := levigo.NewWriteOptions() + woSync := levigo.NewWriteOptions() + woSync.SetSync(true) + database := &CLevelDB{ + db: db, + ro: ro, + wo: wo, + woSync: woSync, + } + return database, nil +} + +// Implements DB. +func (db *CLevelDB) Get(key []byte) []byte { + key = nonNilBytes(key) + res, err := db.db.Get(db.ro, key) + if err != nil { + panic(err) + } + return res +} + +// Implements DB. +func (db *CLevelDB) Has(key []byte) bool { + return db.Get(key) != nil +} + +// Implements DB. +func (db *CLevelDB) Set(key []byte, value []byte) { + key = nonNilBytes(key) + value = nonNilBytes(value) + err := db.db.Put(db.wo, key, value) + if err != nil { + panic(err) + } +} + +// Implements DB. +func (db *CLevelDB) SetSync(key []byte, value []byte) { + key = nonNilBytes(key) + value = nonNilBytes(value) + err := db.db.Put(db.woSync, key, value) + if err != nil { + panic(err) + } +} + +// Implements DB. +func (db *CLevelDB) Delete(key []byte) { + key = nonNilBytes(key) + err := db.db.Delete(db.wo, key) + if err != nil { + panic(err) + } +} + +// Implements DB. +func (db *CLevelDB) DeleteSync(key []byte) { + key = nonNilBytes(key) + err := db.db.Delete(db.woSync, key) + if err != nil { + panic(err) + } +} + +func (db *CLevelDB) DB() *levigo.DB { + return db.db +} + +// Implements DB. +func (db *CLevelDB) Close() { + db.db.Close() + db.ro.Close() + db.wo.Close() + db.woSync.Close() +} + +// Implements DB. +func (db *CLevelDB) Print() { + itr := db.Iterator(nil, nil) + defer itr.Close() + for ; itr.Valid(); itr.Next() { + key := itr.Key() + value := itr.Value() + fmt.Printf("[%X]:\t[%X]\n", key, value) + } +} + +// Implements DB. +func (db *CLevelDB) Stats() map[string]string { + // TODO: Find the available properties for the C LevelDB implementation + keys := []string{} + + stats := make(map[string]string) + for _, key := range keys { + str := db.db.PropertyValue(key) + stats[key] = str + } + return stats +} + +//---------------------------------------- +// Batch + +// Implements DB. +func (db *CLevelDB) NewBatch() Batch { + batch := levigo.NewWriteBatch() + return &cLevelDBBatch{db, batch} +} + +type cLevelDBBatch struct { + db *CLevelDB + batch *levigo.WriteBatch +} + +// Implements Batch. +func (mBatch *cLevelDBBatch) Set(key, value []byte) { + mBatch.batch.Put(key, value) +} + +// Implements Batch. +func (mBatch *cLevelDBBatch) Delete(key []byte) { + mBatch.batch.Delete(key) +} + +// Implements Batch. +func (mBatch *cLevelDBBatch) Write() { + err := mBatch.db.db.Write(mBatch.db.wo, mBatch.batch) + if err != nil { + panic(err) + } +} + +// Implements Batch. +func (mBatch *cLevelDBBatch) WriteSync() { + err := mBatch.db.db.Write(mBatch.db.woSync, mBatch.batch) + if err != nil { + panic(err) + } +} + +//---------------------------------------- +// Iterator +// NOTE This is almost identical to db/go_level_db.Iterator +// Before creating a third version, refactor. + +func (db *CLevelDB) Iterator(start, end []byte) Iterator { + itr := db.db.NewIterator(db.ro) + return newCLevelDBIterator(itr, start, end, false) +} + +func (db *CLevelDB) ReverseIterator(start, end []byte) Iterator { + itr := db.db.NewIterator(db.ro) + return newCLevelDBIterator(itr, start, end, true) +} + +var _ Iterator = (*cLevelDBIterator)(nil) + +type cLevelDBIterator struct { + source *levigo.Iterator + start, end []byte + isReverse bool + isInvalid bool +} + +func newCLevelDBIterator(source *levigo.Iterator, start, end []byte, isReverse bool) *cLevelDBIterator { + if isReverse { + if start == nil { + source.SeekToLast() + } else { + source.Seek(start) + if source.Valid() { + soakey := source.Key() // start or after key + if bytes.Compare(start, soakey) < 0 { + source.Prev() + } + } else { + source.SeekToLast() + } + } + } else { + if start == nil { + source.SeekToFirst() + } else { + source.Seek(start) + } + } + return &cLevelDBIterator{ + source: source, + start: start, + end: end, + isReverse: isReverse, + isInvalid: false, + } +} + +func (itr cLevelDBIterator) Domain() ([]byte, []byte) { + return itr.start, itr.end +} + +func (itr cLevelDBIterator) Valid() bool { + + // Once invalid, forever invalid. + if itr.isInvalid { + return false + } + + // Panic on DB error. No way to recover. + itr.assertNoError() + + // If source is invalid, invalid. + if !itr.source.Valid() { + itr.isInvalid = true + return false + } + + // If key is end or past it, invalid. + var end = itr.end + var key = itr.source.Key() + if itr.isReverse { + if end != nil && bytes.Compare(key, end) <= 0 { + itr.isInvalid = true + return false + } + } else { + if end != nil && bytes.Compare(end, key) <= 0 { + itr.isInvalid = true + return false + } + } + + // It's valid. + return true +} + +func (itr cLevelDBIterator) Key() []byte { + itr.assertNoError() + itr.assertIsValid() + return itr.source.Key() +} + +func (itr cLevelDBIterator) Value() []byte { + itr.assertNoError() + itr.assertIsValid() + return itr.source.Value() +} + +func (itr cLevelDBIterator) Next() { + itr.assertNoError() + itr.assertIsValid() + if itr.isReverse { + itr.source.Prev() + } else { + itr.source.Next() + } +} + +func (itr cLevelDBIterator) Close() { + itr.source.Close() +} + +func (itr cLevelDBIterator) assertNoError() { + if err := itr.source.GetError(); err != nil { + panic(err) + } +} + +func (itr cLevelDBIterator) assertIsValid() { + if !itr.Valid() { + panic("cLevelDBIterator is invalid") + } +} diff --git a/libs/db/c_level_db_test.go b/libs/db/c_level_db_test.go new file mode 100644 index 000000000..2d30500dd --- /dev/null +++ b/libs/db/c_level_db_test.go @@ -0,0 +1,96 @@ +// +build gcc + +package db + +import ( + "bytes" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + cmn "github.com/tendermint/tendermint/libs/common" +) + +func BenchmarkRandomReadsWrites2(b *testing.B) { + b.StopTimer() + + numItems := int64(1000000) + internal := map[int64]int64{} + for i := 0; i < int(numItems); i++ { + internal[int64(i)] = int64(0) + } + db, err := NewCLevelDB(cmn.Fmt("test_%x", cmn.RandStr(12)), "") + if err != nil { + b.Fatal(err.Error()) + return + } + + fmt.Println("ok, starting") + b.StartTimer() + + for i := 0; i < b.N; i++ { + // Write something + { + idx := (int64(cmn.RandInt()) % numItems) + internal[idx] += 1 + val := internal[idx] + idxBytes := int642Bytes(int64(idx)) + valBytes := int642Bytes(int64(val)) + //fmt.Printf("Set %X -> %X\n", idxBytes, valBytes) + db.Set( + idxBytes, + valBytes, + ) + } + // Read something + { + idx := (int64(cmn.RandInt()) % numItems) + val := internal[idx] + idxBytes := int642Bytes(int64(idx)) + valBytes := db.Get(idxBytes) + //fmt.Printf("Get %X -> %X\n", idxBytes, valBytes) + if val == 0 { + if !bytes.Equal(valBytes, nil) { + b.Errorf("Expected %v for %v, got %X", + nil, idx, valBytes) + break + } + } else { + if len(valBytes) != 8 { + b.Errorf("Expected length 8 for %v, got %X", + idx, valBytes) + break + } + valGot := bytes2Int64(valBytes) + if val != valGot { + b.Errorf("Expected %v for %v, got %v", + val, idx, valGot) + break + } + } + } + } + + db.Close() +} + +/* +func int642Bytes(i int64) []byte { + buf := make([]byte, 8) + binary.BigEndian.PutUint64(buf, uint64(i)) + return buf +} + +func bytes2Int64(buf []byte) int64 { + return int64(binary.BigEndian.Uint64(buf)) +} +*/ + +func TestCLevelDBBackend(t *testing.T) { + name := cmn.Fmt("test_%x", cmn.RandStr(12)) + db := NewDB(name, LevelDBBackend, "") + defer cleanupDBDir("", name) + + _, ok := db.(*CLevelDB) + assert.True(t, ok) +} diff --git a/libs/db/common_test.go b/libs/db/common_test.go new file mode 100644 index 000000000..027b8ee53 --- /dev/null +++ b/libs/db/common_test.go @@ -0,0 +1,191 @@ +package db + +import ( + "fmt" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + cmn "github.com/tendermint/tendermint/libs/common" +) + +//---------------------------------------- +// Helper functions. + +func checkValue(t *testing.T, db DB, key []byte, valueWanted []byte) { + valueGot := db.Get(key) + assert.Equal(t, valueWanted, valueGot) +} + +func checkValid(t *testing.T, itr Iterator, expected bool) { + valid := itr.Valid() + require.Equal(t, expected, valid) +} + +func checkNext(t *testing.T, itr Iterator, expected bool) { + itr.Next() + valid := itr.Valid() + require.Equal(t, expected, valid) +} + +func checkNextPanics(t *testing.T, itr Iterator) { + assert.Panics(t, func() { itr.Next() }, "checkNextPanics expected panic but didn't") +} + +func checkDomain(t *testing.T, itr Iterator, start, end []byte) { + ds, de := itr.Domain() + assert.Equal(t, start, ds, "checkDomain domain start incorrect") + assert.Equal(t, end, de, "checkDomain domain end incorrect") +} + +func checkItem(t *testing.T, itr Iterator, key []byte, value []byte) { + k, v := itr.Key(), itr.Value() + assert.Exactly(t, key, k) + assert.Exactly(t, value, v) +} + +func checkInvalid(t *testing.T, itr Iterator) { + checkValid(t, itr, false) + checkKeyPanics(t, itr) + checkValuePanics(t, itr) + checkNextPanics(t, itr) +} + +func checkKeyPanics(t *testing.T, itr Iterator) { + assert.Panics(t, func() { itr.Key() }, "checkKeyPanics expected panic but didn't") +} + +func checkValuePanics(t *testing.T, itr Iterator) { + assert.Panics(t, func() { itr.Key() }, "checkValuePanics expected panic but didn't") +} + +func newTempDB(t *testing.T, backend DBBackendType) (db DB) { + dir, dirname := cmn.Tempdir("db_common_test") + db = NewDB("testdb", backend, dirname) + dir.Close() + return db +} + +//---------------------------------------- +// mockDB + +// NOTE: not actually goroutine safe. +// If you want something goroutine safe, maybe you just want a MemDB. +type mockDB struct { + mtx sync.Mutex + calls map[string]int +} + +func newMockDB() *mockDB { + return &mockDB{ + calls: make(map[string]int), + } +} + +func (mdb *mockDB) Mutex() *sync.Mutex { + return &(mdb.mtx) +} + +func (mdb *mockDB) Get([]byte) []byte { + mdb.calls["Get"]++ + return nil +} + +func (mdb *mockDB) Has([]byte) bool { + mdb.calls["Has"]++ + return false +} + +func (mdb *mockDB) Set([]byte, []byte) { + mdb.calls["Set"]++ +} + +func (mdb *mockDB) SetSync([]byte, []byte) { + mdb.calls["SetSync"]++ +} + +func (mdb *mockDB) SetNoLock([]byte, []byte) { + mdb.calls["SetNoLock"]++ +} + +func (mdb *mockDB) SetNoLockSync([]byte, []byte) { + mdb.calls["SetNoLockSync"]++ +} + +func (mdb *mockDB) Delete([]byte) { + mdb.calls["Delete"]++ +} + +func (mdb *mockDB) DeleteSync([]byte) { + mdb.calls["DeleteSync"]++ +} + +func (mdb *mockDB) DeleteNoLock([]byte) { + mdb.calls["DeleteNoLock"]++ +} + +func (mdb *mockDB) DeleteNoLockSync([]byte) { + mdb.calls["DeleteNoLockSync"]++ +} + +func (mdb *mockDB) Iterator(start, end []byte) Iterator { + mdb.calls["Iterator"]++ + return &mockIterator{} +} + +func (mdb *mockDB) ReverseIterator(start, end []byte) Iterator { + mdb.calls["ReverseIterator"]++ + return &mockIterator{} +} + +func (mdb *mockDB) Close() { + mdb.calls["Close"]++ +} + +func (mdb *mockDB) NewBatch() Batch { + mdb.calls["NewBatch"]++ + return &memBatch{db: mdb} +} + +func (mdb *mockDB) Print() { + mdb.calls["Print"]++ + fmt.Printf("mockDB{%v}", mdb.Stats()) +} + +func (mdb *mockDB) Stats() map[string]string { + mdb.calls["Stats"]++ + + res := make(map[string]string) + for key, count := range mdb.calls { + res[key] = fmt.Sprintf("%d", count) + } + return res +} + +//---------------------------------------- +// mockIterator + +type mockIterator struct{} + +func (mockIterator) Domain() (start []byte, end []byte) { + return nil, nil +} + +func (mockIterator) Valid() bool { + return false +} + +func (mockIterator) Next() { +} + +func (mockIterator) Key() []byte { + return nil +} + +func (mockIterator) Value() []byte { + return nil +} + +func (mockIterator) Close() { +} diff --git a/libs/db/db.go b/libs/db/db.go new file mode 100644 index 000000000..869937660 --- /dev/null +++ b/libs/db/db.go @@ -0,0 +1,36 @@ +package db + +import "fmt" + +//---------------------------------------- +// Main entry + +type DBBackendType string + +const ( + LevelDBBackend DBBackendType = "leveldb" // legacy, defaults to goleveldb unless +gcc + CLevelDBBackend DBBackendType = "cleveldb" + GoLevelDBBackend DBBackendType = "goleveldb" + MemDBBackend DBBackendType = "memdb" + FSDBBackend DBBackendType = "fsdb" // using the filesystem naively +) + +type dbCreator func(name string, dir string) (DB, error) + +var backends = map[DBBackendType]dbCreator{} + +func registerDBCreator(backend DBBackendType, creator dbCreator, force bool) { + _, ok := backends[backend] + if !force && ok { + return + } + backends[backend] = creator +} + +func NewDB(name string, backend DBBackendType, dir string) DB { + db, err := backends[backend](name, dir) + if err != nil { + panic(fmt.Sprintf("Error initializing DB: %v", err)) + } + return db +} diff --git a/libs/db/db_test.go b/libs/db/db_test.go new file mode 100644 index 000000000..a56901016 --- /dev/null +++ b/libs/db/db_test.go @@ -0,0 +1,194 @@ +package db + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDBIteratorSingleKey(t *testing.T) { + for backend := range backends { + t.Run(fmt.Sprintf("Backend %s", backend), func(t *testing.T) { + db := newTempDB(t, backend) + db.SetSync(bz("1"), bz("value_1")) + itr := db.Iterator(nil, nil) + + checkValid(t, itr, true) + checkNext(t, itr, false) + checkValid(t, itr, false) + checkNextPanics(t, itr) + + // Once invalid... + checkInvalid(t, itr) + }) + } +} + +func TestDBIteratorTwoKeys(t *testing.T) { + for backend := range backends { + t.Run(fmt.Sprintf("Backend %s", backend), func(t *testing.T) { + db := newTempDB(t, backend) + db.SetSync(bz("1"), bz("value_1")) + db.SetSync(bz("2"), bz("value_1")) + + { // Fail by calling Next too much + itr := db.Iterator(nil, nil) + checkValid(t, itr, true) + + checkNext(t, itr, true) + checkValid(t, itr, true) + + checkNext(t, itr, false) + checkValid(t, itr, false) + + checkNextPanics(t, itr) + + // Once invalid... + checkInvalid(t, itr) + } + }) + } +} + +func TestDBIteratorMany(t *testing.T) { + for backend := range backends { + t.Run(fmt.Sprintf("Backend %s", backend), func(t *testing.T) { + db := newTempDB(t, backend) + + keys := make([][]byte, 100) + for i := 0; i < 100; i++ { + keys[i] = []byte{byte(i)} + } + + value := []byte{5} + for _, k := range keys { + db.Set(k, value) + } + + itr := db.Iterator(nil, nil) + defer itr.Close() + for ; itr.Valid(); itr.Next() { + assert.Equal(t, db.Get(itr.Key()), itr.Value()) + } + }) + } +} + +func TestDBIteratorEmpty(t *testing.T) { + for backend := range backends { + t.Run(fmt.Sprintf("Backend %s", backend), func(t *testing.T) { + db := newTempDB(t, backend) + itr := db.Iterator(nil, nil) + + checkInvalid(t, itr) + }) + } +} + +func TestDBIteratorEmptyBeginAfter(t *testing.T) { + for backend := range backends { + t.Run(fmt.Sprintf("Backend %s", backend), func(t *testing.T) { + db := newTempDB(t, backend) + itr := db.Iterator(bz("1"), nil) + + checkInvalid(t, itr) + }) + } +} + +func TestDBIteratorNonemptyBeginAfter(t *testing.T) { + for backend := range backends { + t.Run(fmt.Sprintf("Backend %s", backend), func(t *testing.T) { + db := newTempDB(t, backend) + db.SetSync(bz("1"), bz("value_1")) + itr := db.Iterator(bz("2"), nil) + + checkInvalid(t, itr) + }) + } +} + +func TestDBBatchWrite1(t *testing.T) { + mdb := newMockDB() + ddb := NewDebugDB(t.Name(), mdb) + batch := ddb.NewBatch() + + batch.Set(bz("1"), bz("1")) + batch.Set(bz("2"), bz("2")) + batch.Delete(bz("3")) + batch.Set(bz("4"), bz("4")) + batch.Write() + + assert.Equal(t, 0, mdb.calls["Set"]) + assert.Equal(t, 0, mdb.calls["SetSync"]) + assert.Equal(t, 3, mdb.calls["SetNoLock"]) + assert.Equal(t, 0, mdb.calls["SetNoLockSync"]) + assert.Equal(t, 0, mdb.calls["Delete"]) + assert.Equal(t, 0, mdb.calls["DeleteSync"]) + assert.Equal(t, 1, mdb.calls["DeleteNoLock"]) + assert.Equal(t, 0, mdb.calls["DeleteNoLockSync"]) +} + +func TestDBBatchWrite2(t *testing.T) { + mdb := newMockDB() + ddb := NewDebugDB(t.Name(), mdb) + batch := ddb.NewBatch() + + batch.Set(bz("1"), bz("1")) + batch.Set(bz("2"), bz("2")) + batch.Set(bz("4"), bz("4")) + batch.Delete(bz("3")) + batch.Write() + + assert.Equal(t, 0, mdb.calls["Set"]) + assert.Equal(t, 0, mdb.calls["SetSync"]) + assert.Equal(t, 3, mdb.calls["SetNoLock"]) + assert.Equal(t, 0, mdb.calls["SetNoLockSync"]) + assert.Equal(t, 0, mdb.calls["Delete"]) + assert.Equal(t, 0, mdb.calls["DeleteSync"]) + assert.Equal(t, 1, mdb.calls["DeleteNoLock"]) + assert.Equal(t, 0, mdb.calls["DeleteNoLockSync"]) +} + +func TestDBBatchWriteSync1(t *testing.T) { + mdb := newMockDB() + ddb := NewDebugDB(t.Name(), mdb) + batch := ddb.NewBatch() + + batch.Set(bz("1"), bz("1")) + batch.Set(bz("2"), bz("2")) + batch.Delete(bz("3")) + batch.Set(bz("4"), bz("4")) + batch.WriteSync() + + assert.Equal(t, 0, mdb.calls["Set"]) + assert.Equal(t, 0, mdb.calls["SetSync"]) + assert.Equal(t, 2, mdb.calls["SetNoLock"]) + assert.Equal(t, 1, mdb.calls["SetNoLockSync"]) + assert.Equal(t, 0, mdb.calls["Delete"]) + assert.Equal(t, 0, mdb.calls["DeleteSync"]) + assert.Equal(t, 1, mdb.calls["DeleteNoLock"]) + assert.Equal(t, 0, mdb.calls["DeleteNoLockSync"]) +} + +func TestDBBatchWriteSync2(t *testing.T) { + mdb := newMockDB() + ddb := NewDebugDB(t.Name(), mdb) + batch := ddb.NewBatch() + + batch.Set(bz("1"), bz("1")) + batch.Set(bz("2"), bz("2")) + batch.Set(bz("4"), bz("4")) + batch.Delete(bz("3")) + batch.WriteSync() + + assert.Equal(t, 0, mdb.calls["Set"]) + assert.Equal(t, 0, mdb.calls["SetSync"]) + assert.Equal(t, 3, mdb.calls["SetNoLock"]) + assert.Equal(t, 0, mdb.calls["SetNoLockSync"]) + assert.Equal(t, 0, mdb.calls["Delete"]) + assert.Equal(t, 0, mdb.calls["DeleteSync"]) + assert.Equal(t, 0, mdb.calls["DeleteNoLock"]) + assert.Equal(t, 1, mdb.calls["DeleteNoLockSync"]) +} diff --git a/libs/db/debug_db.go b/libs/db/debug_db.go new file mode 100644 index 000000000..bb361a266 --- /dev/null +++ b/libs/db/debug_db.go @@ -0,0 +1,252 @@ +package db + +import ( + "fmt" + "sync" + + cmn "github.com/tendermint/tendermint/libs/common" +) + +//---------------------------------------- +// debugDB + +type debugDB struct { + label string + db DB +} + +// For printing all operationgs to the console for debugging. +func NewDebugDB(label string, db DB) debugDB { + return debugDB{ + label: label, + db: db, + } +} + +// Implements atomicSetDeleter. +func (ddb debugDB) Mutex() *sync.Mutex { return nil } + +// Implements DB. +func (ddb debugDB) Get(key []byte) (value []byte) { + defer func() { + fmt.Printf("%v.Get(%v) %v\n", ddb.label, + cmn.ColoredBytes(key, cmn.Cyan, cmn.Blue), + cmn.ColoredBytes(value, cmn.Green, cmn.Blue)) + }() + value = ddb.db.Get(key) + return +} + +// Implements DB. +func (ddb debugDB) Has(key []byte) (has bool) { + defer func() { + fmt.Printf("%v.Has(%v) %v\n", ddb.label, + cmn.ColoredBytes(key, cmn.Cyan, cmn.Blue), has) + }() + return ddb.db.Has(key) +} + +// Implements DB. +func (ddb debugDB) Set(key []byte, value []byte) { + fmt.Printf("%v.Set(%v, %v)\n", ddb.label, + cmn.ColoredBytes(key, cmn.Yellow, cmn.Blue), + cmn.ColoredBytes(value, cmn.Green, cmn.Blue)) + ddb.db.Set(key, value) +} + +// Implements DB. +func (ddb debugDB) SetSync(key []byte, value []byte) { + fmt.Printf("%v.SetSync(%v, %v)\n", ddb.label, + cmn.ColoredBytes(key, cmn.Yellow, cmn.Blue), + cmn.ColoredBytes(value, cmn.Green, cmn.Blue)) + ddb.db.SetSync(key, value) +} + +// Implements atomicSetDeleter. +func (ddb debugDB) SetNoLock(key []byte, value []byte) { + fmt.Printf("%v.SetNoLock(%v, %v)\n", ddb.label, + cmn.ColoredBytes(key, cmn.Yellow, cmn.Blue), + cmn.ColoredBytes(value, cmn.Green, cmn.Blue)) + ddb.db.(atomicSetDeleter).SetNoLock(key, value) +} + +// Implements atomicSetDeleter. +func (ddb debugDB) SetNoLockSync(key []byte, value []byte) { + fmt.Printf("%v.SetNoLockSync(%v, %v)\n", ddb.label, + cmn.ColoredBytes(key, cmn.Yellow, cmn.Blue), + cmn.ColoredBytes(value, cmn.Green, cmn.Blue)) + ddb.db.(atomicSetDeleter).SetNoLockSync(key, value) +} + +// Implements DB. +func (ddb debugDB) Delete(key []byte) { + fmt.Printf("%v.Delete(%v)\n", ddb.label, + cmn.ColoredBytes(key, cmn.Red, cmn.Yellow)) + ddb.db.Delete(key) +} + +// Implements DB. +func (ddb debugDB) DeleteSync(key []byte) { + fmt.Printf("%v.DeleteSync(%v)\n", ddb.label, + cmn.ColoredBytes(key, cmn.Red, cmn.Yellow)) + ddb.db.DeleteSync(key) +} + +// Implements atomicSetDeleter. +func (ddb debugDB) DeleteNoLock(key []byte) { + fmt.Printf("%v.DeleteNoLock(%v)\n", ddb.label, + cmn.ColoredBytes(key, cmn.Red, cmn.Yellow)) + ddb.db.(atomicSetDeleter).DeleteNoLock(key) +} + +// Implements atomicSetDeleter. +func (ddb debugDB) DeleteNoLockSync(key []byte) { + fmt.Printf("%v.DeleteNoLockSync(%v)\n", ddb.label, + cmn.ColoredBytes(key, cmn.Red, cmn.Yellow)) + ddb.db.(atomicSetDeleter).DeleteNoLockSync(key) +} + +// Implements DB. +func (ddb debugDB) Iterator(start, end []byte) Iterator { + fmt.Printf("%v.Iterator(%v, %v)\n", ddb.label, + cmn.ColoredBytes(start, cmn.Cyan, cmn.Blue), + cmn.ColoredBytes(end, cmn.Cyan, cmn.Blue)) + return NewDebugIterator(ddb.label, ddb.db.Iterator(start, end)) +} + +// Implements DB. +func (ddb debugDB) ReverseIterator(start, end []byte) Iterator { + fmt.Printf("%v.ReverseIterator(%v, %v)\n", ddb.label, + cmn.ColoredBytes(start, cmn.Cyan, cmn.Blue), + cmn.ColoredBytes(end, cmn.Cyan, cmn.Blue)) + return NewDebugIterator(ddb.label, ddb.db.ReverseIterator(start, end)) +} + +// Implements DB. +// Panics if the underlying db is not an +// atomicSetDeleter. +func (ddb debugDB) NewBatch() Batch { + fmt.Printf("%v.NewBatch()\n", ddb.label) + return NewDebugBatch(ddb.label, ddb.db.NewBatch()) +} + +// Implements DB. +func (ddb debugDB) Close() { + fmt.Printf("%v.Close()\n", ddb.label) + ddb.db.Close() +} + +// Implements DB. +func (ddb debugDB) Print() { + ddb.db.Print() +} + +// Implements DB. +func (ddb debugDB) Stats() map[string]string { + return ddb.db.Stats() +} + +//---------------------------------------- +// debugIterator + +type debugIterator struct { + label string + itr Iterator +} + +// For printing all operationgs to the console for debugging. +func NewDebugIterator(label string, itr Iterator) debugIterator { + return debugIterator{ + label: label, + itr: itr, + } +} + +// Implements Iterator. +func (ditr debugIterator) Domain() (start []byte, end []byte) { + defer func() { + fmt.Printf("%v.itr.Domain() (%X,%X)\n", ditr.label, start, end) + }() + start, end = ditr.itr.Domain() + return +} + +// Implements Iterator. +func (ditr debugIterator) Valid() (ok bool) { + defer func() { + fmt.Printf("%v.itr.Valid() %v\n", ditr.label, ok) + }() + ok = ditr.itr.Valid() + return +} + +// Implements Iterator. +func (ditr debugIterator) Next() { + fmt.Printf("%v.itr.Next()\n", ditr.label) + ditr.itr.Next() +} + +// Implements Iterator. +func (ditr debugIterator) Key() (key []byte) { + key = ditr.itr.Key() + fmt.Printf("%v.itr.Key() %v\n", ditr.label, + cmn.ColoredBytes(key, cmn.Cyan, cmn.Blue)) + return +} + +// Implements Iterator. +func (ditr debugIterator) Value() (value []byte) { + value = ditr.itr.Value() + fmt.Printf("%v.itr.Value() %v\n", ditr.label, + cmn.ColoredBytes(value, cmn.Green, cmn.Blue)) + return +} + +// Implements Iterator. +func (ditr debugIterator) Close() { + fmt.Printf("%v.itr.Close()\n", ditr.label) + ditr.itr.Close() +} + +//---------------------------------------- +// debugBatch + +type debugBatch struct { + label string + bch Batch +} + +// For printing all operationgs to the console for debugging. +func NewDebugBatch(label string, bch Batch) debugBatch { + return debugBatch{ + label: label, + bch: bch, + } +} + +// Implements Batch. +func (dbch debugBatch) Set(key, value []byte) { + fmt.Printf("%v.batch.Set(%v, %v)\n", dbch.label, + cmn.ColoredBytes(key, cmn.Yellow, cmn.Blue), + cmn.ColoredBytes(value, cmn.Green, cmn.Blue)) + dbch.bch.Set(key, value) +} + +// Implements Batch. +func (dbch debugBatch) Delete(key []byte) { + fmt.Printf("%v.batch.Delete(%v)\n", dbch.label, + cmn.ColoredBytes(key, cmn.Red, cmn.Yellow)) + dbch.bch.Delete(key) +} + +// Implements Batch. +func (dbch debugBatch) Write() { + fmt.Printf("%v.batch.Write()\n", dbch.label) + dbch.bch.Write() +} + +// Implements Batch. +func (dbch debugBatch) WriteSync() { + fmt.Printf("%v.batch.WriteSync()\n", dbch.label) + dbch.bch.WriteSync() +} diff --git a/libs/db/fsdb.go b/libs/db/fsdb.go new file mode 100644 index 000000000..fc861decc --- /dev/null +++ b/libs/db/fsdb.go @@ -0,0 +1,262 @@ +package db + +import ( + "fmt" + "io/ioutil" + "net/url" + "os" + "path/filepath" + "sort" + "sync" + + "github.com/pkg/errors" + cmn "github.com/tendermint/tendermint/libs/common" +) + +const ( + keyPerm = os.FileMode(0600) + dirPerm = os.FileMode(0700) +) + +func init() { + registerDBCreator(FSDBBackend, func(name string, dir string) (DB, error) { + dbPath := filepath.Join(dir, name+".db") + return NewFSDB(dbPath), nil + }, false) +} + +var _ DB = (*FSDB)(nil) + +// It's slow. +type FSDB struct { + mtx sync.Mutex + dir string +} + +func NewFSDB(dir string) *FSDB { + err := os.MkdirAll(dir, dirPerm) + if err != nil { + panic(errors.Wrap(err, "Creating FSDB dir "+dir)) + } + database := &FSDB{ + dir: dir, + } + return database +} + +func (db *FSDB) Get(key []byte) []byte { + db.mtx.Lock() + defer db.mtx.Unlock() + key = escapeKey(key) + + path := db.nameToPath(key) + value, err := read(path) + if os.IsNotExist(err) { + return nil + } else if err != nil { + panic(errors.Wrapf(err, "Getting key %s (0x%X)", string(key), key)) + } + return value +} + +func (db *FSDB) Has(key []byte) bool { + db.mtx.Lock() + defer db.mtx.Unlock() + key = escapeKey(key) + + path := db.nameToPath(key) + return cmn.FileExists(path) +} + +func (db *FSDB) Set(key []byte, value []byte) { + db.mtx.Lock() + defer db.mtx.Unlock() + + db.SetNoLock(key, value) +} + +func (db *FSDB) SetSync(key []byte, value []byte) { + db.mtx.Lock() + defer db.mtx.Unlock() + + db.SetNoLock(key, value) +} + +// NOTE: Implements atomicSetDeleter. +func (db *FSDB) SetNoLock(key []byte, value []byte) { + key = escapeKey(key) + value = nonNilBytes(value) + path := db.nameToPath(key) + err := write(path, value) + if err != nil { + panic(errors.Wrapf(err, "Setting key %s (0x%X)", string(key), key)) + } +} + +func (db *FSDB) Delete(key []byte) { + db.mtx.Lock() + defer db.mtx.Unlock() + + db.DeleteNoLock(key) +} + +func (db *FSDB) DeleteSync(key []byte) { + db.mtx.Lock() + defer db.mtx.Unlock() + + db.DeleteNoLock(key) +} + +// NOTE: Implements atomicSetDeleter. +func (db *FSDB) DeleteNoLock(key []byte) { + key = escapeKey(key) + path := db.nameToPath(key) + err := remove(path) + if os.IsNotExist(err) { + return + } else if err != nil { + panic(errors.Wrapf(err, "Removing key %s (0x%X)", string(key), key)) + } +} + +func (db *FSDB) Close() { + // Nothing to do. +} + +func (db *FSDB) Print() { + db.mtx.Lock() + defer db.mtx.Unlock() + + panic("FSDB.Print not yet implemented") +} + +func (db *FSDB) Stats() map[string]string { + db.mtx.Lock() + defer db.mtx.Unlock() + + panic("FSDB.Stats not yet implemented") +} + +func (db *FSDB) NewBatch() Batch { + db.mtx.Lock() + defer db.mtx.Unlock() + + // Not sure we would ever want to try... + // It doesn't seem easy for general filesystems. + panic("FSDB.NewBatch not yet implemented") +} + +func (db *FSDB) Mutex() *sync.Mutex { + return &(db.mtx) +} + +func (db *FSDB) Iterator(start, end []byte) Iterator { + return db.MakeIterator(start, end, false) +} + +func (db *FSDB) MakeIterator(start, end []byte, isReversed bool) Iterator { + db.mtx.Lock() + defer db.mtx.Unlock() + + // We need a copy of all of the keys. + // Not the best, but probably not a bottleneck depending. + keys, err := list(db.dir, start, end, isReversed) + if err != nil { + panic(errors.Wrapf(err, "Listing keys in %s", db.dir)) + } + if isReversed { + sort.Sort(sort.Reverse(sort.StringSlice(keys))) + } else { + sort.Strings(keys) + } + return newMemDBIterator(db, keys, start, end) +} + +func (db *FSDB) ReverseIterator(start, end []byte) Iterator { + return db.MakeIterator(start, end, true) +} + +func (db *FSDB) nameToPath(name []byte) string { + n := url.PathEscape(string(name)) + return filepath.Join(db.dir, n) +} + +// Read some bytes to a file. +// CONTRACT: returns os errors directly without wrapping. +func read(path string) ([]byte, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + d, err := ioutil.ReadAll(f) + if err != nil { + return nil, err + } + return d, nil +} + +// Write some bytes from a file. +// CONTRACT: returns os errors directly without wrapping. +func write(path string, d []byte) error { + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, keyPerm) + if err != nil { + return err + } + defer f.Close() + _, err = f.Write(d) + if err != nil { + return err + } + err = f.Sync() + return err +} + +// Remove a file. +// CONTRACT: returns os errors directly without wrapping. +func remove(path string) error { + return os.Remove(path) +} + +// List keys in a directory, stripping of escape sequences and dir portions. +// CONTRACT: returns os errors directly without wrapping. +func list(dirPath string, start, end []byte, isReversed bool) ([]string, error) { + dir, err := os.Open(dirPath) + if err != nil { + return nil, err + } + defer dir.Close() + + names, err := dir.Readdirnames(0) + if err != nil { + return nil, err + } + var keys []string + for _, name := range names { + n, err := url.PathUnescape(name) + if err != nil { + return nil, fmt.Errorf("Failed to unescape %s while listing", name) + } + key := unescapeKey([]byte(n)) + if IsKeyInDomain(key, start, end, isReversed) { + keys = append(keys, string(key)) + } + } + return keys, nil +} + +// To support empty or nil keys, while the file system doesn't allow empty +// filenames. +func escapeKey(key []byte) []byte { + return []byte("k_" + string(key)) +} +func unescapeKey(escKey []byte) []byte { + if len(escKey) < 2 { + panic(fmt.Sprintf("Invalid esc key: %x", escKey)) + } + if string(escKey[:2]) != "k_" { + panic(fmt.Sprintf("Invalid esc key: %x", escKey)) + } + return escKey[2:] +} diff --git a/libs/db/go_level_db.go b/libs/db/go_level_db.go new file mode 100644 index 000000000..349e447b2 --- /dev/null +++ b/libs/db/go_level_db.go @@ -0,0 +1,327 @@ +package db + +import ( + "bytes" + "fmt" + "path/filepath" + + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/errors" + "github.com/syndtr/goleveldb/leveldb/iterator" + "github.com/syndtr/goleveldb/leveldb/opt" + + cmn "github.com/tendermint/tendermint/libs/common" +) + +func init() { + dbCreator := func(name string, dir string) (DB, error) { + return NewGoLevelDB(name, dir) + } + registerDBCreator(LevelDBBackend, dbCreator, false) + registerDBCreator(GoLevelDBBackend, dbCreator, false) +} + +var _ DB = (*GoLevelDB)(nil) + +type GoLevelDB struct { + db *leveldb.DB +} + +func NewGoLevelDB(name string, dir string) (*GoLevelDB, error) { + dbPath := filepath.Join(dir, name+".db") + db, err := leveldb.OpenFile(dbPath, nil) + if err != nil { + return nil, err + } + database := &GoLevelDB{ + db: db, + } + return database, nil +} + +// Implements DB. +func (db *GoLevelDB) Get(key []byte) []byte { + key = nonNilBytes(key) + res, err := db.db.Get(key, nil) + if err != nil { + if err == errors.ErrNotFound { + return nil + } + panic(err) + } + return res +} + +// Implements DB. +func (db *GoLevelDB) Has(key []byte) bool { + return db.Get(key) != nil +} + +// Implements DB. +func (db *GoLevelDB) Set(key []byte, value []byte) { + key = nonNilBytes(key) + value = nonNilBytes(value) + err := db.db.Put(key, value, nil) + if err != nil { + cmn.PanicCrisis(err) + } +} + +// Implements DB. +func (db *GoLevelDB) SetSync(key []byte, value []byte) { + key = nonNilBytes(key) + value = nonNilBytes(value) + err := db.db.Put(key, value, &opt.WriteOptions{Sync: true}) + if err != nil { + cmn.PanicCrisis(err) + } +} + +// Implements DB. +func (db *GoLevelDB) Delete(key []byte) { + key = nonNilBytes(key) + err := db.db.Delete(key, nil) + if err != nil { + cmn.PanicCrisis(err) + } +} + +// Implements DB. +func (db *GoLevelDB) DeleteSync(key []byte) { + key = nonNilBytes(key) + err := db.db.Delete(key, &opt.WriteOptions{Sync: true}) + if err != nil { + cmn.PanicCrisis(err) + } +} + +func (db *GoLevelDB) DB() *leveldb.DB { + return db.db +} + +// Implements DB. +func (db *GoLevelDB) Close() { + db.db.Close() +} + +// Implements DB. +func (db *GoLevelDB) Print() { + str, _ := db.db.GetProperty("leveldb.stats") + fmt.Printf("%v\n", str) + + itr := db.db.NewIterator(nil, nil) + for itr.Next() { + key := itr.Key() + value := itr.Value() + fmt.Printf("[%X]:\t[%X]\n", key, value) + } +} + +// Implements DB. +func (db *GoLevelDB) Stats() map[string]string { + keys := []string{ + "leveldb.num-files-at-level{n}", + "leveldb.stats", + "leveldb.sstables", + "leveldb.blockpool", + "leveldb.cachedblock", + "leveldb.openedtables", + "leveldb.alivesnaps", + "leveldb.aliveiters", + } + + stats := make(map[string]string) + for _, key := range keys { + str, err := db.db.GetProperty(key) + if err == nil { + stats[key] = str + } + } + return stats +} + +//---------------------------------------- +// Batch + +// Implements DB. +func (db *GoLevelDB) NewBatch() Batch { + batch := new(leveldb.Batch) + return &goLevelDBBatch{db, batch} +} + +type goLevelDBBatch struct { + db *GoLevelDB + batch *leveldb.Batch +} + +// Implements Batch. +func (mBatch *goLevelDBBatch) Set(key, value []byte) { + mBatch.batch.Put(key, value) +} + +// Implements Batch. +func (mBatch *goLevelDBBatch) Delete(key []byte) { + mBatch.batch.Delete(key) +} + +// Implements Batch. +func (mBatch *goLevelDBBatch) Write() { + err := mBatch.db.db.Write(mBatch.batch, &opt.WriteOptions{Sync: false}) + if err != nil { + panic(err) + } +} + +// Implements Batch. +func (mBatch *goLevelDBBatch) WriteSync() { + err := mBatch.db.db.Write(mBatch.batch, &opt.WriteOptions{Sync: true}) + if err != nil { + panic(err) + } +} + +//---------------------------------------- +// Iterator +// NOTE This is almost identical to db/c_level_db.Iterator +// Before creating a third version, refactor. + +// Implements DB. +func (db *GoLevelDB) Iterator(start, end []byte) Iterator { + itr := db.db.NewIterator(nil, nil) + return newGoLevelDBIterator(itr, start, end, false) +} + +// Implements DB. +func (db *GoLevelDB) ReverseIterator(start, end []byte) Iterator { + itr := db.db.NewIterator(nil, nil) + return newGoLevelDBIterator(itr, start, end, true) +} + +type goLevelDBIterator struct { + source iterator.Iterator + start []byte + end []byte + isReverse bool + isInvalid bool +} + +var _ Iterator = (*goLevelDBIterator)(nil) + +func newGoLevelDBIterator(source iterator.Iterator, start, end []byte, isReverse bool) *goLevelDBIterator { + if isReverse { + if start == nil { + source.Last() + } else { + valid := source.Seek(start) + if valid { + soakey := source.Key() // start or after key + if bytes.Compare(start, soakey) < 0 { + source.Prev() + } + } else { + source.Last() + } + } + } else { + if start == nil { + source.First() + } else { + source.Seek(start) + } + } + return &goLevelDBIterator{ + source: source, + start: start, + end: end, + isReverse: isReverse, + isInvalid: false, + } +} + +// Implements Iterator. +func (itr *goLevelDBIterator) Domain() ([]byte, []byte) { + return itr.start, itr.end +} + +// Implements Iterator. +func (itr *goLevelDBIterator) Valid() bool { + + // Once invalid, forever invalid. + if itr.isInvalid { + return false + } + + // Panic on DB error. No way to recover. + itr.assertNoError() + + // If source is invalid, invalid. + if !itr.source.Valid() { + itr.isInvalid = true + return false + } + + // If key is end or past it, invalid. + var end = itr.end + var key = itr.source.Key() + + if itr.isReverse { + if end != nil && bytes.Compare(key, end) <= 0 { + itr.isInvalid = true + return false + } + } else { + if end != nil && bytes.Compare(end, key) <= 0 { + itr.isInvalid = true + return false + } + } + + // Valid + return true +} + +// Implements Iterator. +func (itr *goLevelDBIterator) Key() []byte { + // Key returns a copy of the current key. + // See https://github.com/syndtr/goleveldb/blob/52c212e6c196a1404ea59592d3f1c227c9f034b2/leveldb/iterator/iter.go#L88 + itr.assertNoError() + itr.assertIsValid() + return cp(itr.source.Key()) +} + +// Implements Iterator. +func (itr *goLevelDBIterator) Value() []byte { + // Value returns a copy of the current value. + // See https://github.com/syndtr/goleveldb/blob/52c212e6c196a1404ea59592d3f1c227c9f034b2/leveldb/iterator/iter.go#L88 + itr.assertNoError() + itr.assertIsValid() + return cp(itr.source.Value()) +} + +// Implements Iterator. +func (itr *goLevelDBIterator) Next() { + itr.assertNoError() + itr.assertIsValid() + if itr.isReverse { + itr.source.Prev() + } else { + itr.source.Next() + } +} + +// Implements Iterator. +func (itr *goLevelDBIterator) Close() { + itr.source.Release() +} + +func (itr *goLevelDBIterator) assertNoError() { + if err := itr.source.Error(); err != nil { + panic(err) + } +} + +func (itr goLevelDBIterator) assertIsValid() { + if !itr.Valid() { + panic("goLevelDBIterator is invalid") + } +} diff --git a/libs/db/go_level_db_test.go b/libs/db/go_level_db_test.go new file mode 100644 index 000000000..47be216a6 --- /dev/null +++ b/libs/db/go_level_db_test.go @@ -0,0 +1,83 @@ +package db + +import ( + "bytes" + "encoding/binary" + "fmt" + "testing" + + cmn "github.com/tendermint/tendermint/libs/common" +) + +func BenchmarkRandomReadsWrites(b *testing.B) { + b.StopTimer() + + numItems := int64(1000000) + internal := map[int64]int64{} + for i := 0; i < int(numItems); i++ { + internal[int64(i)] = int64(0) + } + db, err := NewGoLevelDB(cmn.Fmt("test_%x", cmn.RandStr(12)), "") + if err != nil { + b.Fatal(err.Error()) + return + } + + fmt.Println("ok, starting") + b.StartTimer() + + for i := 0; i < b.N; i++ { + // Write something + { + idx := (int64(cmn.RandInt()) % numItems) + internal[idx]++ + val := internal[idx] + idxBytes := int642Bytes(int64(idx)) + valBytes := int642Bytes(int64(val)) + //fmt.Printf("Set %X -> %X\n", idxBytes, valBytes) + db.Set( + idxBytes, + valBytes, + ) + } + // Read something + { + idx := (int64(cmn.RandInt()) % numItems) + val := internal[idx] + idxBytes := int642Bytes(int64(idx)) + valBytes := db.Get(idxBytes) + //fmt.Printf("Get %X -> %X\n", idxBytes, valBytes) + if val == 0 { + if !bytes.Equal(valBytes, nil) { + b.Errorf("Expected %v for %v, got %X", + nil, idx, valBytes) + break + } + } else { + if len(valBytes) != 8 { + b.Errorf("Expected length 8 for %v, got %X", + idx, valBytes) + break + } + valGot := bytes2Int64(valBytes) + if val != valGot { + b.Errorf("Expected %v for %v, got %v", + val, idx, valGot) + break + } + } + } + } + + db.Close() +} + +func int642Bytes(i int64) []byte { + buf := make([]byte, 8) + binary.BigEndian.PutUint64(buf, uint64(i)) + return buf +} + +func bytes2Int64(buf []byte) int64 { + return int64(binary.BigEndian.Uint64(buf)) +} diff --git a/libs/db/mem_batch.go b/libs/db/mem_batch.go new file mode 100644 index 000000000..5c5d0c13a --- /dev/null +++ b/libs/db/mem_batch.go @@ -0,0 +1,72 @@ +package db + +import ( + "sync" +) + +type atomicSetDeleter interface { + Mutex() *sync.Mutex + SetNoLock(key, value []byte) + SetNoLockSync(key, value []byte) + DeleteNoLock(key []byte) + DeleteNoLockSync(key []byte) +} + +type memBatch struct { + db atomicSetDeleter + ops []operation +} + +type opType int + +const ( + opTypeSet opType = 1 + opTypeDelete opType = 2 +) + +type operation struct { + opType + key []byte + value []byte +} + +func (mBatch *memBatch) Set(key, value []byte) { + mBatch.ops = append(mBatch.ops, operation{opTypeSet, key, value}) +} + +func (mBatch *memBatch) Delete(key []byte) { + mBatch.ops = append(mBatch.ops, operation{opTypeDelete, key, nil}) +} + +func (mBatch *memBatch) Write() { + mBatch.write(false) +} + +func (mBatch *memBatch) WriteSync() { + mBatch.write(true) +} + +func (mBatch *memBatch) write(doSync bool) { + if mtx := mBatch.db.Mutex(); mtx != nil { + mtx.Lock() + defer mtx.Unlock() + } + + for i, op := range mBatch.ops { + if doSync && i == (len(mBatch.ops)-1) { + switch op.opType { + case opTypeSet: + mBatch.db.SetNoLockSync(op.key, op.value) + case opTypeDelete: + mBatch.db.DeleteNoLockSync(op.key) + } + break // we're done. + } + switch op.opType { + case opTypeSet: + mBatch.db.SetNoLock(op.key, op.value) + case opTypeDelete: + mBatch.db.DeleteNoLock(op.key) + } + } +} diff --git a/libs/db/mem_db.go b/libs/db/mem_db.go new file mode 100644 index 000000000..580123017 --- /dev/null +++ b/libs/db/mem_db.go @@ -0,0 +1,255 @@ +package db + +import ( + "fmt" + "sort" + "sync" +) + +func init() { + registerDBCreator(MemDBBackend, func(name string, dir string) (DB, error) { + return NewMemDB(), nil + }, false) +} + +var _ DB = (*MemDB)(nil) + +type MemDB struct { + mtx sync.Mutex + db map[string][]byte +} + +func NewMemDB() *MemDB { + database := &MemDB{ + db: make(map[string][]byte), + } + return database +} + +// Implements atomicSetDeleter. +func (db *MemDB) Mutex() *sync.Mutex { + return &(db.mtx) +} + +// Implements DB. +func (db *MemDB) Get(key []byte) []byte { + db.mtx.Lock() + defer db.mtx.Unlock() + key = nonNilBytes(key) + + value := db.db[string(key)] + return value +} + +// Implements DB. +func (db *MemDB) Has(key []byte) bool { + db.mtx.Lock() + defer db.mtx.Unlock() + key = nonNilBytes(key) + + _, ok := db.db[string(key)] + return ok +} + +// Implements DB. +func (db *MemDB) Set(key []byte, value []byte) { + db.mtx.Lock() + defer db.mtx.Unlock() + + db.SetNoLock(key, value) +} + +// Implements DB. +func (db *MemDB) SetSync(key []byte, value []byte) { + db.mtx.Lock() + defer db.mtx.Unlock() + + db.SetNoLock(key, value) +} + +// Implements atomicSetDeleter. +func (db *MemDB) SetNoLock(key []byte, value []byte) { + db.SetNoLockSync(key, value) +} + +// Implements atomicSetDeleter. +func (db *MemDB) SetNoLockSync(key []byte, value []byte) { + key = nonNilBytes(key) + value = nonNilBytes(value) + + db.db[string(key)] = value +} + +// Implements DB. +func (db *MemDB) Delete(key []byte) { + db.mtx.Lock() + defer db.mtx.Unlock() + + db.DeleteNoLock(key) +} + +// Implements DB. +func (db *MemDB) DeleteSync(key []byte) { + db.mtx.Lock() + defer db.mtx.Unlock() + + db.DeleteNoLock(key) +} + +// Implements atomicSetDeleter. +func (db *MemDB) DeleteNoLock(key []byte) { + db.DeleteNoLockSync(key) +} + +// Implements atomicSetDeleter. +func (db *MemDB) DeleteNoLockSync(key []byte) { + key = nonNilBytes(key) + + delete(db.db, string(key)) +} + +// Implements DB. +func (db *MemDB) Close() { + // Close is a noop since for an in-memory + // database, we don't have a destination + // to flush contents to nor do we want + // any data loss on invoking Close() + // See the discussion in https://github.com/tendermint/tendermint/libs/pull/56 +} + +// Implements DB. +func (db *MemDB) Print() { + db.mtx.Lock() + defer db.mtx.Unlock() + + for key, value := range db.db { + fmt.Printf("[%X]:\t[%X]\n", []byte(key), value) + } +} + +// Implements DB. +func (db *MemDB) Stats() map[string]string { + db.mtx.Lock() + defer db.mtx.Unlock() + + stats := make(map[string]string) + stats["database.type"] = "memDB" + stats["database.size"] = fmt.Sprintf("%d", len(db.db)) + return stats +} + +// Implements DB. +func (db *MemDB) NewBatch() Batch { + db.mtx.Lock() + defer db.mtx.Unlock() + + return &memBatch{db, nil} +} + +//---------------------------------------- +// Iterator + +// Implements DB. +func (db *MemDB) Iterator(start, end []byte) Iterator { + db.mtx.Lock() + defer db.mtx.Unlock() + + keys := db.getSortedKeys(start, end, false) + return newMemDBIterator(db, keys, start, end) +} + +// Implements DB. +func (db *MemDB) ReverseIterator(start, end []byte) Iterator { + db.mtx.Lock() + defer db.mtx.Unlock() + + keys := db.getSortedKeys(start, end, true) + return newMemDBIterator(db, keys, start, end) +} + +// We need a copy of all of the keys. +// Not the best, but probably not a bottleneck depending. +type memDBIterator struct { + db DB + cur int + keys []string + start []byte + end []byte +} + +var _ Iterator = (*memDBIterator)(nil) + +// Keys is expected to be in reverse order for reverse iterators. +func newMemDBIterator(db DB, keys []string, start, end []byte) *memDBIterator { + return &memDBIterator{ + db: db, + cur: 0, + keys: keys, + start: start, + end: end, + } +} + +// Implements Iterator. +func (itr *memDBIterator) Domain() ([]byte, []byte) { + return itr.start, itr.end +} + +// Implements Iterator. +func (itr *memDBIterator) Valid() bool { + return 0 <= itr.cur && itr.cur < len(itr.keys) +} + +// Implements Iterator. +func (itr *memDBIterator) Next() { + itr.assertIsValid() + itr.cur++ +} + +// Implements Iterator. +func (itr *memDBIterator) Key() []byte { + itr.assertIsValid() + return []byte(itr.keys[itr.cur]) +} + +// Implements Iterator. +func (itr *memDBIterator) Value() []byte { + itr.assertIsValid() + key := []byte(itr.keys[itr.cur]) + return itr.db.Get(key) +} + +// Implements Iterator. +func (itr *memDBIterator) Close() { + itr.keys = nil + itr.db = nil +} + +func (itr *memDBIterator) assertIsValid() { + if !itr.Valid() { + panic("memDBIterator is invalid") + } +} + +//---------------------------------------- +// Misc. + +func (db *MemDB) getSortedKeys(start, end []byte, reverse bool) []string { + keys := []string{} + for key := range db.db { + inDomain := IsKeyInDomain([]byte(key), start, end, reverse) + if inDomain { + keys = append(keys, key) + } + } + sort.Strings(keys) + if reverse { + nkeys := len(keys) + for i := 0; i < nkeys/2; i++ { + temp := keys[i] + keys[i] = keys[nkeys-i-1] + keys[nkeys-i-1] = temp + } + } + return keys +} diff --git a/libs/db/prefix_db.go b/libs/db/prefix_db.go new file mode 100644 index 000000000..5bb53ebd9 --- /dev/null +++ b/libs/db/prefix_db.go @@ -0,0 +1,355 @@ +package db + +import ( + "bytes" + "fmt" + "sync" +) + +// IteratePrefix is a convenience function for iterating over a key domain +// restricted by prefix. +func IteratePrefix(db DB, prefix []byte) Iterator { + var start, end []byte + if len(prefix) == 0 { + start = nil + end = nil + } else { + start = cp(prefix) + end = cpIncr(prefix) + } + return db.Iterator(start, end) +} + +/* +TODO: Make test, maybe rename. +// Like IteratePrefix but the iterator strips the prefix from the keys. +func IteratePrefixStripped(db DB, prefix []byte) Iterator { + start, end := ... + return newPrefixIterator(prefix, start, end, IteratePrefix(db, prefix)) +} +*/ + +//---------------------------------------- +// prefixDB + +type prefixDB struct { + mtx sync.Mutex + prefix []byte + db DB +} + +// NewPrefixDB lets you namespace multiple DBs within a single DB. +func NewPrefixDB(db DB, prefix []byte) *prefixDB { + return &prefixDB{ + prefix: prefix, + db: db, + } +} + +// Implements atomicSetDeleter. +func (pdb *prefixDB) Mutex() *sync.Mutex { + return &(pdb.mtx) +} + +// Implements DB. +func (pdb *prefixDB) Get(key []byte) []byte { + pdb.mtx.Lock() + defer pdb.mtx.Unlock() + + pkey := pdb.prefixed(key) + value := pdb.db.Get(pkey) + return value +} + +// Implements DB. +func (pdb *prefixDB) Has(key []byte) bool { + pdb.mtx.Lock() + defer pdb.mtx.Unlock() + + return pdb.db.Has(pdb.prefixed(key)) +} + +// Implements DB. +func (pdb *prefixDB) Set(key []byte, value []byte) { + pdb.mtx.Lock() + defer pdb.mtx.Unlock() + + pkey := pdb.prefixed(key) + pdb.db.Set(pkey, value) +} + +// Implements DB. +func (pdb *prefixDB) SetSync(key []byte, value []byte) { + pdb.mtx.Lock() + defer pdb.mtx.Unlock() + + pdb.db.SetSync(pdb.prefixed(key), value) +} + +// Implements DB. +func (pdb *prefixDB) Delete(key []byte) { + pdb.mtx.Lock() + defer pdb.mtx.Unlock() + + pdb.db.Delete(pdb.prefixed(key)) +} + +// Implements DB. +func (pdb *prefixDB) DeleteSync(key []byte) { + pdb.mtx.Lock() + defer pdb.mtx.Unlock() + + pdb.db.DeleteSync(pdb.prefixed(key)) +} + +// Implements DB. +func (pdb *prefixDB) Iterator(start, end []byte) Iterator { + pdb.mtx.Lock() + defer pdb.mtx.Unlock() + + var pstart, pend []byte + pstart = append(cp(pdb.prefix), start...) + if end == nil { + pend = cpIncr(pdb.prefix) + } else { + pend = append(cp(pdb.prefix), end...) + } + return newPrefixIterator( + pdb.prefix, + start, + end, + pdb.db.Iterator( + pstart, + pend, + ), + ) +} + +// Implements DB. +func (pdb *prefixDB) ReverseIterator(start, end []byte) Iterator { + pdb.mtx.Lock() + defer pdb.mtx.Unlock() + + var pstart, pend []byte + if start == nil { + // This may cause the underlying iterator to start with + // an item which doesn't start with prefix. We will skip + // that item later in this function. See 'skipOne'. + pstart = cpIncr(pdb.prefix) + } else { + pstart = append(cp(pdb.prefix), start...) + } + if end == nil { + // This may cause the underlying iterator to end with an + // item which doesn't start with prefix. The + // prefixIterator will terminate iteration + // automatically upon detecting this. + pend = cpDecr(pdb.prefix) + } else { + pend = append(cp(pdb.prefix), end...) + } + ritr := pdb.db.ReverseIterator(pstart, pend) + if start == nil { + skipOne(ritr, cpIncr(pdb.prefix)) + } + return newPrefixIterator( + pdb.prefix, + start, + end, + ritr, + ) +} + +// Implements DB. +// Panics if the underlying DB is not an +// atomicSetDeleter. +func (pdb *prefixDB) NewBatch() Batch { + pdb.mtx.Lock() + defer pdb.mtx.Unlock() + + return newPrefixBatch(pdb.prefix, pdb.db.NewBatch()) +} + +/* NOTE: Uncomment to use memBatch instead of prefixBatch +// Implements atomicSetDeleter. +func (pdb *prefixDB) SetNoLock(key []byte, value []byte) { + pdb.db.(atomicSetDeleter).SetNoLock(pdb.prefixed(key), value) +} + +// Implements atomicSetDeleter. +func (pdb *prefixDB) SetNoLockSync(key []byte, value []byte) { + pdb.db.(atomicSetDeleter).SetNoLockSync(pdb.prefixed(key), value) +} + +// Implements atomicSetDeleter. +func (pdb *prefixDB) DeleteNoLock(key []byte) { + pdb.db.(atomicSetDeleter).DeleteNoLock(pdb.prefixed(key)) +} + +// Implements atomicSetDeleter. +func (pdb *prefixDB) DeleteNoLockSync(key []byte) { + pdb.db.(atomicSetDeleter).DeleteNoLockSync(pdb.prefixed(key)) +} +*/ + +// Implements DB. +func (pdb *prefixDB) Close() { + pdb.mtx.Lock() + defer pdb.mtx.Unlock() + + pdb.db.Close() +} + +// Implements DB. +func (pdb *prefixDB) Print() { + fmt.Printf("prefix: %X\n", pdb.prefix) + + itr := pdb.Iterator(nil, nil) + defer itr.Close() + for ; itr.Valid(); itr.Next() { + key := itr.Key() + value := itr.Value() + fmt.Printf("[%X]:\t[%X]\n", key, value) + } +} + +// Implements DB. +func (pdb *prefixDB) Stats() map[string]string { + stats := make(map[string]string) + stats["prefixdb.prefix.string"] = string(pdb.prefix) + stats["prefixdb.prefix.hex"] = fmt.Sprintf("%X", pdb.prefix) + source := pdb.db.Stats() + for key, value := range source { + stats["prefixdb.source."+key] = value + } + return stats +} + +func (pdb *prefixDB) prefixed(key []byte) []byte { + return append(cp(pdb.prefix), key...) +} + +//---------------------------------------- +// prefixBatch + +type prefixBatch struct { + prefix []byte + source Batch +} + +func newPrefixBatch(prefix []byte, source Batch) prefixBatch { + return prefixBatch{ + prefix: prefix, + source: source, + } +} + +func (pb prefixBatch) Set(key, value []byte) { + pkey := append(cp(pb.prefix), key...) + pb.source.Set(pkey, value) +} + +func (pb prefixBatch) Delete(key []byte) { + pkey := append(cp(pb.prefix), key...) + pb.source.Delete(pkey) +} + +func (pb prefixBatch) Write() { + pb.source.Write() +} + +func (pb prefixBatch) WriteSync() { + pb.source.WriteSync() +} + +//---------------------------------------- +// prefixIterator + +// Strips prefix while iterating from Iterator. +type prefixIterator struct { + prefix []byte + start []byte + end []byte + source Iterator + valid bool +} + +func newPrefixIterator(prefix, start, end []byte, source Iterator) prefixIterator { + if !source.Valid() || !bytes.HasPrefix(source.Key(), prefix) { + return prefixIterator{ + prefix: prefix, + start: start, + end: end, + source: source, + valid: false, + } + } else { + return prefixIterator{ + prefix: prefix, + start: start, + end: end, + source: source, + valid: true, + } + } +} + +func (itr prefixIterator) Domain() (start []byte, end []byte) { + return itr.start, itr.end +} + +func (itr prefixIterator) Valid() bool { + return itr.valid && itr.source.Valid() +} + +func (itr prefixIterator) Next() { + if !itr.valid { + panic("prefixIterator invalid, cannot call Next()") + } + itr.source.Next() + if !itr.source.Valid() || !bytes.HasPrefix(itr.source.Key(), itr.prefix) { + itr.source.Close() + itr.valid = false + return + } +} + +func (itr prefixIterator) Key() (key []byte) { + if !itr.valid { + panic("prefixIterator invalid, cannot call Key()") + } + return stripPrefix(itr.source.Key(), itr.prefix) +} + +func (itr prefixIterator) Value() (value []byte) { + if !itr.valid { + panic("prefixIterator invalid, cannot call Value()") + } + return itr.source.Value() +} + +func (itr prefixIterator) Close() { + itr.source.Close() +} + +//---------------------------------------- + +func stripPrefix(key []byte, prefix []byte) (stripped []byte) { + if len(key) < len(prefix) { + panic("should not happen") + } + if !bytes.Equal(key[:len(prefix)], prefix) { + panic("should not happne") + } + return key[len(prefix):] +} + +// If the first iterator item is skipKey, then +// skip it. +func skipOne(itr Iterator, skipKey []byte) { + if itr.Valid() { + if bytes.Equal(itr.Key(), skipKey) { + itr.Next() + } + } +} diff --git a/libs/db/prefix_db_test.go b/libs/db/prefix_db_test.go new file mode 100644 index 000000000..60809f157 --- /dev/null +++ b/libs/db/prefix_db_test.go @@ -0,0 +1,147 @@ +package db + +import "testing" + +func mockDBWithStuff() DB { + db := NewMemDB() + // Under "key" prefix + db.Set(bz("key"), bz("value")) + db.Set(bz("key1"), bz("value1")) + db.Set(bz("key2"), bz("value2")) + db.Set(bz("key3"), bz("value3")) + db.Set(bz("something"), bz("else")) + db.Set(bz(""), bz("")) + db.Set(bz("k"), bz("val")) + db.Set(bz("ke"), bz("valu")) + db.Set(bz("kee"), bz("valuu")) + return db +} + +func TestPrefixDBSimple(t *testing.T) { + db := mockDBWithStuff() + pdb := NewPrefixDB(db, bz("key")) + + checkValue(t, pdb, bz("key"), nil) + checkValue(t, pdb, bz(""), bz("value")) + checkValue(t, pdb, bz("key1"), nil) + checkValue(t, pdb, bz("1"), bz("value1")) + checkValue(t, pdb, bz("key2"), nil) + checkValue(t, pdb, bz("2"), bz("value2")) + checkValue(t, pdb, bz("key3"), nil) + checkValue(t, pdb, bz("3"), bz("value3")) + checkValue(t, pdb, bz("something"), nil) + checkValue(t, pdb, bz("k"), nil) + checkValue(t, pdb, bz("ke"), nil) + checkValue(t, pdb, bz("kee"), nil) +} + +func TestPrefixDBIterator1(t *testing.T) { + db := mockDBWithStuff() + pdb := NewPrefixDB(db, bz("key")) + + itr := pdb.Iterator(nil, nil) + checkDomain(t, itr, nil, nil) + checkItem(t, itr, bz(""), bz("value")) + checkNext(t, itr, true) + checkItem(t, itr, bz("1"), bz("value1")) + checkNext(t, itr, true) + checkItem(t, itr, bz("2"), bz("value2")) + checkNext(t, itr, true) + checkItem(t, itr, bz("3"), bz("value3")) + checkNext(t, itr, false) + checkInvalid(t, itr) + itr.Close() +} + +func TestPrefixDBIterator2(t *testing.T) { + db := mockDBWithStuff() + pdb := NewPrefixDB(db, bz("key")) + + itr := pdb.Iterator(nil, bz("")) + checkDomain(t, itr, nil, bz("")) + checkInvalid(t, itr) + itr.Close() +} + +func TestPrefixDBIterator3(t *testing.T) { + db := mockDBWithStuff() + pdb := NewPrefixDB(db, bz("key")) + + itr := pdb.Iterator(bz(""), nil) + checkDomain(t, itr, bz(""), nil) + checkItem(t, itr, bz(""), bz("value")) + checkNext(t, itr, true) + checkItem(t, itr, bz("1"), bz("value1")) + checkNext(t, itr, true) + checkItem(t, itr, bz("2"), bz("value2")) + checkNext(t, itr, true) + checkItem(t, itr, bz("3"), bz("value3")) + checkNext(t, itr, false) + checkInvalid(t, itr) + itr.Close() +} + +func TestPrefixDBIterator4(t *testing.T) { + db := mockDBWithStuff() + pdb := NewPrefixDB(db, bz("key")) + + itr := pdb.Iterator(bz(""), bz("")) + checkDomain(t, itr, bz(""), bz("")) + checkInvalid(t, itr) + itr.Close() +} + +func TestPrefixDBReverseIterator1(t *testing.T) { + db := mockDBWithStuff() + pdb := NewPrefixDB(db, bz("key")) + + itr := pdb.ReverseIterator(nil, nil) + checkDomain(t, itr, nil, nil) + checkItem(t, itr, bz("3"), bz("value3")) + checkNext(t, itr, true) + checkItem(t, itr, bz("2"), bz("value2")) + checkNext(t, itr, true) + checkItem(t, itr, bz("1"), bz("value1")) + checkNext(t, itr, true) + checkItem(t, itr, bz(""), bz("value")) + checkNext(t, itr, false) + checkInvalid(t, itr) + itr.Close() +} + +func TestPrefixDBReverseIterator2(t *testing.T) { + db := mockDBWithStuff() + pdb := NewPrefixDB(db, bz("key")) + + itr := pdb.ReverseIterator(nil, bz("")) + checkDomain(t, itr, nil, bz("")) + checkItem(t, itr, bz("3"), bz("value3")) + checkNext(t, itr, true) + checkItem(t, itr, bz("2"), bz("value2")) + checkNext(t, itr, true) + checkItem(t, itr, bz("1"), bz("value1")) + checkNext(t, itr, false) + checkInvalid(t, itr) + itr.Close() +} + +func TestPrefixDBReverseIterator3(t *testing.T) { + db := mockDBWithStuff() + pdb := NewPrefixDB(db, bz("key")) + + itr := pdb.ReverseIterator(bz(""), nil) + checkDomain(t, itr, bz(""), nil) + checkItem(t, itr, bz(""), bz("value")) + checkNext(t, itr, false) + checkInvalid(t, itr) + itr.Close() +} + +func TestPrefixDBReverseIterator4(t *testing.T) { + db := mockDBWithStuff() + pdb := NewPrefixDB(db, bz("key")) + + itr := pdb.ReverseIterator(bz(""), bz("")) + checkInvalid(t, itr) + itr.Close() +} diff --git a/libs/db/remotedb/doc.go b/libs/db/remotedb/doc.go new file mode 100644 index 000000000..07c95a56a --- /dev/null +++ b/libs/db/remotedb/doc.go @@ -0,0 +1,37 @@ +/* +remotedb is a package for connecting to distributed Tendermint db.DB +instances. The purpose is to detach difficult deployments such as +CLevelDB that requires gcc or perhaps for databases that require +custom configurations such as extra disk space. It also eases +the burden and cost of deployment of dependencies for databases +to be used by Tendermint developers. Most importantly it is built +over the high performant gRPC transport. + +remotedb's RemoteDB implements db.DB so can be used normally +like other databases. One just has to explicitly connect to the +remote database with a client setup such as: + + client, err := remotedb.NewInsecure(addr) + // Make sure to invoke InitRemote! + if err := client.InitRemote(&remotedb.Init{Name: "test-remote-db", Type: "leveldb"}); err != nil { + log.Fatalf("Failed to initialize the remote db") + } + + client.Set(key1, value) + gv1 := client.SetSync(k2, v2) + + client.Delete(k1) + gv2 := client.Get(k1) + + for itr := client.Iterator(k1, k9); itr.Valid(); itr.Next() { + ik, iv := itr.Key(), itr.Value() + ds, de := itr.Domain() + } + + stats := client.Stats() + + if !client.Has(dk1) { + client.SetSync(dk1, dv1) + } +*/ +package remotedb diff --git a/libs/db/remotedb/grpcdb/client.go b/libs/db/remotedb/grpcdb/client.go new file mode 100644 index 000000000..e11b7839b --- /dev/null +++ b/libs/db/remotedb/grpcdb/client.go @@ -0,0 +1,30 @@ +package grpcdb + +import ( + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + + protodb "github.com/tendermint/tendermint/libs/db/remotedb/proto" +) + +// Security defines how the client will talk to the gRPC server. +type Security uint + +const ( + Insecure Security = iota + Secure +) + +// NewClient creates a gRPC client connected to the bound gRPC server at serverAddr. +// Use kind to set the level of security to either Secure or Insecure. +func NewClient(serverAddr, serverCert string) (protodb.DBClient, error) { + creds, err := credentials.NewClientTLSFromFile(serverCert, "") + if err != nil { + return nil, err + } + cc, err := grpc.Dial(serverAddr, grpc.WithTransportCredentials(creds)) + if err != nil { + return nil, err + } + return protodb.NewDBClient(cc), nil +} diff --git a/libs/db/remotedb/grpcdb/doc.go b/libs/db/remotedb/grpcdb/doc.go new file mode 100644 index 000000000..0d8e380ce --- /dev/null +++ b/libs/db/remotedb/grpcdb/doc.go @@ -0,0 +1,32 @@ +/* +grpcdb is the distribution of Tendermint's db.DB instances using +the gRPC transport to decouple local db.DB usages from applications, +to using them over a network in a highly performant manner. + +grpcdb allows users to initialize a database's server like +they would locally and invoke the respective methods of db.DB. + +Most users shouldn't use this package, but should instead use +remotedb. Only the lower level users and database server deployers +should use it, for functionality such as: + + ln, err := net.Listen("tcp", "0.0.0.0:0") + srv := grpcdb.NewServer() + defer srv.Stop() + go func() { + if err := srv.Serve(ln); err != nil { + t.Fatalf("BindServer: %v", err) + } + }() + +or + addr := ":8998" + cert := "server.crt" + key := "server.key" + go func() { + if err := grpcdb.ListenAndServe(addr, cert, key); err != nil { + log.Fatalf("BindServer: %v", err) + } + }() +*/ +package grpcdb diff --git a/libs/db/remotedb/grpcdb/example_test.go b/libs/db/remotedb/grpcdb/example_test.go new file mode 100644 index 000000000..eba0d6914 --- /dev/null +++ b/libs/db/remotedb/grpcdb/example_test.go @@ -0,0 +1,52 @@ +package grpcdb_test + +import ( + "bytes" + "context" + "log" + + grpcdb "github.com/tendermint/tendermint/libs/db/remotedb/grpcdb" + protodb "github.com/tendermint/tendermint/libs/db/remotedb/proto" +) + +func Example() { + addr := ":8998" + cert := "server.crt" + key := "server.key" + go func() { + if err := grpcdb.ListenAndServe(addr, cert, key); err != nil { + log.Fatalf("BindServer: %v", err) + } + }() + + client, err := grpcdb.NewClient(addr, cert) + if err != nil { + log.Fatalf("Failed to create grpcDB client: %v", err) + } + + ctx := context.Background() + // 1. Initialize the DB + in := &protodb.Init{ + Type: "leveldb", + Name: "grpc-uno-test", + Dir: ".", + } + if _, err := client.Init(ctx, in); err != nil { + log.Fatalf("Init error: %v", err) + } + + // 2. Now it can be used! + query1 := &protodb.Entity{Key: []byte("Project"), Value: []byte("Tmlibs-on-gRPC")} + if _, err := client.SetSync(ctx, query1); err != nil { + log.Fatalf("SetSync err: %v", err) + } + + query2 := &protodb.Entity{Key: []byte("Project")} + read, err := client.Get(ctx, query2) + if err != nil { + log.Fatalf("Get err: %v", err) + } + if g, w := read.Value, []byte("Tmlibs-on-gRPC"); !bytes.Equal(g, w) { + log.Fatalf("got= (%q ==> % X)\nwant=(%q ==> % X)", g, g, w, w) + } +} diff --git a/libs/db/remotedb/grpcdb/server.go b/libs/db/remotedb/grpcdb/server.go new file mode 100644 index 000000000..3a9955ddf --- /dev/null +++ b/libs/db/remotedb/grpcdb/server.go @@ -0,0 +1,197 @@ +package grpcdb + +import ( + "context" + "net" + "sync" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + + "github.com/tendermint/tendermint/libs/db" + protodb "github.com/tendermint/tendermint/libs/db/remotedb/proto" +) + +// ListenAndServe is a blocking function that sets up a gRPC based +// server at the address supplied, with the gRPC options passed in. +// Normally in usage, invoke it in a goroutine like you would for http.ListenAndServe. +func ListenAndServe(addr, cert, key string, opts ...grpc.ServerOption) error { + ln, err := net.Listen("tcp", addr) + if err != nil { + return err + } + srv, err := NewServer(cert, key, opts...) + if err != nil { + return err + } + return srv.Serve(ln) +} + +func NewServer(cert, key string, opts ...grpc.ServerOption) (*grpc.Server, error) { + creds, err := credentials.NewServerTLSFromFile(cert, key) + if err != nil { + return nil, err + } + opts = append(opts, grpc.Creds(creds)) + srv := grpc.NewServer(opts...) + protodb.RegisterDBServer(srv, new(server)) + return srv, nil +} + +type server struct { + mu sync.Mutex + db db.DB +} + +var _ protodb.DBServer = (*server)(nil) + +// Init initializes the server's database. Only one type of database +// can be initialized per server. +// +// Dir is the directory on the file system in which the DB will be stored(if backed by disk) (TODO: remove) +// +// Name is representative filesystem entry's basepath +// +// Type can be either one of: +// * cleveldb (if built with gcc enabled) +// * fsdb +// * memdB +// * leveldb +// See https://godoc.org/github.com/tendermint/tendermint/libs/db#DBBackendType +func (s *server) Init(ctx context.Context, in *protodb.Init) (*protodb.Entity, error) { + s.mu.Lock() + defer s.mu.Unlock() + + s.db = db.NewDB(in.Name, db.DBBackendType(in.Type), in.Dir) + return &protodb.Entity{CreatedAt: time.Now().Unix()}, nil +} + +func (s *server) Delete(ctx context.Context, in *protodb.Entity) (*protodb.Nothing, error) { + s.db.Delete(in.Key) + return nothing, nil +} + +var nothing = new(protodb.Nothing) + +func (s *server) DeleteSync(ctx context.Context, in *protodb.Entity) (*protodb.Nothing, error) { + s.db.DeleteSync(in.Key) + return nothing, nil +} + +func (s *server) Get(ctx context.Context, in *protodb.Entity) (*protodb.Entity, error) { + value := s.db.Get(in.Key) + return &protodb.Entity{Value: value}, nil +} + +func (s *server) GetStream(ds protodb.DB_GetStreamServer) error { + // Receive routine + responsesChan := make(chan *protodb.Entity) + go func() { + defer close(responsesChan) + ctx := context.Background() + for { + in, err := ds.Recv() + if err != nil { + responsesChan <- &protodb.Entity{Err: err.Error()} + return + } + out, err := s.Get(ctx, in) + if err != nil { + if out == nil { + out = new(protodb.Entity) + out.Key = in.Key + } + out.Err = err.Error() + responsesChan <- out + return + } + + // Otherwise continue on + responsesChan <- out + } + }() + + // Send routine, block until we return + for out := range responsesChan { + if err := ds.Send(out); err != nil { + return err + } + } + return nil +} + +func (s *server) Has(ctx context.Context, in *protodb.Entity) (*protodb.Entity, error) { + exists := s.db.Has(in.Key) + return &protodb.Entity{Exists: exists}, nil +} + +func (s *server) Set(ctx context.Context, in *protodb.Entity) (*protodb.Nothing, error) { + s.db.Set(in.Key, in.Value) + return nothing, nil +} + +func (s *server) SetSync(ctx context.Context, in *protodb.Entity) (*protodb.Nothing, error) { + s.db.SetSync(in.Key, in.Value) + return nothing, nil +} + +func (s *server) Iterator(query *protodb.Entity, dis protodb.DB_IteratorServer) error { + it := s.db.Iterator(query.Start, query.End) + return s.handleIterator(it, dis.Send) +} + +func (s *server) handleIterator(it db.Iterator, sendFunc func(*protodb.Iterator) error) error { + for it.Valid() { + start, end := it.Domain() + out := &protodb.Iterator{ + Domain: &protodb.Domain{Start: start, End: end}, + Valid: it.Valid(), + Key: it.Key(), + Value: it.Value(), + } + if err := sendFunc(out); err != nil { + return err + } + + // Finally move the iterator forward + it.Next() + } + return nil +} + +func (s *server) ReverseIterator(query *protodb.Entity, dis protodb.DB_ReverseIteratorServer) error { + it := s.db.ReverseIterator(query.Start, query.End) + return s.handleIterator(it, dis.Send) +} + +func (s *server) Stats(context.Context, *protodb.Nothing) (*protodb.Stats, error) { + stats := s.db.Stats() + return &protodb.Stats{Data: stats, TimeAt: time.Now().Unix()}, nil +} + +func (s *server) BatchWrite(c context.Context, b *protodb.Batch) (*protodb.Nothing, error) { + return s.batchWrite(c, b, false) +} + +func (s *server) BatchWriteSync(c context.Context, b *protodb.Batch) (*protodb.Nothing, error) { + return s.batchWrite(c, b, true) +} + +func (s *server) batchWrite(c context.Context, b *protodb.Batch, sync bool) (*protodb.Nothing, error) { + bat := s.db.NewBatch() + for _, op := range b.Ops { + switch op.Type { + case protodb.Operation_SET: + bat.Set(op.Entity.Key, op.Entity.Value) + case protodb.Operation_DELETE: + bat.Delete(op.Entity.Key) + } + } + if sync { + bat.WriteSync() + } else { + bat.Write() + } + return nothing, nil +} diff --git a/libs/db/remotedb/proto/defs.pb.go b/libs/db/remotedb/proto/defs.pb.go new file mode 100644 index 000000000..4d9f0b272 --- /dev/null +++ b/libs/db/remotedb/proto/defs.pb.go @@ -0,0 +1,914 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: defs.proto + +/* +Package protodb is a generated protocol buffer package. + +It is generated from these files: + defs.proto + +It has these top-level messages: + Batch + Operation + Entity + Nothing + Domain + Iterator + Stats + Init +*/ +package protodb + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +import ( + context "golang.org/x/net/context" + grpc "google.golang.org/grpc" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +type Operation_Type int32 + +const ( + Operation_SET Operation_Type = 0 + Operation_DELETE Operation_Type = 1 +) + +var Operation_Type_name = map[int32]string{ + 0: "SET", + 1: "DELETE", +} +var Operation_Type_value = map[string]int32{ + "SET": 0, + "DELETE": 1, +} + +func (x Operation_Type) String() string { + return proto.EnumName(Operation_Type_name, int32(x)) +} +func (Operation_Type) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{1, 0} } + +type Batch struct { + Ops []*Operation `protobuf:"bytes,1,rep,name=ops" json:"ops,omitempty"` +} + +func (m *Batch) Reset() { *m = Batch{} } +func (m *Batch) String() string { return proto.CompactTextString(m) } +func (*Batch) ProtoMessage() {} +func (*Batch) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } + +func (m *Batch) GetOps() []*Operation { + if m != nil { + return m.Ops + } + return nil +} + +type Operation struct { + Entity *Entity `protobuf:"bytes,1,opt,name=entity" json:"entity,omitempty"` + Type Operation_Type `protobuf:"varint,2,opt,name=type,enum=protodb.Operation_Type" json:"type,omitempty"` +} + +func (m *Operation) Reset() { *m = Operation{} } +func (m *Operation) String() string { return proto.CompactTextString(m) } +func (*Operation) ProtoMessage() {} +func (*Operation) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } + +func (m *Operation) GetEntity() *Entity { + if m != nil { + return m.Entity + } + return nil +} + +func (m *Operation) GetType() Operation_Type { + if m != nil { + return m.Type + } + return Operation_SET +} + +type Entity struct { + Id int32 `protobuf:"varint,1,opt,name=id" json:"id,omitempty"` + Key []byte `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` + Value []byte `protobuf:"bytes,3,opt,name=value,proto3" json:"value,omitempty"` + Exists bool `protobuf:"varint,4,opt,name=exists" json:"exists,omitempty"` + Start []byte `protobuf:"bytes,5,opt,name=start,proto3" json:"start,omitempty"` + End []byte `protobuf:"bytes,6,opt,name=end,proto3" json:"end,omitempty"` + Err string `protobuf:"bytes,7,opt,name=err" json:"err,omitempty"` + CreatedAt int64 `protobuf:"varint,8,opt,name=created_at,json=createdAt" json:"created_at,omitempty"` +} + +func (m *Entity) Reset() { *m = Entity{} } +func (m *Entity) String() string { return proto.CompactTextString(m) } +func (*Entity) ProtoMessage() {} +func (*Entity) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} } + +func (m *Entity) GetId() int32 { + if m != nil { + return m.Id + } + return 0 +} + +func (m *Entity) GetKey() []byte { + if m != nil { + return m.Key + } + return nil +} + +func (m *Entity) GetValue() []byte { + if m != nil { + return m.Value + } + return nil +} + +func (m *Entity) GetExists() bool { + if m != nil { + return m.Exists + } + return false +} + +func (m *Entity) GetStart() []byte { + if m != nil { + return m.Start + } + return nil +} + +func (m *Entity) GetEnd() []byte { + if m != nil { + return m.End + } + return nil +} + +func (m *Entity) GetErr() string { + if m != nil { + return m.Err + } + return "" +} + +func (m *Entity) GetCreatedAt() int64 { + if m != nil { + return m.CreatedAt + } + return 0 +} + +type Nothing struct { +} + +func (m *Nothing) Reset() { *m = Nothing{} } +func (m *Nothing) String() string { return proto.CompactTextString(m) } +func (*Nothing) ProtoMessage() {} +func (*Nothing) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} } + +type Domain struct { + Start []byte `protobuf:"bytes,1,opt,name=start,proto3" json:"start,omitempty"` + End []byte `protobuf:"bytes,2,opt,name=end,proto3" json:"end,omitempty"` +} + +func (m *Domain) Reset() { *m = Domain{} } +func (m *Domain) String() string { return proto.CompactTextString(m) } +func (*Domain) ProtoMessage() {} +func (*Domain) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} } + +func (m *Domain) GetStart() []byte { + if m != nil { + return m.Start + } + return nil +} + +func (m *Domain) GetEnd() []byte { + if m != nil { + return m.End + } + return nil +} + +type Iterator struct { + Domain *Domain `protobuf:"bytes,1,opt,name=domain" json:"domain,omitempty"` + Valid bool `protobuf:"varint,2,opt,name=valid" json:"valid,omitempty"` + Key []byte `protobuf:"bytes,3,opt,name=key,proto3" json:"key,omitempty"` + Value []byte `protobuf:"bytes,4,opt,name=value,proto3" json:"value,omitempty"` +} + +func (m *Iterator) Reset() { *m = Iterator{} } +func (m *Iterator) String() string { return proto.CompactTextString(m) } +func (*Iterator) ProtoMessage() {} +func (*Iterator) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} } + +func (m *Iterator) GetDomain() *Domain { + if m != nil { + return m.Domain + } + return nil +} + +func (m *Iterator) GetValid() bool { + if m != nil { + return m.Valid + } + return false +} + +func (m *Iterator) GetKey() []byte { + if m != nil { + return m.Key + } + return nil +} + +func (m *Iterator) GetValue() []byte { + if m != nil { + return m.Value + } + return nil +} + +type Stats struct { + Data map[string]string `protobuf:"bytes,1,rep,name=data" json:"data,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + TimeAt int64 `protobuf:"varint,2,opt,name=time_at,json=timeAt" json:"time_at,omitempty"` +} + +func (m *Stats) Reset() { *m = Stats{} } +func (m *Stats) String() string { return proto.CompactTextString(m) } +func (*Stats) ProtoMessage() {} +func (*Stats) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} } + +func (m *Stats) GetData() map[string]string { + if m != nil { + return m.Data + } + return nil +} + +func (m *Stats) GetTimeAt() int64 { + if m != nil { + return m.TimeAt + } + return 0 +} + +type Init struct { + Type string `protobuf:"bytes,1,opt,name=Type" json:"Type,omitempty"` + Name string `protobuf:"bytes,2,opt,name=Name" json:"Name,omitempty"` + Dir string `protobuf:"bytes,3,opt,name=Dir" json:"Dir,omitempty"` +} + +func (m *Init) Reset() { *m = Init{} } +func (m *Init) String() string { return proto.CompactTextString(m) } +func (*Init) ProtoMessage() {} +func (*Init) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} } + +func (m *Init) GetType() string { + if m != nil { + return m.Type + } + return "" +} + +func (m *Init) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *Init) GetDir() string { + if m != nil { + return m.Dir + } + return "" +} + +func init() { + proto.RegisterType((*Batch)(nil), "protodb.Batch") + proto.RegisterType((*Operation)(nil), "protodb.Operation") + proto.RegisterType((*Entity)(nil), "protodb.Entity") + proto.RegisterType((*Nothing)(nil), "protodb.Nothing") + proto.RegisterType((*Domain)(nil), "protodb.Domain") + proto.RegisterType((*Iterator)(nil), "protodb.Iterator") + proto.RegisterType((*Stats)(nil), "protodb.Stats") + proto.RegisterType((*Init)(nil), "protodb.Init") + proto.RegisterEnum("protodb.Operation_Type", Operation_Type_name, Operation_Type_value) +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion4 + +// Client API for DB service + +type DBClient interface { + Init(ctx context.Context, in *Init, opts ...grpc.CallOption) (*Entity, error) + Get(ctx context.Context, in *Entity, opts ...grpc.CallOption) (*Entity, error) + GetStream(ctx context.Context, opts ...grpc.CallOption) (DB_GetStreamClient, error) + Has(ctx context.Context, in *Entity, opts ...grpc.CallOption) (*Entity, error) + Set(ctx context.Context, in *Entity, opts ...grpc.CallOption) (*Nothing, error) + SetSync(ctx context.Context, in *Entity, opts ...grpc.CallOption) (*Nothing, error) + Delete(ctx context.Context, in *Entity, opts ...grpc.CallOption) (*Nothing, error) + DeleteSync(ctx context.Context, in *Entity, opts ...grpc.CallOption) (*Nothing, error) + Iterator(ctx context.Context, in *Entity, opts ...grpc.CallOption) (DB_IteratorClient, error) + ReverseIterator(ctx context.Context, in *Entity, opts ...grpc.CallOption) (DB_ReverseIteratorClient, error) + // rpc print(Nothing) returns (Entity) {} + Stats(ctx context.Context, in *Nothing, opts ...grpc.CallOption) (*Stats, error) + BatchWrite(ctx context.Context, in *Batch, opts ...grpc.CallOption) (*Nothing, error) + BatchWriteSync(ctx context.Context, in *Batch, opts ...grpc.CallOption) (*Nothing, error) +} + +type dBClient struct { + cc *grpc.ClientConn +} + +func NewDBClient(cc *grpc.ClientConn) DBClient { + return &dBClient{cc} +} + +func (c *dBClient) Init(ctx context.Context, in *Init, opts ...grpc.CallOption) (*Entity, error) { + out := new(Entity) + err := grpc.Invoke(ctx, "/protodb.DB/init", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *dBClient) Get(ctx context.Context, in *Entity, opts ...grpc.CallOption) (*Entity, error) { + out := new(Entity) + err := grpc.Invoke(ctx, "/protodb.DB/get", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *dBClient) GetStream(ctx context.Context, opts ...grpc.CallOption) (DB_GetStreamClient, error) { + stream, err := grpc.NewClientStream(ctx, &_DB_serviceDesc.Streams[0], c.cc, "/protodb.DB/getStream", opts...) + if err != nil { + return nil, err + } + x := &dBGetStreamClient{stream} + return x, nil +} + +type DB_GetStreamClient interface { + Send(*Entity) error + Recv() (*Entity, error) + grpc.ClientStream +} + +type dBGetStreamClient struct { + grpc.ClientStream +} + +func (x *dBGetStreamClient) Send(m *Entity) error { + return x.ClientStream.SendMsg(m) +} + +func (x *dBGetStreamClient) Recv() (*Entity, error) { + m := new(Entity) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *dBClient) Has(ctx context.Context, in *Entity, opts ...grpc.CallOption) (*Entity, error) { + out := new(Entity) + err := grpc.Invoke(ctx, "/protodb.DB/has", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *dBClient) Set(ctx context.Context, in *Entity, opts ...grpc.CallOption) (*Nothing, error) { + out := new(Nothing) + err := grpc.Invoke(ctx, "/protodb.DB/set", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *dBClient) SetSync(ctx context.Context, in *Entity, opts ...grpc.CallOption) (*Nothing, error) { + out := new(Nothing) + err := grpc.Invoke(ctx, "/protodb.DB/setSync", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *dBClient) Delete(ctx context.Context, in *Entity, opts ...grpc.CallOption) (*Nothing, error) { + out := new(Nothing) + err := grpc.Invoke(ctx, "/protodb.DB/delete", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *dBClient) DeleteSync(ctx context.Context, in *Entity, opts ...grpc.CallOption) (*Nothing, error) { + out := new(Nothing) + err := grpc.Invoke(ctx, "/protodb.DB/deleteSync", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *dBClient) Iterator(ctx context.Context, in *Entity, opts ...grpc.CallOption) (DB_IteratorClient, error) { + stream, err := grpc.NewClientStream(ctx, &_DB_serviceDesc.Streams[1], c.cc, "/protodb.DB/iterator", opts...) + if err != nil { + return nil, err + } + x := &dBIteratorClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type DB_IteratorClient interface { + Recv() (*Iterator, error) + grpc.ClientStream +} + +type dBIteratorClient struct { + grpc.ClientStream +} + +func (x *dBIteratorClient) Recv() (*Iterator, error) { + m := new(Iterator) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *dBClient) ReverseIterator(ctx context.Context, in *Entity, opts ...grpc.CallOption) (DB_ReverseIteratorClient, error) { + stream, err := grpc.NewClientStream(ctx, &_DB_serviceDesc.Streams[2], c.cc, "/protodb.DB/reverseIterator", opts...) + if err != nil { + return nil, err + } + x := &dBReverseIteratorClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type DB_ReverseIteratorClient interface { + Recv() (*Iterator, error) + grpc.ClientStream +} + +type dBReverseIteratorClient struct { + grpc.ClientStream +} + +func (x *dBReverseIteratorClient) Recv() (*Iterator, error) { + m := new(Iterator) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *dBClient) Stats(ctx context.Context, in *Nothing, opts ...grpc.CallOption) (*Stats, error) { + out := new(Stats) + err := grpc.Invoke(ctx, "/protodb.DB/stats", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *dBClient) BatchWrite(ctx context.Context, in *Batch, opts ...grpc.CallOption) (*Nothing, error) { + out := new(Nothing) + err := grpc.Invoke(ctx, "/protodb.DB/batchWrite", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *dBClient) BatchWriteSync(ctx context.Context, in *Batch, opts ...grpc.CallOption) (*Nothing, error) { + out := new(Nothing) + err := grpc.Invoke(ctx, "/protodb.DB/batchWriteSync", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// Server API for DB service + +type DBServer interface { + Init(context.Context, *Init) (*Entity, error) + Get(context.Context, *Entity) (*Entity, error) + GetStream(DB_GetStreamServer) error + Has(context.Context, *Entity) (*Entity, error) + Set(context.Context, *Entity) (*Nothing, error) + SetSync(context.Context, *Entity) (*Nothing, error) + Delete(context.Context, *Entity) (*Nothing, error) + DeleteSync(context.Context, *Entity) (*Nothing, error) + Iterator(*Entity, DB_IteratorServer) error + ReverseIterator(*Entity, DB_ReverseIteratorServer) error + // rpc print(Nothing) returns (Entity) {} + Stats(context.Context, *Nothing) (*Stats, error) + BatchWrite(context.Context, *Batch) (*Nothing, error) + BatchWriteSync(context.Context, *Batch) (*Nothing, error) +} + +func RegisterDBServer(s *grpc.Server, srv DBServer) { + s.RegisterService(&_DB_serviceDesc, srv) +} + +func _DB_Init_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Init) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DBServer).Init(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/protodb.DB/Init", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DBServer).Init(ctx, req.(*Init)) + } + return interceptor(ctx, in, info, handler) +} + +func _DB_Get_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Entity) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DBServer).Get(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/protodb.DB/Get", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DBServer).Get(ctx, req.(*Entity)) + } + return interceptor(ctx, in, info, handler) +} + +func _DB_GetStream_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(DBServer).GetStream(&dBGetStreamServer{stream}) +} + +type DB_GetStreamServer interface { + Send(*Entity) error + Recv() (*Entity, error) + grpc.ServerStream +} + +type dBGetStreamServer struct { + grpc.ServerStream +} + +func (x *dBGetStreamServer) Send(m *Entity) error { + return x.ServerStream.SendMsg(m) +} + +func (x *dBGetStreamServer) Recv() (*Entity, error) { + m := new(Entity) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func _DB_Has_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Entity) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DBServer).Has(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/protodb.DB/Has", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DBServer).Has(ctx, req.(*Entity)) + } + return interceptor(ctx, in, info, handler) +} + +func _DB_Set_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Entity) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DBServer).Set(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/protodb.DB/Set", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DBServer).Set(ctx, req.(*Entity)) + } + return interceptor(ctx, in, info, handler) +} + +func _DB_SetSync_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Entity) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DBServer).SetSync(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/protodb.DB/SetSync", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DBServer).SetSync(ctx, req.(*Entity)) + } + return interceptor(ctx, in, info, handler) +} + +func _DB_Delete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Entity) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DBServer).Delete(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/protodb.DB/Delete", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DBServer).Delete(ctx, req.(*Entity)) + } + return interceptor(ctx, in, info, handler) +} + +func _DB_DeleteSync_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Entity) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DBServer).DeleteSync(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/protodb.DB/DeleteSync", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DBServer).DeleteSync(ctx, req.(*Entity)) + } + return interceptor(ctx, in, info, handler) +} + +func _DB_Iterator_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(Entity) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(DBServer).Iterator(m, &dBIteratorServer{stream}) +} + +type DB_IteratorServer interface { + Send(*Iterator) error + grpc.ServerStream +} + +type dBIteratorServer struct { + grpc.ServerStream +} + +func (x *dBIteratorServer) Send(m *Iterator) error { + return x.ServerStream.SendMsg(m) +} + +func _DB_ReverseIterator_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(Entity) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(DBServer).ReverseIterator(m, &dBReverseIteratorServer{stream}) +} + +type DB_ReverseIteratorServer interface { + Send(*Iterator) error + grpc.ServerStream +} + +type dBReverseIteratorServer struct { + grpc.ServerStream +} + +func (x *dBReverseIteratorServer) Send(m *Iterator) error { + return x.ServerStream.SendMsg(m) +} + +func _DB_Stats_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Nothing) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DBServer).Stats(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/protodb.DB/Stats", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DBServer).Stats(ctx, req.(*Nothing)) + } + return interceptor(ctx, in, info, handler) +} + +func _DB_BatchWrite_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Batch) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DBServer).BatchWrite(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/protodb.DB/BatchWrite", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DBServer).BatchWrite(ctx, req.(*Batch)) + } + return interceptor(ctx, in, info, handler) +} + +func _DB_BatchWriteSync_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Batch) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DBServer).BatchWriteSync(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/protodb.DB/BatchWriteSync", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DBServer).BatchWriteSync(ctx, req.(*Batch)) + } + return interceptor(ctx, in, info, handler) +} + +var _DB_serviceDesc = grpc.ServiceDesc{ + ServiceName: "protodb.DB", + HandlerType: (*DBServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "init", + Handler: _DB_Init_Handler, + }, + { + MethodName: "get", + Handler: _DB_Get_Handler, + }, + { + MethodName: "has", + Handler: _DB_Has_Handler, + }, + { + MethodName: "set", + Handler: _DB_Set_Handler, + }, + { + MethodName: "setSync", + Handler: _DB_SetSync_Handler, + }, + { + MethodName: "delete", + Handler: _DB_Delete_Handler, + }, + { + MethodName: "deleteSync", + Handler: _DB_DeleteSync_Handler, + }, + { + MethodName: "stats", + Handler: _DB_Stats_Handler, + }, + { + MethodName: "batchWrite", + Handler: _DB_BatchWrite_Handler, + }, + { + MethodName: "batchWriteSync", + Handler: _DB_BatchWriteSync_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "getStream", + Handler: _DB_GetStream_Handler, + ServerStreams: true, + ClientStreams: true, + }, + { + StreamName: "iterator", + Handler: _DB_Iterator_Handler, + ServerStreams: true, + }, + { + StreamName: "reverseIterator", + Handler: _DB_ReverseIterator_Handler, + ServerStreams: true, + }, + }, + Metadata: "defs.proto", +} + +func init() { proto.RegisterFile("defs.proto", fileDescriptor0) } + +var fileDescriptor0 = []byte{ + // 606 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x54, 0x4f, 0x6f, 0xd3, 0x4e, + 0x10, 0xcd, 0xda, 0x8e, 0x13, 0x4f, 0x7f, 0xbf, 0x34, 0x8c, 0x10, 0xb5, 0x8a, 0x90, 0x22, 0x0b, + 0x09, 0x43, 0x69, 0x14, 0x52, 0x24, 0xfe, 0x9c, 0x68, 0x95, 0x1c, 0x2a, 0xa1, 0x22, 0x39, 0x95, + 0x38, 0xa2, 0x6d, 0x3d, 0x34, 0x2b, 0x1a, 0x3b, 0xac, 0x87, 0x8a, 0x5c, 0xb8, 0xf2, 0x79, 0xf8, + 0x7c, 0x5c, 0xd0, 0xae, 0x1d, 0x87, 0x36, 0x39, 0x84, 0x53, 0x76, 0x66, 0xde, 0x7b, 0xb3, 0xf3, + 0x32, 0x5e, 0x80, 0x94, 0x3e, 0x17, 0xfd, 0xb9, 0xce, 0x39, 0xc7, 0x96, 0xfd, 0x49, 0x2f, 0xa2, + 0x43, 0x68, 0x9e, 0x48, 0xbe, 0x9c, 0xe2, 0x63, 0x70, 0xf3, 0x79, 0x11, 0x8a, 0x9e, 0x1b, 0xef, + 0x0c, 0xb1, 0x5f, 0xd5, 0xfb, 0x1f, 0xe6, 0xa4, 0x25, 0xab, 0x3c, 0x4b, 0x4c, 0x39, 0xfa, 0x01, + 0x41, 0x9d, 0xc1, 0x27, 0xe0, 0x53, 0xc6, 0x8a, 0x17, 0xa1, 0xe8, 0x89, 0x78, 0x67, 0xb8, 0x5b, + 0xb3, 0xc6, 0x36, 0x9d, 0x54, 0x65, 0x3c, 0x00, 0x8f, 0x17, 0x73, 0x0a, 0x9d, 0x9e, 0x88, 0x3b, + 0xc3, 0xbd, 0x75, 0xf1, 0xfe, 0xf9, 0x62, 0x4e, 0x89, 0x05, 0x45, 0x0f, 0xc1, 0x33, 0x11, 0xb6, + 0xc0, 0x9d, 0x8c, 0xcf, 0xbb, 0x0d, 0x04, 0xf0, 0x47, 0xe3, 0xf7, 0xe3, 0xf3, 0x71, 0x57, 0x44, + 0xbf, 0x04, 0xf8, 0xa5, 0x38, 0x76, 0xc0, 0x51, 0xa9, 0xed, 0xdc, 0x4c, 0x1c, 0x95, 0x62, 0x17, + 0xdc, 0x2f, 0xb4, 0xb0, 0x3d, 0xfe, 0x4b, 0xcc, 0x11, 0xef, 0x43, 0xf3, 0x46, 0x5e, 0x7f, 0xa3, + 0xd0, 0xb5, 0xb9, 0x32, 0xc0, 0x07, 0xe0, 0xd3, 0x77, 0x55, 0x70, 0x11, 0x7a, 0x3d, 0x11, 0xb7, + 0x93, 0x2a, 0x32, 0xe8, 0x82, 0xa5, 0xe6, 0xb0, 0x59, 0xa2, 0x6d, 0x60, 0x54, 0x29, 0x4b, 0x43, + 0xbf, 0x54, 0xa5, 0xcc, 0xf6, 0x21, 0xad, 0xc3, 0x56, 0x4f, 0xc4, 0x41, 0x62, 0x8e, 0xf8, 0x08, + 0xe0, 0x52, 0x93, 0x64, 0x4a, 0x3f, 0x49, 0x0e, 0xdb, 0x3d, 0x11, 0xbb, 0x49, 0x50, 0x65, 0x8e, + 0x39, 0x0a, 0xa0, 0x75, 0x96, 0xf3, 0x54, 0x65, 0x57, 0xd1, 0x00, 0xfc, 0x51, 0x3e, 0x93, 0x2a, + 0x5b, 0x75, 0x13, 0x1b, 0xba, 0x39, 0x75, 0xb7, 0xe8, 0x2b, 0xb4, 0x4f, 0xd9, 0xb8, 0x94, 0x6b, + 0xe3, 0x77, 0x6a, 0xd9, 0x6b, 0x7e, 0x97, 0xa2, 0x49, 0x55, 0xae, 0x06, 0x57, 0xa5, 0x50, 0x3b, + 0x29, 0x83, 0xa5, 0x41, 0xee, 0x06, 0x83, 0xbc, 0xbf, 0x0c, 0x8a, 0x7e, 0x0a, 0x68, 0x4e, 0x58, + 0x72, 0x81, 0xcf, 0xc1, 0x4b, 0x25, 0xcb, 0x6a, 0x29, 0xc2, 0xba, 0x9d, 0xad, 0xf6, 0x47, 0x92, + 0xe5, 0x38, 0x63, 0xbd, 0x48, 0x2c, 0x0a, 0xf7, 0xa0, 0xc5, 0x6a, 0x46, 0xc6, 0x03, 0xc7, 0x7a, + 0xe0, 0x9b, 0xf0, 0x98, 0xf7, 0x5f, 0x41, 0x50, 0x63, 0x97, 0xb7, 0x10, 0xa5, 0x7d, 0xb7, 0x6e, + 0xe1, 0xd8, 0x5c, 0x19, 0xbc, 0x75, 0x5e, 0x8b, 0xe8, 0x1d, 0x78, 0xa7, 0x99, 0x62, 0xc4, 0x72, + 0x25, 0x2a, 0x52, 0xb9, 0x1e, 0x08, 0xde, 0x99, 0x9c, 0x2d, 0x49, 0xf6, 0x6c, 0xb4, 0x47, 0x4a, + 0xdb, 0x09, 0x83, 0xc4, 0x1c, 0x87, 0xbf, 0x3d, 0x70, 0x46, 0x27, 0x18, 0x83, 0xa7, 0x8c, 0xd0, + 0xff, 0xf5, 0x08, 0x46, 0x77, 0xff, 0xee, 0xc2, 0x46, 0x0d, 0x7c, 0x0a, 0xee, 0x15, 0x31, 0xde, + 0xad, 0x6c, 0x82, 0x1e, 0x41, 0x70, 0x45, 0x3c, 0x61, 0x4d, 0x72, 0xb6, 0x0d, 0x21, 0x16, 0x03, + 0x61, 0xf4, 0xa7, 0xb2, 0xd8, 0x4a, 0xff, 0x19, 0xb8, 0xc5, 0xa6, 0xab, 0x74, 0xeb, 0xc4, 0x72, + 0xad, 0x1a, 0xd8, 0x87, 0x56, 0x41, 0x3c, 0x59, 0x64, 0x97, 0xdb, 0xe1, 0x0f, 0xc1, 0x4f, 0xe9, + 0x9a, 0x98, 0xb6, 0x83, 0xbf, 0x30, 0x8f, 0x87, 0x81, 0x6f, 0xdf, 0x61, 0x08, 0x6d, 0xb5, 0x5c, + 0xdc, 0x35, 0xc2, 0xbd, 0xd5, 0xff, 0x50, 0x61, 0xa2, 0xc6, 0x40, 0xe0, 0x1b, 0xd8, 0xd5, 0x74, + 0x43, 0xba, 0xa0, 0xd3, 0x7f, 0xa5, 0x1e, 0xd8, 0xef, 0x89, 0x0b, 0x5c, 0xbb, 0xcb, 0x7e, 0xe7, + 0xf6, 0xde, 0x46, 0x0d, 0x1c, 0x00, 0x5c, 0x98, 0x47, 0xef, 0xa3, 0x56, 0x4c, 0xb8, 0xaa, 0xdb, + 0x97, 0x70, 0xe3, 0x34, 0x2f, 0xa1, 0xb3, 0x62, 0x58, 0x13, 0xb6, 0x60, 0x5d, 0xf8, 0x36, 0x75, + 0xf4, 0x27, 0x00, 0x00, 0xff, 0xff, 0x95, 0xf4, 0xe3, 0x82, 0x7a, 0x05, 0x00, 0x00, +} diff --git a/libs/db/remotedb/proto/defs.proto b/libs/db/remotedb/proto/defs.proto new file mode 100644 index 000000000..70471f234 --- /dev/null +++ b/libs/db/remotedb/proto/defs.proto @@ -0,0 +1,71 @@ +syntax = "proto3"; + +package protodb; + +message Batch { + repeated Operation ops = 1; +} + +message Operation { + Entity entity = 1; + enum Type { + SET = 0; + DELETE = 1; + } + Type type = 2; +} + +message Entity { + int32 id = 1; + bytes key = 2; + bytes value = 3; + bool exists = 4; + bytes start = 5; + bytes end = 6; + string err = 7; + int64 created_at = 8; +} + +message Nothing { +} + +message Domain { + bytes start = 1; + bytes end = 2; +} + +message Iterator { + Domain domain = 1; + bool valid = 2; + bytes key = 3; + bytes value = 4; +} + +message Stats { + map data = 1; + int64 time_at = 2; +} + +message Init { + string Type = 1; + string Name = 2; + string Dir = 3; +} + +service DB { + rpc init(Init) returns (Entity) {} + rpc get(Entity) returns (Entity) {} + rpc getStream(stream Entity) returns (stream Entity) {} + + rpc has(Entity) returns (Entity) {} + rpc set(Entity) returns (Nothing) {} + rpc setSync(Entity) returns (Nothing) {} + rpc delete(Entity) returns (Nothing) {} + rpc deleteSync(Entity) returns (Nothing) {} + rpc iterator(Entity) returns (stream Iterator) {} + rpc reverseIterator(Entity) returns (stream Iterator) {} + // rpc print(Nothing) returns (Entity) {} + rpc stats(Nothing) returns (Stats) {} + rpc batchWrite(Batch) returns (Nothing) {} + rpc batchWriteSync(Batch) returns (Nothing) {} +} diff --git a/libs/db/remotedb/remotedb.go b/libs/db/remotedb/remotedb.go new file mode 100644 index 000000000..2b60d8159 --- /dev/null +++ b/libs/db/remotedb/remotedb.go @@ -0,0 +1,262 @@ +package remotedb + +import ( + "context" + "fmt" + + "github.com/tendermint/tendermint/libs/db" + "github.com/tendermint/tendermint/libs/db/remotedb/grpcdb" + protodb "github.com/tendermint/tendermint/libs/db/remotedb/proto" +) + +type RemoteDB struct { + ctx context.Context + dc protodb.DBClient +} + +func NewRemoteDB(serverAddr string, serverKey string) (*RemoteDB, error) { + return newRemoteDB(grpcdb.NewClient(serverAddr, serverKey)) +} + +func newRemoteDB(gdc protodb.DBClient, err error) (*RemoteDB, error) { + if err != nil { + return nil, err + } + return &RemoteDB{dc: gdc, ctx: context.Background()}, nil +} + +type Init struct { + Dir string + Name string + Type string +} + +func (rd *RemoteDB) InitRemote(in *Init) error { + _, err := rd.dc.Init(rd.ctx, &protodb.Init{Dir: in.Dir, Type: in.Type, Name: in.Name}) + return err +} + +var _ db.DB = (*RemoteDB)(nil) + +// Close is a noop currently +func (rd *RemoteDB) Close() { +} + +func (rd *RemoteDB) Delete(key []byte) { + if _, err := rd.dc.Delete(rd.ctx, &protodb.Entity{Key: key}); err != nil { + panic(fmt.Sprintf("RemoteDB.Delete: %v", err)) + } +} + +func (rd *RemoteDB) DeleteSync(key []byte) { + if _, err := rd.dc.DeleteSync(rd.ctx, &protodb.Entity{Key: key}); err != nil { + panic(fmt.Sprintf("RemoteDB.DeleteSync: %v", err)) + } +} + +func (rd *RemoteDB) Set(key, value []byte) { + if _, err := rd.dc.Set(rd.ctx, &protodb.Entity{Key: key, Value: value}); err != nil { + panic(fmt.Sprintf("RemoteDB.Set: %v", err)) + } +} + +func (rd *RemoteDB) SetSync(key, value []byte) { + if _, err := rd.dc.SetSync(rd.ctx, &protodb.Entity{Key: key, Value: value}); err != nil { + panic(fmt.Sprintf("RemoteDB.SetSync: %v", err)) + } +} + +func (rd *RemoteDB) Get(key []byte) []byte { + res, err := rd.dc.Get(rd.ctx, &protodb.Entity{Key: key}) + if err != nil { + panic(fmt.Sprintf("RemoteDB.Get error: %v", err)) + } + return res.Value +} + +func (rd *RemoteDB) Has(key []byte) bool { + res, err := rd.dc.Has(rd.ctx, &protodb.Entity{Key: key}) + if err != nil { + panic(fmt.Sprintf("RemoteDB.Has error: %v", err)) + } + return res.Exists +} + +func (rd *RemoteDB) ReverseIterator(start, end []byte) db.Iterator { + dic, err := rd.dc.ReverseIterator(rd.ctx, &protodb.Entity{Start: start, End: end}) + if err != nil { + panic(fmt.Sprintf("RemoteDB.Iterator error: %v", err)) + } + return makeReverseIterator(dic) +} + +func (rd *RemoteDB) NewBatch() db.Batch { + return &batch{ + db: rd, + ops: nil, + } +} + +// TODO: Implement Print when db.DB implements a method +// to print to a string and not db.Print to stdout. +func (rd *RemoteDB) Print() { + panic("Unimplemented") +} + +func (rd *RemoteDB) Stats() map[string]string { + stats, err := rd.dc.Stats(rd.ctx, &protodb.Nothing{}) + if err != nil { + panic(fmt.Sprintf("RemoteDB.Stats error: %v", err)) + } + if stats == nil { + return nil + } + return stats.Data +} + +func (rd *RemoteDB) Iterator(start, end []byte) db.Iterator { + dic, err := rd.dc.Iterator(rd.ctx, &protodb.Entity{Start: start, End: end}) + if err != nil { + panic(fmt.Sprintf("RemoteDB.Iterator error: %v", err)) + } + return makeIterator(dic) +} + +func makeIterator(dic protodb.DB_IteratorClient) db.Iterator { + return &iterator{dic: dic} +} + +func makeReverseIterator(dric protodb.DB_ReverseIteratorClient) db.Iterator { + return &reverseIterator{dric: dric} +} + +type reverseIterator struct { + dric protodb.DB_ReverseIteratorClient + cur *protodb.Iterator +} + +var _ db.Iterator = (*iterator)(nil) + +func (rItr *reverseIterator) Valid() bool { + return rItr.cur != nil && rItr.cur.Valid +} + +func (rItr *reverseIterator) Domain() (start, end []byte) { + if rItr.cur == nil || rItr.cur.Domain == nil { + return nil, nil + } + return rItr.cur.Domain.Start, rItr.cur.Domain.End +} + +// Next advances the current reverseIterator +func (rItr *reverseIterator) Next() { + var err error + rItr.cur, err = rItr.dric.Recv() + if err != nil { + panic(fmt.Sprintf("RemoteDB.ReverseIterator.Next error: %v", err)) + } +} + +func (rItr *reverseIterator) Key() []byte { + if rItr.cur == nil { + return nil + } + return rItr.cur.Key +} + +func (rItr *reverseIterator) Value() []byte { + if rItr.cur == nil { + return nil + } + return rItr.cur.Value +} + +func (rItr *reverseIterator) Close() { +} + +// iterator implements the db.Iterator by retrieving +// streamed iterators from the remote backend as +// needed. It is NOT safe for concurrent usage, +// matching the behavior of other iterators. +type iterator struct { + dic protodb.DB_IteratorClient + cur *protodb.Iterator +} + +var _ db.Iterator = (*iterator)(nil) + +func (itr *iterator) Valid() bool { + return itr.cur != nil && itr.cur.Valid +} + +func (itr *iterator) Domain() (start, end []byte) { + if itr.cur == nil || itr.cur.Domain == nil { + return nil, nil + } + return itr.cur.Domain.Start, itr.cur.Domain.End +} + +// Next advances the current iterator +func (itr *iterator) Next() { + var err error + itr.cur, err = itr.dic.Recv() + if err != nil { + panic(fmt.Sprintf("RemoteDB.Iterator.Next error: %v", err)) + } +} + +func (itr *iterator) Key() []byte { + if itr.cur == nil { + return nil + } + return itr.cur.Key +} + +func (itr *iterator) Value() []byte { + if itr.cur == nil { + return nil + } + return itr.cur.Value +} + +func (itr *iterator) Close() { + err := itr.dic.CloseSend() + if err != nil { + panic(fmt.Sprintf("Error closing iterator: %v", err)) + } +} + +type batch struct { + db *RemoteDB + ops []*protodb.Operation +} + +var _ db.Batch = (*batch)(nil) + +func (bat *batch) Set(key, value []byte) { + op := &protodb.Operation{ + Entity: &protodb.Entity{Key: key, Value: value}, + Type: protodb.Operation_SET, + } + bat.ops = append(bat.ops, op) +} + +func (bat *batch) Delete(key []byte) { + op := &protodb.Operation{ + Entity: &protodb.Entity{Key: key}, + Type: protodb.Operation_DELETE, + } + bat.ops = append(bat.ops, op) +} + +func (bat *batch) Write() { + if _, err := bat.db.dc.BatchWrite(bat.db.ctx, &protodb.Batch{Ops: bat.ops}); err != nil { + panic(fmt.Sprintf("RemoteDB.BatchWrite: %v", err)) + } +} + +func (bat *batch) WriteSync() { + if _, err := bat.db.dc.BatchWriteSync(bat.db.ctx, &protodb.Batch{Ops: bat.ops}); err != nil { + panic(fmt.Sprintf("RemoteDB.BatchWriteSync: %v", err)) + } +} diff --git a/libs/db/remotedb/remotedb_test.go b/libs/db/remotedb/remotedb_test.go new file mode 100644 index 000000000..0e7319971 --- /dev/null +++ b/libs/db/remotedb/remotedb_test.go @@ -0,0 +1,123 @@ +package remotedb_test + +import ( + "net" + "os" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/tendermint/tendermint/libs/db/remotedb" + "github.com/tendermint/tendermint/libs/db/remotedb/grpcdb" +) + +func TestRemoteDB(t *testing.T) { + cert := "test.crt" + key := "test.key" + ln, err := net.Listen("tcp", "0.0.0.0:0") + require.Nil(t, err, "expecting a port to have been assigned on which we can listen") + srv, err := grpcdb.NewServer(cert, key) + require.Nil(t, err) + defer srv.Stop() + go func() { + if err := srv.Serve(ln); err != nil { + t.Fatalf("BindServer: %v", err) + } + }() + + client, err := remotedb.NewRemoteDB(ln.Addr().String(), cert) + require.Nil(t, err, "expecting a successful client creation") + dbName := "test-remote-db" + require.Nil(t, client.InitRemote(&remotedb.Init{Name: dbName, Type: "leveldb"})) + defer func() { + err := os.RemoveAll(dbName + ".db") + if err != nil { + panic(err) + } + }() + + k1 := []byte("key-1") + v1 := client.Get(k1) + require.Equal(t, 0, len(v1), "expecting no key1 to have been stored, got %X (%s)", v1, v1) + vv1 := []byte("value-1") + client.Set(k1, vv1) + gv1 := client.Get(k1) + require.Equal(t, gv1, vv1) + + // Simple iteration + itr := client.Iterator(nil, nil) + itr.Next() + require.Equal(t, itr.Key(), []byte("key-1")) + require.Equal(t, itr.Value(), []byte("value-1")) + require.Panics(t, itr.Next) + itr.Close() + + // Set some more keys + k2 := []byte("key-2") + v2 := []byte("value-2") + client.SetSync(k2, v2) + has := client.Has(k2) + require.True(t, has) + gv2 := client.Get(k2) + require.Equal(t, gv2, v2) + + // More iteration + itr = client.Iterator(nil, nil) + itr.Next() + require.Equal(t, itr.Key(), []byte("key-1")) + require.Equal(t, itr.Value(), []byte("value-1")) + itr.Next() + require.Equal(t, itr.Key(), []byte("key-2")) + require.Equal(t, itr.Value(), []byte("value-2")) + require.Panics(t, itr.Next) + itr.Close() + + // Deletion + client.Delete(k1) + client.DeleteSync(k2) + gv1 = client.Get(k1) + gv2 = client.Get(k2) + require.Equal(t, len(gv2), 0, "after deletion, not expecting the key to exist anymore") + require.Equal(t, len(gv1), 0, "after deletion, not expecting the key to exist anymore") + + // Batch tests - set + k3 := []byte("key-3") + k4 := []byte("key-4") + k5 := []byte("key-5") + v3 := []byte("value-3") + v4 := []byte("value-4") + v5 := []byte("value-5") + bat := client.NewBatch() + bat.Set(k3, v3) + bat.Set(k4, v4) + rv3 := client.Get(k3) + require.Equal(t, 0, len(rv3), "expecting no k3 to have been stored") + rv4 := client.Get(k4) + require.Equal(t, 0, len(rv4), "expecting no k4 to have been stored") + bat.Write() + rv3 = client.Get(k3) + require.Equal(t, rv3, v3, "expecting k3 to have been stored") + rv4 = client.Get(k4) + require.Equal(t, rv4, v4, "expecting k4 to have been stored") + + // Batch tests - deletion + bat = client.NewBatch() + bat.Delete(k4) + bat.Delete(k3) + bat.WriteSync() + rv3 = client.Get(k3) + require.Equal(t, 0, len(rv3), "expecting k3 to have been deleted") + rv4 = client.Get(k4) + require.Equal(t, 0, len(rv4), "expecting k4 to have been deleted") + + // Batch tests - set and delete + bat = client.NewBatch() + bat.Set(k4, v4) + bat.Set(k5, v5) + bat.Delete(k4) + bat.WriteSync() + rv4 = client.Get(k4) + require.Equal(t, 0, len(rv4), "expecting k4 to have been deleted") + rv5 := client.Get(k5) + require.Equal(t, rv5, v5, "expecting k5 to have been stored") +} diff --git a/libs/db/remotedb/test.crt b/libs/db/remotedb/test.crt new file mode 100644 index 000000000..bdc8a0f29 --- /dev/null +++ b/libs/db/remotedb/test.crt @@ -0,0 +1,25 @@ +-----BEGIN CERTIFICATE----- +MIIEQTCCAimgAwIBAgIRANqF1HD19i/uvQ3n62TAKTwwDQYJKoZIhvcNAQELBQAw +GTEXMBUGA1UEAxMOdGVuZGVybWludC5jb20wHhcNMTgwNzAyMDMwNzMyWhcNMjAw +MTAyMDMwNzMwWjANMQswCQYDVQQDEwI6OjCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBAOuWUMCSzYJmvKU1vsouDTe7OxnPWO3oV0FjSH8vKYoi2zpZQX35 +dQDPtLDF2/v/ANZJ5pzMJR8yMMtEQ4tWxKuGzJw1ZgTgHtASPbj/M5fDnDO7Hqg4 +D09eLTkZAUfiBf6BzDyQIHn22CUexhaS70TbIT9AOAoOsGXMZz9d+iImKIm+gbzf +pR52LNbBGesHWGjwIuGF4InstIMsKSwGv2DctzhWI+i/m5Goi3rd1V8z/lzUbsf1 +0uXqQcSfTyv3ee6YiCWj2W8vcdc5H+B6KzSlGjAR4sRcHTHOQJYO9BgA9evQ3qsJ +Pp00iez13RdheJWPtbfUqQy4gdpu8HFeZx8CAwEAAaOBjzCBjDAOBgNVHQ8BAf8E +BAMCA7gwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMB0GA1UdDgQWBBRc +XBo+bJILrLcJiGkTWeMPpXb1TDAfBgNVHSMEGDAWgBQqk1Xu65Ww7EBCROw4KLGw +KuToaDAbBgNVHREEFDAShxAAAAAAAAAAAAAAAAAAAAAAMA0GCSqGSIb3DQEBCwUA +A4ICAQAbGsIMhL8clczNmhGl9xZhmyNz6FbLq6g163x9LTgfvwHPt+7urthtd++O +uy4Ut8zFurh/yk7eooPlzf8jO7QUJBAFVy4vj8IcsvpWbFa7cuEOIulbjIzyAm/v +lgy7vUQ6xrWn8x8O9K1ww9z7wugwCyl22BD0wSHZKclJz++AwpL6vUVOD76IIuJO ++S6bE6z26/0ndpundh2AkA++2eIleD6ygnTeTl0PWu6aGoCggBmos50f8KgYHZF/ +OZVef203kDls9xCaOiMzaU91VsgLqq/gNcT+2cBd5r3IZTY3C8Rve6EEHS+/4zxf +PKlmiLN7lU9GFZogKecYzY+zPT7OArY7OVFnGTo4qdhdmxnXzHsI+anMCjxLOgEJ +381hyplQGPQOouEupCBxFcwa7oMYoGu20+1nLWYEqFcIXCeyH+s77MyteJSsseqL +xivG5PT+jKJn9hrnFb39bBmht9Vsa+Th6vk953zi5wCSe1j2wXsxFaENDq6BQZOK +f86Kp86M2elYnv3lJ3j2DE2ZTMpw+PA5ThYUnB+HVqYeeB2Y3ErRS8P1FOp1LBE8 ++eTz7yXQO5OM2wdYhNNL1zDri/41fHXi9b6337PZVqc39GM+N74x/O4Q7xEBiWgQ +T0dT8SNwf55kv63MeZh63ImxFV0FNRkCteYLcJMle3ohIY4zyQ== +-----END CERTIFICATE----- diff --git a/libs/db/remotedb/test.key b/libs/db/remotedb/test.key new file mode 100644 index 000000000..14d285584 --- /dev/null +++ b/libs/db/remotedb/test.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpgIBAAKCAQEA65ZQwJLNgma8pTW+yi4NN7s7Gc9Y7ehXQWNIfy8piiLbOllB +ffl1AM+0sMXb+/8A1knmnMwlHzIwy0RDi1bEq4bMnDVmBOAe0BI9uP8zl8OcM7se +qDgPT14tORkBR+IF/oHMPJAgefbYJR7GFpLvRNshP0A4Cg6wZcxnP136IiYoib6B +vN+lHnYs1sEZ6wdYaPAi4YXgiey0gywpLAa/YNy3OFYj6L+bkaiLet3VXzP+XNRu +x/XS5epBxJ9PK/d57piIJaPZby9x1zkf4HorNKUaMBHixFwdMc5Alg70GAD169De +qwk+nTSJ7PXdF2F4lY+1t9SpDLiB2m7wcV5nHwIDAQABAoIBAQCB2/ilPgaUE8d2 +ldqWHa5hgw4/2uCdO04ll/GVUczm/PG1BxAnvYL2MIfcTSRGkrjGZjP9SDZKLONi +mD1XKDv+hK5yiKi0lUnGzddCC0JILKYEieeLOGOQD0yERblEA13kfW20EIomUJ+y +TnVIajQD03pPIDoDqTco1fQvpMDFYw5Q//UhH7VBC261GO1akvhT2Gqdb4aKLaYQ +iDW9IEButL5cRKIJuRxToB/JbmPVEF7xIZtm0sf9dtYVOlBQLeID0uHXgaci0enc +de6GMajmj7NFqc36ypb+Ct18fqEwQBYD+TSQdKs7/lMsAXwRjd5HW4RbYiMZyYnf +Dxgh7QVBAoGBAP9aLLIUcIG7+gk1x7xd+8wRhfo+dhsungeCluSigI9AsfDr6dpR +G9/0lEJH56noZZKQueACTmj7shmRB40xFFLc8w0IDRZCnofsl+Z15k9K84uFPA3W +hdZH9nMieU/mRKdcUYK7pHGqbicHTaJQ5ydZ+xb2E+zYQHOzYpQacHv/AoGBAOwv +TjDZSiassnAPYmmfcHtkUF4gf7PTpiZfH0hXHGAb0mJX4cXAoktAeDeHSi2tz3LW +dAc0ReP8Pdf3uSNv7wkJ1KpNRxAhU5bhnDFmjRc7gMZknVOU+az2M+4yGOn/SOiJ +I6uMHgQDS/VsI+N583n6gbGxVHbQfr9TOc4bLpThAoGBAKin0JmWMnEdzRnEMbZS +hPrWIB2Wn794XNws/qjoQ+1aF60+xGhz5etXyYy1nWd1nZDekkZIf62LgKiuR8ST +xA6u7MGQrcQkID06oWGQQZvhr1ZZm76wEBnl0ftdq66AMpwvt46XjReeL78LbdVl +hidRoSwbQDHQ61EADH4xsFXVAoGBAISXqhXSZsZ/fU1b1avmTod3MYcmR4r07vnr +vOwnu05ZUCrVm3IhSvtkHhlOYl5yjVuy+UByICp1mWJ9N/qlBFTWqAVTjOmJTBwQ +XFd/cwXv6cN3CLu7js+DCHRYu5PiNVQWaWgNKWynTSViqGM0O3PnJphTLU/mjMFs +P69toyEBAoGBALh9YsqxHdYdS5WK9chzDfGlaTQ79jwN+gEzQuP1ooLF0JkMgh5W +//2C6kCrgBsGTm1gfHAjEfC04ZDZLFbKLm56YVKUGL6JJNapm6e5kfiZGjbRKWAg +ViCeRS2qQnVbH74GfHyimeTPDI9cJMiJfDDTPbfosqWSsPEcg2jfsySJ +-----END RSA PRIVATE KEY----- diff --git a/libs/db/types.go b/libs/db/types.go new file mode 100644 index 000000000..ad78859a7 --- /dev/null +++ b/libs/db/types.go @@ -0,0 +1,134 @@ +package db + +// DBs are goroutine safe. +type DB interface { + + // Get returns nil iff key doesn't exist. + // A nil key is interpreted as an empty byteslice. + // CONTRACT: key, value readonly []byte + Get([]byte) []byte + + // Has checks if a key exists. + // A nil key is interpreted as an empty byteslice. + // CONTRACT: key, value readonly []byte + Has(key []byte) bool + + // Set sets the key. + // A nil key is interpreted as an empty byteslice. + // CONTRACT: key, value readonly []byte + Set([]byte, []byte) + SetSync([]byte, []byte) + + // Delete deletes the key. + // A nil key is interpreted as an empty byteslice. + // CONTRACT: key readonly []byte + Delete([]byte) + DeleteSync([]byte) + + // Iterate over a domain of keys in ascending order. End is exclusive. + // Start must be less than end, or the Iterator is invalid. + // A nil start is interpreted as an empty byteslice. + // If end is nil, iterates up to the last item (inclusive). + // CONTRACT: No writes may happen within a domain while an iterator exists over it. + // CONTRACT: start, end readonly []byte + Iterator(start, end []byte) Iterator + + // Iterate over a domain of keys in descending order. End is exclusive. + // Start must be greater than end, or the Iterator is invalid. + // If start is nil, iterates from the last/greatest item (inclusive). + // If end is nil, iterates up to the first/least item (inclusive). + // CONTRACT: No writes may happen within a domain while an iterator exists over it. + // CONTRACT: start, end readonly []byte + ReverseIterator(start, end []byte) Iterator + + // Closes the connection. + Close() + + // Creates a batch for atomic updates. + NewBatch() Batch + + // For debugging + Print() + + // Stats returns a map of property values for all keys and the size of the cache. + Stats() map[string]string +} + +//---------------------------------------- +// Batch + +type Batch interface { + SetDeleter + Write() + WriteSync() +} + +type SetDeleter interface { + Set(key, value []byte) // CONTRACT: key, value readonly []byte + Delete(key []byte) // CONTRACT: key readonly []byte +} + +//---------------------------------------- +// Iterator + +/* + Usage: + + var itr Iterator = ... + defer itr.Close() + + for ; itr.Valid(); itr.Next() { + k, v := itr.Key(); itr.Value() + // ... + } +*/ +type Iterator interface { + + // The start & end (exclusive) limits to iterate over. + // If end < start, then the Iterator goes in reverse order. + // + // A domain of ([]byte{12, 13}, []byte{12, 14}) will iterate + // over anything with the prefix []byte{12, 13}. + // + // The smallest key is the empty byte array []byte{} - see BeginningKey(). + // The largest key is the nil byte array []byte(nil) - see EndingKey(). + // CONTRACT: start, end readonly []byte + Domain() (start []byte, end []byte) + + // Valid returns whether the current position is valid. + // Once invalid, an Iterator is forever invalid. + Valid() bool + + // Next moves the iterator to the next sequential key in the database, as + // defined by order of iteration. + // + // If Valid returns false, this method will panic. + Next() + + // Key returns the key of the cursor. + // If Valid returns false, this method will panic. + // CONTRACT: key readonly []byte + Key() (key []byte) + + // Value returns the value of the cursor. + // If Valid returns false, this method will panic. + // CONTRACT: value readonly []byte + Value() (value []byte) + + // Close releases the Iterator. + Close() +} + +// For testing convenience. +func bz(s string) []byte { + return []byte(s) +} + +// We defensively turn nil keys or values into []byte{} for +// most operations. +func nonNilBytes(bz []byte) []byte { + if bz == nil { + return []byte{} + } + return bz +} diff --git a/libs/db/util.go b/libs/db/util.go new file mode 100644 index 000000000..51277ac42 --- /dev/null +++ b/libs/db/util.go @@ -0,0 +1,78 @@ +package db + +import ( + "bytes" +) + +func cp(bz []byte) (ret []byte) { + ret = make([]byte, len(bz)) + copy(ret, bz) + return ret +} + +// Returns a slice of the same length (big endian) +// except incremented by one. +// Returns nil on overflow (e.g. if bz bytes are all 0xFF) +// CONTRACT: len(bz) > 0 +func cpIncr(bz []byte) (ret []byte) { + if len(bz) == 0 { + panic("cpIncr expects non-zero bz length") + } + ret = cp(bz) + for i := len(bz) - 1; i >= 0; i-- { + if ret[i] < byte(0xFF) { + ret[i]++ + return + } + ret[i] = byte(0x00) + if i == 0 { + // Overflow + return nil + } + } + return nil +} + +// Returns a slice of the same length (big endian) +// except decremented by one. +// Returns nil on underflow (e.g. if bz bytes are all 0x00) +// CONTRACT: len(bz) > 0 +func cpDecr(bz []byte) (ret []byte) { + if len(bz) == 0 { + panic("cpDecr expects non-zero bz length") + } + ret = cp(bz) + for i := len(bz) - 1; i >= 0; i-- { + if ret[i] > byte(0x00) { + ret[i]-- + return + } + ret[i] = byte(0xFF) + if i == 0 { + // Underflow + return nil + } + } + return nil +} + +// See DB interface documentation for more information. +func IsKeyInDomain(key, start, end []byte, isReverse bool) bool { + if !isReverse { + if bytes.Compare(key, start) < 0 { + return false + } + if end != nil && bytes.Compare(end, key) <= 0 { + return false + } + return true + } else { + if start != nil && bytes.Compare(start, key) < 0 { + return false + } + if end != nil && bytes.Compare(key, end) <= 0 { + return false + } + return true + } +} diff --git a/libs/db/util_test.go b/libs/db/util_test.go new file mode 100644 index 000000000..44f1f9f73 --- /dev/null +++ b/libs/db/util_test.go @@ -0,0 +1,93 @@ +package db + +import ( + "fmt" + "testing" +) + +// Empty iterator for empty db. +func TestPrefixIteratorNoMatchNil(t *testing.T) { + for backend := range backends { + t.Run(fmt.Sprintf("Prefix w/ backend %s", backend), func(t *testing.T) { + db := newTempDB(t, backend) + itr := IteratePrefix(db, []byte("2")) + + checkInvalid(t, itr) + }) + } +} + +// Empty iterator for db populated after iterator created. +func TestPrefixIteratorNoMatch1(t *testing.T) { + for backend := range backends { + t.Run(fmt.Sprintf("Prefix w/ backend %s", backend), func(t *testing.T) { + db := newTempDB(t, backend) + itr := IteratePrefix(db, []byte("2")) + db.SetSync(bz("1"), bz("value_1")) + + checkInvalid(t, itr) + }) + } +} + +// Empty iterator for prefix starting after db entry. +func TestPrefixIteratorNoMatch2(t *testing.T) { + for backend := range backends { + t.Run(fmt.Sprintf("Prefix w/ backend %s", backend), func(t *testing.T) { + db := newTempDB(t, backend) + db.SetSync(bz("3"), bz("value_3")) + itr := IteratePrefix(db, []byte("4")) + + checkInvalid(t, itr) + }) + } +} + +// Iterator with single val for db with single val, starting from that val. +func TestPrefixIteratorMatch1(t *testing.T) { + for backend := range backends { + t.Run(fmt.Sprintf("Prefix w/ backend %s", backend), func(t *testing.T) { + db := newTempDB(t, backend) + db.SetSync(bz("2"), bz("value_2")) + itr := IteratePrefix(db, bz("2")) + + checkValid(t, itr, true) + checkItem(t, itr, bz("2"), bz("value_2")) + checkNext(t, itr, false) + + // Once invalid... + checkInvalid(t, itr) + }) + } +} + +// Iterator with prefix iterates over everything with same prefix. +func TestPrefixIteratorMatches1N(t *testing.T) { + for backend := range backends { + t.Run(fmt.Sprintf("Prefix w/ backend %s", backend), func(t *testing.T) { + db := newTempDB(t, backend) + + // prefixed + db.SetSync(bz("a/1"), bz("value_1")) + db.SetSync(bz("a/3"), bz("value_3")) + + // not + db.SetSync(bz("b/3"), bz("value_3")) + db.SetSync(bz("a-3"), bz("value_3")) + db.SetSync(bz("a.3"), bz("value_3")) + db.SetSync(bz("abcdefg"), bz("value_3")) + itr := IteratePrefix(db, bz("a/")) + + checkValid(t, itr, true) + checkItem(t, itr, bz("a/1"), bz("value_1")) + checkNext(t, itr, true) + checkItem(t, itr, bz("a/3"), bz("value_3")) + + // Bad! + checkNext(t, itr, false) + + //Once invalid... + checkInvalid(t, itr) + }) + } +} diff --git a/libs/events/events.go b/libs/events/events.go index 075f9b42b..9c7f0fd05 100644 --- a/libs/events/events.go +++ b/libs/events/events.go @@ -6,7 +6,7 @@ package events import ( "sync" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) // Generic event data can be typed and registered with tendermint/go-amino diff --git a/libs/flowrate/README.md b/libs/flowrate/README.md new file mode 100644 index 000000000..db428090c --- /dev/null +++ b/libs/flowrate/README.md @@ -0,0 +1,10 @@ +Data Flow Rate Control +====================== + +To download and install this package run: + +go get github.com/mxk/go-flowrate/flowrate + +The documentation is available at: + +http://godoc.org/github.com/mxk/go-flowrate/flowrate diff --git a/libs/flowrate/flowrate.go b/libs/flowrate/flowrate.go new file mode 100644 index 000000000..e233eae0f --- /dev/null +++ b/libs/flowrate/flowrate.go @@ -0,0 +1,275 @@ +// +// Written by Maxim Khitrov (November 2012) +// + +// Package flowrate provides the tools for monitoring and limiting the flow rate +// of an arbitrary data stream. +package flowrate + +import ( + "math" + "sync" + "time" +) + +// Monitor monitors and limits the transfer rate of a data stream. +type Monitor struct { + mu sync.Mutex // Mutex guarding access to all internal fields + active bool // Flag indicating an active transfer + start time.Duration // Transfer start time (clock() value) + bytes int64 // Total number of bytes transferred + samples int64 // Total number of samples taken + + rSample float64 // Most recent transfer rate sample (bytes per second) + rEMA float64 // Exponential moving average of rSample + rPeak float64 // Peak transfer rate (max of all rSamples) + rWindow float64 // rEMA window (seconds) + + sBytes int64 // Number of bytes transferred since sLast + sLast time.Duration // Most recent sample time (stop time when inactive) + sRate time.Duration // Sampling rate + + tBytes int64 // Number of bytes expected in the current transfer + tLast time.Duration // Time of the most recent transfer of at least 1 byte +} + +// New creates a new flow control monitor. Instantaneous transfer rate is +// measured and updated for each sampleRate interval. windowSize determines the +// weight of each sample in the exponential moving average (EMA) calculation. +// The exact formulas are: +// +// sampleTime = currentTime - prevSampleTime +// sampleRate = byteCount / sampleTime +// weight = 1 - exp(-sampleTime/windowSize) +// newRate = weight*sampleRate + (1-weight)*oldRate +// +// The default values for sampleRate and windowSize (if <= 0) are 100ms and 1s, +// respectively. +func New(sampleRate, windowSize time.Duration) *Monitor { + if sampleRate = clockRound(sampleRate); sampleRate <= 0 { + sampleRate = 5 * clockRate + } + if windowSize <= 0 { + windowSize = 1 * time.Second + } + now := clock() + return &Monitor{ + active: true, + start: now, + rWindow: windowSize.Seconds(), + sLast: now, + sRate: sampleRate, + tLast: now, + } +} + +// Update records the transfer of n bytes and returns n. It should be called +// after each Read/Write operation, even if n is 0. +func (m *Monitor) Update(n int) int { + m.mu.Lock() + m.update(n) + m.mu.Unlock() + return n +} + +// Hack to set the current rEMA. +func (m *Monitor) SetREMA(rEMA float64) { + m.mu.Lock() + m.rEMA = rEMA + m.samples++ + m.mu.Unlock() +} + +// IO is a convenience method intended to wrap io.Reader and io.Writer method +// execution. It calls m.Update(n) and then returns (n, err) unmodified. +func (m *Monitor) IO(n int, err error) (int, error) { + return m.Update(n), err +} + +// Done marks the transfer as finished and prevents any further updates or +// limiting. Instantaneous and current transfer rates drop to 0. Update, IO, and +// Limit methods become NOOPs. It returns the total number of bytes transferred. +func (m *Monitor) Done() int64 { + m.mu.Lock() + if now := m.update(0); m.sBytes > 0 { + m.reset(now) + } + m.active = false + m.tLast = 0 + n := m.bytes + m.mu.Unlock() + return n +} + +// timeRemLimit is the maximum Status.TimeRem value. +const timeRemLimit = 999*time.Hour + 59*time.Minute + 59*time.Second + +// Status represents the current Monitor status. All transfer rates are in bytes +// per second rounded to the nearest byte. +type Status struct { + Active bool // Flag indicating an active transfer + Start time.Time // Transfer start time + Duration time.Duration // Time period covered by the statistics + Idle time.Duration // Time since the last transfer of at least 1 byte + Bytes int64 // Total number of bytes transferred + Samples int64 // Total number of samples taken + InstRate int64 // Instantaneous transfer rate + CurRate int64 // Current transfer rate (EMA of InstRate) + AvgRate int64 // Average transfer rate (Bytes / Duration) + PeakRate int64 // Maximum instantaneous transfer rate + BytesRem int64 // Number of bytes remaining in the transfer + TimeRem time.Duration // Estimated time to completion + Progress Percent // Overall transfer progress +} + +// Status returns current transfer status information. The returned value +// becomes static after a call to Done. +func (m *Monitor) Status() Status { + m.mu.Lock() + now := m.update(0) + s := Status{ + Active: m.active, + Start: clockToTime(m.start), + Duration: m.sLast - m.start, + Idle: now - m.tLast, + Bytes: m.bytes, + Samples: m.samples, + PeakRate: round(m.rPeak), + BytesRem: m.tBytes - m.bytes, + Progress: percentOf(float64(m.bytes), float64(m.tBytes)), + } + if s.BytesRem < 0 { + s.BytesRem = 0 + } + if s.Duration > 0 { + rAvg := float64(s.Bytes) / s.Duration.Seconds() + s.AvgRate = round(rAvg) + if s.Active { + s.InstRate = round(m.rSample) + s.CurRate = round(m.rEMA) + if s.BytesRem > 0 { + if tRate := 0.8*m.rEMA + 0.2*rAvg; tRate > 0 { + ns := float64(s.BytesRem) / tRate * 1e9 + if ns > float64(timeRemLimit) { + ns = float64(timeRemLimit) + } + s.TimeRem = clockRound(time.Duration(ns)) + } + } + } + } + m.mu.Unlock() + return s +} + +// Limit restricts the instantaneous (per-sample) data flow to rate bytes per +// second. It returns the maximum number of bytes (0 <= n <= want) that may be +// transferred immediately without exceeding the limit. If block == true, the +// call blocks until n > 0. want is returned unmodified if want < 1, rate < 1, +// or the transfer is inactive (after a call to Done). +// +// At least one byte is always allowed to be transferred in any given sampling +// period. Thus, if the sampling rate is 100ms, the lowest achievable flow rate +// is 10 bytes per second. +// +// For usage examples, see the implementation of Reader and Writer in io.go. +func (m *Monitor) Limit(want int, rate int64, block bool) (n int) { + if want < 1 || rate < 1 { + return want + } + m.mu.Lock() + + // Determine the maximum number of bytes that can be sent in one sample + limit := round(float64(rate) * m.sRate.Seconds()) + if limit <= 0 { + limit = 1 + } + + // If block == true, wait until m.sBytes < limit + if now := m.update(0); block { + for m.sBytes >= limit && m.active { + now = m.waitNextSample(now) + } + } + + // Make limit <= want (unlimited if the transfer is no longer active) + if limit -= m.sBytes; limit > int64(want) || !m.active { + limit = int64(want) + } + m.mu.Unlock() + + if limit < 0 { + limit = 0 + } + return int(limit) +} + +// SetTransferSize specifies the total size of the data transfer, which allows +// the Monitor to calculate the overall progress and time to completion. +func (m *Monitor) SetTransferSize(bytes int64) { + if bytes < 0 { + bytes = 0 + } + m.mu.Lock() + m.tBytes = bytes + m.mu.Unlock() +} + +// update accumulates the transferred byte count for the current sample until +// clock() - m.sLast >= m.sRate. The monitor status is updated once the current +// sample is done. +func (m *Monitor) update(n int) (now time.Duration) { + if !m.active { + return + } + if now = clock(); n > 0 { + m.tLast = now + } + m.sBytes += int64(n) + if sTime := now - m.sLast; sTime >= m.sRate { + t := sTime.Seconds() + if m.rSample = float64(m.sBytes) / t; m.rSample > m.rPeak { + m.rPeak = m.rSample + } + + // Exponential moving average using a method similar to *nix load + // average calculation. Longer sampling periods carry greater weight. + if m.samples > 0 { + w := math.Exp(-t / m.rWindow) + m.rEMA = m.rSample + w*(m.rEMA-m.rSample) + } else { + m.rEMA = m.rSample + } + m.reset(now) + } + return +} + +// reset clears the current sample state in preparation for the next sample. +func (m *Monitor) reset(sampleTime time.Duration) { + m.bytes += m.sBytes + m.samples++ + m.sBytes = 0 + m.sLast = sampleTime +} + +// waitNextSample sleeps for the remainder of the current sample. The lock is +// released and reacquired during the actual sleep period, so it's possible for +// the transfer to be inactive when this method returns. +func (m *Monitor) waitNextSample(now time.Duration) time.Duration { + const minWait = 5 * time.Millisecond + current := m.sLast + + // sleep until the last sample time changes (ideally, just one iteration) + for m.sLast == current && m.active { + d := current + m.sRate - now + m.mu.Unlock() + if d < minWait { + d = minWait + } + time.Sleep(d) + m.mu.Lock() + now = m.update(0) + } + return now +} diff --git a/libs/flowrate/io.go b/libs/flowrate/io.go new file mode 100644 index 000000000..fbe090972 --- /dev/null +++ b/libs/flowrate/io.go @@ -0,0 +1,133 @@ +// +// Written by Maxim Khitrov (November 2012) +// + +package flowrate + +import ( + "errors" + "io" +) + +// ErrLimit is returned by the Writer when a non-blocking write is short due to +// the transfer rate limit. +var ErrLimit = errors.New("flowrate: flow rate limit exceeded") + +// Limiter is implemented by the Reader and Writer to provide a consistent +// interface for monitoring and controlling data transfer. +type Limiter interface { + Done() int64 + Status() Status + SetTransferSize(bytes int64) + SetLimit(new int64) (old int64) + SetBlocking(new bool) (old bool) +} + +// Reader implements io.ReadCloser with a restriction on the rate of data +// transfer. +type Reader struct { + io.Reader // Data source + *Monitor // Flow control monitor + + limit int64 // Rate limit in bytes per second (unlimited when <= 0) + block bool // What to do when no new bytes can be read due to the limit +} + +// NewReader restricts all Read operations on r to limit bytes per second. +func NewReader(r io.Reader, limit int64) *Reader { + return &Reader{r, New(0, 0), limit, true} +} + +// Read reads up to len(p) bytes into p without exceeding the current transfer +// rate limit. It returns (0, nil) immediately if r is non-blocking and no new +// bytes can be read at this time. +func (r *Reader) Read(p []byte) (n int, err error) { + p = p[:r.Limit(len(p), r.limit, r.block)] + if len(p) > 0 { + n, err = r.IO(r.Reader.Read(p)) + } + return +} + +// SetLimit changes the transfer rate limit to new bytes per second and returns +// the previous setting. +func (r *Reader) SetLimit(new int64) (old int64) { + old, r.limit = r.limit, new + return +} + +// SetBlocking changes the blocking behavior and returns the previous setting. A +// Read call on a non-blocking reader returns immediately if no additional bytes +// may be read at this time due to the rate limit. +func (r *Reader) SetBlocking(new bool) (old bool) { + old, r.block = r.block, new + return +} + +// Close closes the underlying reader if it implements the io.Closer interface. +func (r *Reader) Close() error { + defer r.Done() + if c, ok := r.Reader.(io.Closer); ok { + return c.Close() + } + return nil +} + +// Writer implements io.WriteCloser with a restriction on the rate of data +// transfer. +type Writer struct { + io.Writer // Data destination + *Monitor // Flow control monitor + + limit int64 // Rate limit in bytes per second (unlimited when <= 0) + block bool // What to do when no new bytes can be written due to the limit +} + +// NewWriter restricts all Write operations on w to limit bytes per second. The +// transfer rate and the default blocking behavior (true) can be changed +// directly on the returned *Writer. +func NewWriter(w io.Writer, limit int64) *Writer { + return &Writer{w, New(0, 0), limit, true} +} + +// Write writes len(p) bytes from p to the underlying data stream without +// exceeding the current transfer rate limit. It returns (n, ErrLimit) if w is +// non-blocking and no additional bytes can be written at this time. +func (w *Writer) Write(p []byte) (n int, err error) { + var c int + for len(p) > 0 && err == nil { + s := p[:w.Limit(len(p), w.limit, w.block)] + if len(s) > 0 { + c, err = w.IO(w.Writer.Write(s)) + } else { + return n, ErrLimit + } + p = p[c:] + n += c + } + return +} + +// SetLimit changes the transfer rate limit to new bytes per second and returns +// the previous setting. +func (w *Writer) SetLimit(new int64) (old int64) { + old, w.limit = w.limit, new + return +} + +// SetBlocking changes the blocking behavior and returns the previous setting. A +// Write call on a non-blocking writer returns as soon as no additional bytes +// may be written at this time due to the rate limit. +func (w *Writer) SetBlocking(new bool) (old bool) { + old, w.block = w.block, new + return +} + +// Close closes the underlying writer if it implements the io.Closer interface. +func (w *Writer) Close() error { + defer w.Done() + if c, ok := w.Writer.(io.Closer); ok { + return c.Close() + } + return nil +} diff --git a/libs/flowrate/io_test.go b/libs/flowrate/io_test.go new file mode 100644 index 000000000..c84029d5e --- /dev/null +++ b/libs/flowrate/io_test.go @@ -0,0 +1,194 @@ +// +// Written by Maxim Khitrov (November 2012) +// + +package flowrate + +import ( + "bytes" + "testing" + "time" +) + +const ( + _50ms = 50 * time.Millisecond + _100ms = 100 * time.Millisecond + _200ms = 200 * time.Millisecond + _300ms = 300 * time.Millisecond + _400ms = 400 * time.Millisecond + _500ms = 500 * time.Millisecond +) + +func nextStatus(m *Monitor) Status { + samples := m.samples + for i := 0; i < 30; i++ { + if s := m.Status(); s.Samples != samples { + return s + } + time.Sleep(5 * time.Millisecond) + } + return m.Status() +} + +func TestReader(t *testing.T) { + in := make([]byte, 100) + for i := range in { + in[i] = byte(i) + } + b := make([]byte, 100) + r := NewReader(bytes.NewReader(in), 100) + start := time.Now() + + // Make sure r implements Limiter + _ = Limiter(r) + + // 1st read of 10 bytes is performed immediately + if n, err := r.Read(b); n != 10 || err != nil { + t.Fatalf("r.Read(b) expected 10 (); got %v (%v)", n, err) + } else if rt := time.Since(start); rt > _50ms { + t.Fatalf("r.Read(b) took too long (%v)", rt) + } + + // No new Reads allowed in the current sample + r.SetBlocking(false) + if n, err := r.Read(b); n != 0 || err != nil { + t.Fatalf("r.Read(b) expected 0 (); got %v (%v)", n, err) + } else if rt := time.Since(start); rt > _50ms { + t.Fatalf("r.Read(b) took too long (%v)", rt) + } + + status := [6]Status{0: r.Status()} // No samples in the first status + + // 2nd read of 10 bytes blocks until the next sample + r.SetBlocking(true) + if n, err := r.Read(b[10:]); n != 10 || err != nil { + t.Fatalf("r.Read(b[10:]) expected 10 (); got %v (%v)", n, err) + } else if rt := time.Since(start); rt < _100ms { + t.Fatalf("r.Read(b[10:]) returned ahead of time (%v)", rt) + } + + status[1] = r.Status() // 1st sample + status[2] = nextStatus(r.Monitor) // 2nd sample + status[3] = nextStatus(r.Monitor) // No activity for the 3rd sample + + if n := r.Done(); n != 20 { + t.Fatalf("r.Done() expected 20; got %v", n) + } + + status[4] = r.Status() + status[5] = nextStatus(r.Monitor) // Timeout + start = status[0].Start + + // Active, Start, Duration, Idle, Bytes, Samples, InstRate, CurRate, AvgRate, PeakRate, BytesRem, TimeRem, Progress + want := []Status{ + Status{true, start, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + Status{true, start, _100ms, 0, 10, 1, 100, 100, 100, 100, 0, 0, 0}, + Status{true, start, _200ms, _100ms, 20, 2, 100, 100, 100, 100, 0, 0, 0}, + Status{true, start, _300ms, _200ms, 20, 3, 0, 90, 67, 100, 0, 0, 0}, + Status{false, start, _300ms, 0, 20, 3, 0, 0, 67, 100, 0, 0, 0}, + Status{false, start, _300ms, 0, 20, 3, 0, 0, 67, 100, 0, 0, 0}, + } + for i, s := range status { + if !statusesAreEqual(&s, &want[i]) { + t.Errorf("r.Status(%v)\nexpected: %v\ngot : %v", i, want[i], s) + } + } + if !bytes.Equal(b[:20], in[:20]) { + t.Errorf("r.Read() input doesn't match output") + } +} + +func TestWriter(t *testing.T) { + b := make([]byte, 100) + for i := range b { + b[i] = byte(i) + } + w := NewWriter(&bytes.Buffer{}, 200) + start := time.Now() + + // Make sure w implements Limiter + _ = Limiter(w) + + // Non-blocking 20-byte write for the first sample returns ErrLimit + w.SetBlocking(false) + if n, err := w.Write(b); n != 20 || err != ErrLimit { + t.Fatalf("w.Write(b) expected 20 (ErrLimit); got %v (%v)", n, err) + } else if rt := time.Since(start); rt > _50ms { + t.Fatalf("w.Write(b) took too long (%v)", rt) + } + + // Blocking 80-byte write + w.SetBlocking(true) + if n, err := w.Write(b[20:]); n != 80 || err != nil { + t.Fatalf("w.Write(b[20:]) expected 80 (); got %v (%v)", n, err) + } else if rt := time.Since(start); rt < _300ms { + // Explanation for `rt < _300ms` (as opposed to `< _400ms`) + // + // |<-- start | | + // epochs: -----0ms|---100ms|---200ms|---300ms|---400ms + // sends: 20|20 |20 |20 |20# + // + // NOTE: The '#' symbol can thus happen before 400ms is up. + // Thus, we can only panic if rt < _300ms. + t.Fatalf("w.Write(b[20:]) returned ahead of time (%v)", rt) + } + + w.SetTransferSize(100) + status := []Status{w.Status(), nextStatus(w.Monitor)} + start = status[0].Start + + // Active, Start, Duration, Idle, Bytes, Samples, InstRate, CurRate, AvgRate, PeakRate, BytesRem, TimeRem, Progress + want := []Status{ + Status{true, start, _400ms, 0, 80, 4, 200, 200, 200, 200, 20, _100ms, 80000}, + Status{true, start, _500ms, _100ms, 100, 5, 200, 200, 200, 200, 0, 0, 100000}, + } + for i, s := range status { + if !statusesAreEqual(&s, &want[i]) { + t.Errorf("w.Status(%v)\nexpected: %v\ngot : %v\n", i, want[i], s) + } + } + if !bytes.Equal(b, w.Writer.(*bytes.Buffer).Bytes()) { + t.Errorf("w.Write() input doesn't match output") + } +} + +const maxDeviationForDuration = 50 * time.Millisecond +const maxDeviationForRate int64 = 50 + +// statusesAreEqual returns true if s1 is equal to s2. Equality here means +// general equality of fields except for the duration and rates, which can +// drift due to unpredictable delays (e.g. thread wakes up 25ms after +// `time.Sleep` has ended). +func statusesAreEqual(s1 *Status, s2 *Status) bool { + if s1.Active == s2.Active && + s1.Start == s2.Start && + durationsAreEqual(s1.Duration, s2.Duration, maxDeviationForDuration) && + s1.Idle == s2.Idle && + s1.Bytes == s2.Bytes && + s1.Samples == s2.Samples && + ratesAreEqual(s1.InstRate, s2.InstRate, maxDeviationForRate) && + ratesAreEqual(s1.CurRate, s2.CurRate, maxDeviationForRate) && + ratesAreEqual(s1.AvgRate, s2.AvgRate, maxDeviationForRate) && + ratesAreEqual(s1.PeakRate, s2.PeakRate, maxDeviationForRate) && + s1.BytesRem == s2.BytesRem && + durationsAreEqual(s1.TimeRem, s2.TimeRem, maxDeviationForDuration) && + s1.Progress == s2.Progress { + return true + } + return false +} + +func durationsAreEqual(d1 time.Duration, d2 time.Duration, maxDeviation time.Duration) bool { + return d2-d1 <= maxDeviation +} + +func ratesAreEqual(r1 int64, r2 int64, maxDeviation int64) bool { + sub := r1 - r2 + if sub < 0 { + sub = -sub + } + if sub <= maxDeviation { + return true + } + return false +} diff --git a/libs/flowrate/util.go b/libs/flowrate/util.go new file mode 100644 index 000000000..b33ddc701 --- /dev/null +++ b/libs/flowrate/util.go @@ -0,0 +1,67 @@ +// +// Written by Maxim Khitrov (November 2012) +// + +package flowrate + +import ( + "math" + "strconv" + "time" +) + +// clockRate is the resolution and precision of clock(). +const clockRate = 20 * time.Millisecond + +// czero is the process start time rounded down to the nearest clockRate +// increment. +var czero = time.Now().Round(clockRate) + +// clock returns a low resolution timestamp relative to the process start time. +func clock() time.Duration { + return time.Now().Round(clockRate).Sub(czero) +} + +// clockToTime converts a clock() timestamp to an absolute time.Time value. +func clockToTime(c time.Duration) time.Time { + return czero.Add(c) +} + +// clockRound returns d rounded to the nearest clockRate increment. +func clockRound(d time.Duration) time.Duration { + return (d + clockRate>>1) / clockRate * clockRate +} + +// round returns x rounded to the nearest int64 (non-negative values only). +func round(x float64) int64 { + if _, frac := math.Modf(x); frac >= 0.5 { + return int64(math.Ceil(x)) + } + return int64(math.Floor(x)) +} + +// Percent represents a percentage in increments of 1/1000th of a percent. +type Percent uint32 + +// percentOf calculates what percent of the total is x. +func percentOf(x, total float64) Percent { + if x < 0 || total <= 0 { + return 0 + } else if p := round(x / total * 1e5); p <= math.MaxUint32 { + return Percent(p) + } + return Percent(math.MaxUint32) +} + +func (p Percent) Float() float64 { + return float64(p) * 1e-3 +} + +func (p Percent) String() string { + var buf [12]byte + b := strconv.AppendUint(buf[:0], uint64(p)/1000, 10) + n := len(b) + b = strconv.AppendUint(b, 1000+uint64(p)%1000, 10) + b[n] = '.' + return string(append(b, '%')) +} diff --git a/libs/glide.lock b/libs/glide.lock new file mode 100644 index 000000000..a0ada5a4a --- /dev/null +++ b/libs/glide.lock @@ -0,0 +1,108 @@ +hash: 98752078f39da926f655268b3b143f713d64edd379fc9fcb1210d9d8aa7ab4e0 +updated: 2018-02-03T01:28:00.221548057-05:00 +imports: +- name: github.com/fsnotify/fsnotify + version: c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9 +- name: github.com/go-kit/kit + version: 4dc7be5d2d12881735283bcab7352178e190fc71 + subpackages: + - log + - log/level + - log/term +- name: github.com/go-logfmt/logfmt + version: 390ab7935ee28ec6b286364bba9b4dd6410cb3d5 +- name: github.com/go-stack/stack + version: 817915b46b97fd7bb80e8ab6b69f01a53ac3eebf +- name: github.com/gogo/protobuf + version: 1adfc126b41513cc696b209667c8656ea7aac67c + subpackages: + - gogoproto + - proto + - protoc-gen-gogo/descriptor +- name: github.com/golang/snappy + version: 553a641470496b2327abcac10b36396bd98e45c9 +- name: github.com/hashicorp/hcl + version: 23c074d0eceb2b8a5bfdbb271ab780cde70f05a8 + subpackages: + - hcl/ast + - hcl/parser + - hcl/scanner + - hcl/strconv + - hcl/token + - json/parser + - json/scanner + - json/token +- name: github.com/inconshreveable/mousetrap + version: 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75 +- name: github.com/jmhodges/levigo + version: c42d9e0ca023e2198120196f842701bb4c55d7b9 +- name: github.com/kr/logfmt + version: b84e30acd515aadc4b783ad4ff83aff3299bdfe0 +- name: github.com/magiconair/properties + version: 49d762b9817ba1c2e9d0c69183c2b4a8b8f1d934 +- name: github.com/mitchellh/mapstructure + version: b4575eea38cca1123ec2dc90c26529b5c5acfcff +- name: github.com/pelletier/go-toml + version: acdc4509485b587f5e675510c4f2c63e90ff68a8 +- name: github.com/pkg/errors + version: 645ef00459ed84a119197bfb8d8205042c6df63d +- name: github.com/spf13/afero + version: bb8f1927f2a9d3ab41c9340aa034f6b803f4359c + subpackages: + - mem +- name: github.com/spf13/cast + version: acbeb36b902d72a7a4c18e8f3241075e7ab763e4 +- name: github.com/spf13/cobra + version: 7b2c5ac9fc04fc5efafb60700713d4fa609b777b +- name: github.com/spf13/jwalterweatherman + version: 7c0cea34c8ece3fbeb2b27ab9b59511d360fb394 +- name: github.com/spf13/pflag + version: 97afa5e7ca8a08a383cb259e06636b5e2cc7897f +- name: github.com/spf13/viper + version: 25b30aa063fc18e48662b86996252eabdcf2f0c7 +- name: github.com/syndtr/goleveldb + version: b89cc31ef7977104127d34c1bd31ebd1a9db2199 + subpackages: + - leveldb + - leveldb/cache + - leveldb/comparer + - leveldb/errors + - leveldb/filter + - leveldb/iterator + - leveldb/journal + - leveldb/memdb + - leveldb/opt + - leveldb/storage + - leveldb/table + - leveldb/util +- name: golang.org/x/crypto + version: edd5e9b0879d13ee6970a50153d85b8fec9f7686 + subpackages: + - ripemd160 +- name: golang.org/x/sys + version: 37707fdb30a5b38865cfb95e5aab41707daec7fd + subpackages: + - unix +- name: golang.org/x/text + version: c01e4764d870b77f8abe5096ee19ad20d80e8075 + subpackages: + - transform + - unicode/norm +- name: gopkg.in/yaml.v2 + version: d670f9405373e636a5a2765eea47fac0c9bc91a4 +testImports: +- name: github.com/davecgh/go-spew + version: 346938d642f2ec3594ed81d874461961cd0faa76 + subpackages: + - spew +- name: github.com/fortytw2/leaktest + version: 3b724c3d7b8729a35bf4e577f71653aec6e53513 +- name: github.com/pmezard/go-difflib + version: d8ed2627bdf02c080bf22230dbb337003b7aba2d + subpackages: + - difflib +- name: github.com/stretchr/testify + version: 12b6f73e6084dad08a7c6e575284b177ecafbc71 + subpackages: + - assert + - require diff --git a/libs/glide.yaml b/libs/glide.yaml new file mode 100644 index 000000000..cf3da346b --- /dev/null +++ b/libs/glide.yaml @@ -0,0 +1,38 @@ +package: github.com/tendermint/tmlibs +import: +- package: github.com/go-kit/kit + version: ^0.6.0 + subpackages: + - log + - log/level + - log/term +- package: github.com/go-logfmt/logfmt + version: ^0.3.0 +- package: github.com/gogo/protobuf + version: ^1.0.0 + subpackages: + - gogoproto + - proto +- package: github.com/jmhodges/levigo +- package: github.com/pkg/errors + version: ^0.8.0 +- package: github.com/spf13/cobra + version: ^0.0.1 +- package: github.com/spf13/viper + version: ^1.0.0 +- package: github.com/syndtr/goleveldb + subpackages: + - leveldb + - leveldb/errors + - leveldb/iterator + - leveldb/opt +- package: golang.org/x/crypto + subpackages: + - ripemd160 +testImport: +- package: github.com/stretchr/testify + version: ^1.2.1 + subpackages: + - assert + - require +- package: github.com/fortytw2/leaktest diff --git a/libs/log/filter.go b/libs/log/filter.go new file mode 100644 index 000000000..768c09b85 --- /dev/null +++ b/libs/log/filter.go @@ -0,0 +1,158 @@ +package log + +import "fmt" + +type level byte + +const ( + levelDebug level = 1 << iota + levelInfo + levelError +) + +type filter struct { + next Logger + allowed level // XOR'd levels for default case + allowedKeyvals map[keyval]level // When key-value match, use this level +} + +type keyval struct { + key interface{} + value interface{} +} + +// NewFilter wraps next and implements filtering. See the commentary on the +// Option functions for a detailed description of how to configure levels. If +// no options are provided, all leveled log events created with Debug, Info or +// Error helper methods are squelched. +func NewFilter(next Logger, options ...Option) Logger { + l := &filter{ + next: next, + allowedKeyvals: make(map[keyval]level), + } + for _, option := range options { + option(l) + } + return l +} + +func (l *filter) Info(msg string, keyvals ...interface{}) { + levelAllowed := l.allowed&levelInfo != 0 + if !levelAllowed { + return + } + l.next.Info(msg, keyvals...) +} + +func (l *filter) Debug(msg string, keyvals ...interface{}) { + levelAllowed := l.allowed&levelDebug != 0 + if !levelAllowed { + return + } + l.next.Debug(msg, keyvals...) +} + +func (l *filter) Error(msg string, keyvals ...interface{}) { + levelAllowed := l.allowed&levelError != 0 + if !levelAllowed { + return + } + l.next.Error(msg, keyvals...) +} + +// With implements Logger by constructing a new filter with a keyvals appended +// to the logger. +// +// If custom level was set for a keyval pair using one of the +// Allow*With methods, it is used as the logger's level. +// +// Examples: +// logger = log.NewFilter(logger, log.AllowError(), log.AllowInfoWith("module", "crypto")) +// logger.With("module", "crypto").Info("Hello") # produces "I... Hello module=crypto" +// +// logger = log.NewFilter(logger, log.AllowError(), log.AllowInfoWith("module", "crypto"), log.AllowNoneWith("user", "Sam")) +// logger.With("module", "crypto", "user", "Sam").Info("Hello") # returns nil +// +// logger = log.NewFilter(logger, log.AllowError(), log.AllowInfoWith("module", "crypto"), log.AllowNoneWith("user", "Sam")) +// logger.With("user", "Sam").With("module", "crypto").Info("Hello") # produces "I... Hello module=crypto user=Sam" +func (l *filter) With(keyvals ...interface{}) Logger { + for i := len(keyvals) - 2; i >= 0; i -= 2 { + for kv, allowed := range l.allowedKeyvals { + if keyvals[i] == kv.key && keyvals[i+1] == kv.value { + return &filter{next: l.next.With(keyvals...), allowed: allowed, allowedKeyvals: l.allowedKeyvals} + } + } + } + return &filter{next: l.next.With(keyvals...), allowed: l.allowed, allowedKeyvals: l.allowedKeyvals} +} + +//-------------------------------------------------------------------------------- + +// Option sets a parameter for the filter. +type Option func(*filter) + +// AllowLevel returns an option for the given level or error if no option exist +// for such level. +func AllowLevel(lvl string) (Option, error) { + switch lvl { + case "debug": + return AllowDebug(), nil + case "info": + return AllowInfo(), nil + case "error": + return AllowError(), nil + case "none": + return AllowNone(), nil + default: + return nil, fmt.Errorf("Expected either \"info\", \"debug\", \"error\" or \"none\" level, given %s", lvl) + } +} + +// AllowAll is an alias for AllowDebug. +func AllowAll() Option { + return AllowDebug() +} + +// AllowDebug allows error, info and debug level log events to pass. +func AllowDebug() Option { + return allowed(levelError | levelInfo | levelDebug) +} + +// AllowInfo allows error and info level log events to pass. +func AllowInfo() Option { + return allowed(levelError | levelInfo) +} + +// AllowError allows only error level log events to pass. +func AllowError() Option { + return allowed(levelError) +} + +// AllowNone allows no leveled log events to pass. +func AllowNone() Option { + return allowed(0) +} + +func allowed(allowed level) Option { + return func(l *filter) { l.allowed = allowed } +} + +// AllowDebugWith allows error, info and debug level log events to pass for a specific key value pair. +func AllowDebugWith(key interface{}, value interface{}) Option { + return func(l *filter) { l.allowedKeyvals[keyval{key, value}] = levelError | levelInfo | levelDebug } +} + +// AllowInfoWith allows error and info level log events to pass for a specific key value pair. +func AllowInfoWith(key interface{}, value interface{}) Option { + return func(l *filter) { l.allowedKeyvals[keyval{key, value}] = levelError | levelInfo } +} + +// AllowErrorWith allows only error level log events to pass for a specific key value pair. +func AllowErrorWith(key interface{}, value interface{}) Option { + return func(l *filter) { l.allowedKeyvals[keyval{key, value}] = levelError } +} + +// AllowNoneWith allows no leveled log events to pass for a specific key value pair. +func AllowNoneWith(key interface{}, value interface{}) Option { + return func(l *filter) { l.allowedKeyvals[keyval{key, value}] = 0 } +} diff --git a/libs/log/filter_test.go b/libs/log/filter_test.go new file mode 100644 index 000000000..f9957f043 --- /dev/null +++ b/libs/log/filter_test.go @@ -0,0 +1,118 @@ +package log_test + +import ( + "bytes" + "strings" + "testing" + + "github.com/tendermint/tendermint/libs/log" +) + +func TestVariousLevels(t *testing.T) { + testCases := []struct { + name string + allowed log.Option + want string + }{ + { + "AllowAll", + log.AllowAll(), + strings.Join([]string{ + `{"_msg":"here","level":"debug","this is":"debug log"}`, + `{"_msg":"here","level":"info","this is":"info log"}`, + `{"_msg":"here","level":"error","this is":"error log"}`, + }, "\n"), + }, + { + "AllowDebug", + log.AllowDebug(), + strings.Join([]string{ + `{"_msg":"here","level":"debug","this is":"debug log"}`, + `{"_msg":"here","level":"info","this is":"info log"}`, + `{"_msg":"here","level":"error","this is":"error log"}`, + }, "\n"), + }, + { + "AllowInfo", + log.AllowInfo(), + strings.Join([]string{ + `{"_msg":"here","level":"info","this is":"info log"}`, + `{"_msg":"here","level":"error","this is":"error log"}`, + }, "\n"), + }, + { + "AllowError", + log.AllowError(), + strings.Join([]string{ + `{"_msg":"here","level":"error","this is":"error log"}`, + }, "\n"), + }, + { + "AllowNone", + log.AllowNone(), + ``, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var buf bytes.Buffer + logger := log.NewFilter(log.NewTMJSONLogger(&buf), tc.allowed) + + logger.Debug("here", "this is", "debug log") + logger.Info("here", "this is", "info log") + logger.Error("here", "this is", "error log") + + if want, have := tc.want, strings.TrimSpace(buf.String()); want != have { + t.Errorf("\nwant:\n%s\nhave:\n%s", want, have) + } + }) + } +} + +func TestLevelContext(t *testing.T) { + var buf bytes.Buffer + + logger := log.NewTMJSONLogger(&buf) + logger = log.NewFilter(logger, log.AllowError()) + logger = logger.With("context", "value") + + logger.Error("foo", "bar", "baz") + if want, have := `{"_msg":"foo","bar":"baz","context":"value","level":"error"}`, strings.TrimSpace(buf.String()); want != have { + t.Errorf("\nwant '%s'\nhave '%s'", want, have) + } + + buf.Reset() + logger.Info("foo", "bar", "baz") + if want, have := ``, strings.TrimSpace(buf.String()); want != have { + t.Errorf("\nwant '%s'\nhave '%s'", want, have) + } +} + +func TestVariousAllowWith(t *testing.T) { + var buf bytes.Buffer + + logger := log.NewTMJSONLogger(&buf) + + logger1 := log.NewFilter(logger, log.AllowError(), log.AllowInfoWith("context", "value")) + logger1.With("context", "value").Info("foo", "bar", "baz") + if want, have := `{"_msg":"foo","bar":"baz","context":"value","level":"info"}`, strings.TrimSpace(buf.String()); want != have { + t.Errorf("\nwant '%s'\nhave '%s'", want, have) + } + + buf.Reset() + + logger2 := log.NewFilter(logger, log.AllowError(), log.AllowInfoWith("context", "value"), log.AllowNoneWith("user", "Sam")) + logger2.With("context", "value", "user", "Sam").Info("foo", "bar", "baz") + if want, have := ``, strings.TrimSpace(buf.String()); want != have { + t.Errorf("\nwant '%s'\nhave '%s'", want, have) + } + + buf.Reset() + + logger3 := log.NewFilter(logger, log.AllowError(), log.AllowInfoWith("context", "value"), log.AllowNoneWith("user", "Sam")) + logger3.With("user", "Sam").With("context", "value").Info("foo", "bar", "baz") + if want, have := `{"_msg":"foo","bar":"baz","context":"value","level":"info","user":"Sam"}`, strings.TrimSpace(buf.String()); want != have { + t.Errorf("\nwant '%s'\nhave '%s'", want, have) + } +} diff --git a/libs/log/logger.go b/libs/log/logger.go new file mode 100644 index 000000000..ddb187bc7 --- /dev/null +++ b/libs/log/logger.go @@ -0,0 +1,30 @@ +package log + +import ( + "io" + + kitlog "github.com/go-kit/kit/log" +) + +// Logger is what any Tendermint library should take. +type Logger interface { + Debug(msg string, keyvals ...interface{}) + Info(msg string, keyvals ...interface{}) + Error(msg string, keyvals ...interface{}) + + With(keyvals ...interface{}) Logger +} + +// NewSyncWriter returns a new writer that is safe for concurrent use by +// multiple goroutines. Writes to the returned writer are passed on to w. If +// another write is already in progress, the calling goroutine blocks until +// the writer is available. +// +// If w implements the following interface, so does the returned writer. +// +// interface { +// Fd() uintptr +// } +func NewSyncWriter(w io.Writer) io.Writer { + return kitlog.NewSyncWriter(w) +} diff --git a/libs/log/nop_logger.go b/libs/log/nop_logger.go new file mode 100644 index 000000000..12d75abe6 --- /dev/null +++ b/libs/log/nop_logger.go @@ -0,0 +1,17 @@ +package log + +type nopLogger struct{} + +// Interface assertions +var _ Logger = (*nopLogger)(nil) + +// NewNopLogger returns a logger that doesn't do anything. +func NewNopLogger() Logger { return &nopLogger{} } + +func (nopLogger) Info(string, ...interface{}) {} +func (nopLogger) Debug(string, ...interface{}) {} +func (nopLogger) Error(string, ...interface{}) {} + +func (l *nopLogger) With(...interface{}) Logger { + return l +} diff --git a/libs/log/testing_logger.go b/libs/log/testing_logger.go new file mode 100644 index 000000000..81482bef5 --- /dev/null +++ b/libs/log/testing_logger.go @@ -0,0 +1,49 @@ +package log + +import ( + "os" + "testing" + + "github.com/go-kit/kit/log/term" +) + +var ( + // reuse the same logger across all tests + _testingLogger Logger +) + +// TestingLogger returns a TMLogger which writes to STDOUT if testing being run +// with the verbose (-v) flag, NopLogger otherwise. +// +// Note that the call to TestingLogger() must be made +// inside a test (not in the init func) because +// verbose flag only set at the time of testing. +func TestingLogger() Logger { + if _testingLogger != nil { + return _testingLogger + } + + if testing.Verbose() { + _testingLogger = NewTMLogger(NewSyncWriter(os.Stdout)) + } else { + _testingLogger = NewNopLogger() + } + + return _testingLogger +} + +// TestingLoggerWithColorFn allow you to provide your own color function. See +// TestingLogger for documentation. +func TestingLoggerWithColorFn(colorFn func(keyvals ...interface{}) term.FgBgColor) Logger { + if _testingLogger != nil { + return _testingLogger + } + + if testing.Verbose() { + _testingLogger = NewTMLoggerWithColorFn(NewSyncWriter(os.Stdout), colorFn) + } else { + _testingLogger = NewNopLogger() + } + + return _testingLogger +} diff --git a/libs/log/tm_json_logger.go b/libs/log/tm_json_logger.go new file mode 100644 index 000000000..a71ac1034 --- /dev/null +++ b/libs/log/tm_json_logger.go @@ -0,0 +1,15 @@ +package log + +import ( + "io" + + kitlog "github.com/go-kit/kit/log" +) + +// NewTMJSONLogger returns a Logger that encodes keyvals to the Writer as a +// single JSON object. Each log event produces no more than one call to +// w.Write. The passed Writer must be safe for concurrent use by multiple +// goroutines if the returned Logger will be used concurrently. +func NewTMJSONLogger(w io.Writer) Logger { + return &tmLogger{kitlog.NewJSONLogger(w)} +} diff --git a/libs/log/tm_logger.go b/libs/log/tm_logger.go new file mode 100644 index 000000000..d49e8d22b --- /dev/null +++ b/libs/log/tm_logger.go @@ -0,0 +1,83 @@ +package log + +import ( + "fmt" + "io" + + kitlog "github.com/go-kit/kit/log" + kitlevel "github.com/go-kit/kit/log/level" + "github.com/go-kit/kit/log/term" +) + +const ( + msgKey = "_msg" // "_" prefixed to avoid collisions + moduleKey = "module" +) + +type tmLogger struct { + srcLogger kitlog.Logger +} + +// Interface assertions +var _ Logger = (*tmLogger)(nil) + +// NewTMTermLogger returns a logger that encodes msg and keyvals to the Writer +// using go-kit's log as an underlying logger and our custom formatter. Note +// that underlying logger could be swapped with something else. +func NewTMLogger(w io.Writer) Logger { + // Color by level value + colorFn := func(keyvals ...interface{}) term.FgBgColor { + if keyvals[0] != kitlevel.Key() { + panic(fmt.Sprintf("expected level key to be first, got %v", keyvals[0])) + } + switch keyvals[1].(kitlevel.Value).String() { + case "debug": + return term.FgBgColor{Fg: term.DarkGray} + case "error": + return term.FgBgColor{Fg: term.Red} + default: + return term.FgBgColor{} + } + } + + return &tmLogger{term.NewLogger(w, NewTMFmtLogger, colorFn)} +} + +// NewTMLoggerWithColorFn allows you to provide your own color function. See +// NewTMLogger for documentation. +func NewTMLoggerWithColorFn(w io.Writer, colorFn func(keyvals ...interface{}) term.FgBgColor) Logger { + return &tmLogger{term.NewLogger(w, NewTMFmtLogger, colorFn)} +} + +// Info logs a message at level Info. +func (l *tmLogger) Info(msg string, keyvals ...interface{}) { + lWithLevel := kitlevel.Info(l.srcLogger) + if err := kitlog.With(lWithLevel, msgKey, msg).Log(keyvals...); err != nil { + errLogger := kitlevel.Error(l.srcLogger) + kitlog.With(errLogger, msgKey, msg).Log("err", err) + } +} + +// Debug logs a message at level Debug. +func (l *tmLogger) Debug(msg string, keyvals ...interface{}) { + lWithLevel := kitlevel.Debug(l.srcLogger) + if err := kitlog.With(lWithLevel, msgKey, msg).Log(keyvals...); err != nil { + errLogger := kitlevel.Error(l.srcLogger) + kitlog.With(errLogger, msgKey, msg).Log("err", err) + } +} + +// Error logs a message at level Error. +func (l *tmLogger) Error(msg string, keyvals ...interface{}) { + lWithLevel := kitlevel.Error(l.srcLogger) + lWithMsg := kitlog.With(lWithLevel, msgKey, msg) + if err := lWithMsg.Log(keyvals...); err != nil { + lWithMsg.Log("err", err) + } +} + +// With returns a new contextual logger with keyvals prepended to those passed +// to calls to Info, Debug or Error. +func (l *tmLogger) With(keyvals ...interface{}) Logger { + return &tmLogger{kitlog.With(l.srcLogger, keyvals...)} +} diff --git a/libs/log/tm_logger_test.go b/libs/log/tm_logger_test.go new file mode 100644 index 000000000..1f890cef1 --- /dev/null +++ b/libs/log/tm_logger_test.go @@ -0,0 +1,44 @@ +package log_test + +import ( + "bytes" + "io/ioutil" + "strings" + "testing" + + "github.com/go-logfmt/logfmt" + "github.com/tendermint/tendermint/libs/log" +) + +func TestLoggerLogsItsErrors(t *testing.T) { + var buf bytes.Buffer + + logger := log.NewTMLogger(&buf) + logger.Info("foo", "baz baz", "bar") + msg := strings.TrimSpace(buf.String()) + if !strings.Contains(msg, logfmt.ErrInvalidKey.Error()) { + t.Errorf("Expected logger msg to contain ErrInvalidKey, got %s", msg) + } +} + +func BenchmarkTMLoggerSimple(b *testing.B) { + benchmarkRunner(b, log.NewTMLogger(ioutil.Discard), baseInfoMessage) +} + +func BenchmarkTMLoggerContextual(b *testing.B) { + benchmarkRunner(b, log.NewTMLogger(ioutil.Discard), withInfoMessage) +} + +func benchmarkRunner(b *testing.B, logger log.Logger, f func(log.Logger)) { + lc := logger.With("common_key", "common_value") + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + f(lc) + } +} + +var ( + baseInfoMessage = func(logger log.Logger) { logger.Info("foo_message", "foo_key", "foo_value") } + withInfoMessage = func(logger log.Logger) { logger.With("a", "b").Info("c", "d", "f") } +) diff --git a/libs/log/tmfmt_logger.go b/libs/log/tmfmt_logger.go new file mode 100644 index 000000000..d03979718 --- /dev/null +++ b/libs/log/tmfmt_logger.go @@ -0,0 +1,127 @@ +package log + +import ( + "bytes" + "fmt" + "io" + "sync" + "time" + + kitlog "github.com/go-kit/kit/log" + kitlevel "github.com/go-kit/kit/log/level" + "github.com/go-logfmt/logfmt" +) + +type tmfmtEncoder struct { + *logfmt.Encoder + buf bytes.Buffer +} + +func (l *tmfmtEncoder) Reset() { + l.Encoder.Reset() + l.buf.Reset() +} + +var tmfmtEncoderPool = sync.Pool{ + New: func() interface{} { + var enc tmfmtEncoder + enc.Encoder = logfmt.NewEncoder(&enc.buf) + return &enc + }, +} + +type tmfmtLogger struct { + w io.Writer +} + +// NewTMFmtLogger returns a logger that encodes keyvals to the Writer in +// Tendermint custom format. Note complex types (structs, maps, slices) +// formatted as "%+v". +// +// Each log event produces no more than one call to w.Write. +// The passed Writer must be safe for concurrent use by multiple goroutines if +// the returned Logger will be used concurrently. +func NewTMFmtLogger(w io.Writer) kitlog.Logger { + return &tmfmtLogger{w} +} + +func (l tmfmtLogger) Log(keyvals ...interface{}) error { + enc := tmfmtEncoderPool.Get().(*tmfmtEncoder) + enc.Reset() + defer tmfmtEncoderPool.Put(enc) + + const unknown = "unknown" + lvl := "none" + msg := unknown + module := unknown + + // indexes of keys to skip while encoding later + excludeIndexes := make([]int, 0) + + for i := 0; i < len(keyvals)-1; i += 2 { + // Extract level + if keyvals[i] == kitlevel.Key() { + excludeIndexes = append(excludeIndexes, i) + switch keyvals[i+1].(type) { + case string: + lvl = keyvals[i+1].(string) + case kitlevel.Value: + lvl = keyvals[i+1].(kitlevel.Value).String() + default: + panic(fmt.Sprintf("level value of unknown type %T", keyvals[i+1])) + } + // and message + } else if keyvals[i] == msgKey { + excludeIndexes = append(excludeIndexes, i) + msg = keyvals[i+1].(string) + // and module (could be multiple keyvals; if such case last keyvalue wins) + } else if keyvals[i] == moduleKey { + excludeIndexes = append(excludeIndexes, i) + module = keyvals[i+1].(string) + } + } + + // Form a custom Tendermint line + // + // Example: + // D[05-02|11:06:44.322] Stopping AddrBook (ignoring: already stopped) + // + // Description: + // D - first character of the level, uppercase (ASCII only) + // [05-02|11:06:44.322] - our time format (see https://golang.org/src/time/format.go) + // Stopping ... - message + enc.buf.WriteString(fmt.Sprintf("%c[%s] %-44s ", lvl[0]-32, time.Now().UTC().Format("01-02|15:04:05.000"), msg)) + + if module != unknown { + enc.buf.WriteString("module=" + module + " ") + } + +KeyvalueLoop: + for i := 0; i < len(keyvals)-1; i += 2 { + for _, j := range excludeIndexes { + if i == j { + continue KeyvalueLoop + } + } + + err := enc.EncodeKeyval(keyvals[i], keyvals[i+1]) + if err == logfmt.ErrUnsupportedValueType { + enc.EncodeKeyval(keyvals[i], fmt.Sprintf("%+v", keyvals[i+1])) + } else if err != nil { + return err + } + } + + // Add newline to the end of the buffer + if err := enc.EndRecord(); err != nil { + return err + } + + // The Logger interface requires implementations to be safe for concurrent + // use by multiple goroutines. For this implementation that means making + // only one call to l.w.Write() for each call to Log. + if _, err := l.w.Write(enc.buf.Bytes()); err != nil { + return err + } + return nil +} diff --git a/libs/log/tmfmt_logger_test.go b/libs/log/tmfmt_logger_test.go new file mode 100644 index 000000000..d6f039ce4 --- /dev/null +++ b/libs/log/tmfmt_logger_test.go @@ -0,0 +1,118 @@ +package log_test + +import ( + "bytes" + "errors" + "io/ioutil" + "math" + "regexp" + "testing" + + kitlog "github.com/go-kit/kit/log" + "github.com/stretchr/testify/assert" + "github.com/tendermint/tendermint/libs/log" +) + +func TestTMFmtLogger(t *testing.T) { + t.Parallel() + buf := &bytes.Buffer{} + logger := log.NewTMFmtLogger(buf) + + if err := logger.Log("hello", "world"); err != nil { + t.Fatal(err) + } + assert.Regexp(t, regexp.MustCompile(`N\[.+\] unknown \s+ hello=world\n$`), buf.String()) + + buf.Reset() + if err := logger.Log("a", 1, "err", errors.New("error")); err != nil { + t.Fatal(err) + } + assert.Regexp(t, regexp.MustCompile(`N\[.+\] unknown \s+ a=1 err=error\n$`), buf.String()) + + buf.Reset() + if err := logger.Log("std_map", map[int]int{1: 2}, "my_map", mymap{0: 0}); err != nil { + t.Fatal(err) + } + assert.Regexp(t, regexp.MustCompile(`N\[.+\] unknown \s+ std_map=map\[1:2\] my_map=special_behavior\n$`), buf.String()) + + buf.Reset() + if err := logger.Log("level", "error"); err != nil { + t.Fatal(err) + } + assert.Regexp(t, regexp.MustCompile(`E\[.+\] unknown \s+\n$`), buf.String()) + + buf.Reset() + if err := logger.Log("_msg", "Hello"); err != nil { + t.Fatal(err) + } + assert.Regexp(t, regexp.MustCompile(`N\[.+\] Hello \s+\n$`), buf.String()) + + buf.Reset() + if err := logger.Log("module", "main", "module", "crypto", "module", "wire"); err != nil { + t.Fatal(err) + } + assert.Regexp(t, regexp.MustCompile(`N\[.+\] unknown \s+module=wire\s+\n$`), buf.String()) +} + +func BenchmarkTMFmtLoggerSimple(b *testing.B) { + benchmarkRunnerKitlog(b, log.NewTMFmtLogger(ioutil.Discard), baseMessage) +} + +func BenchmarkTMFmtLoggerContextual(b *testing.B) { + benchmarkRunnerKitlog(b, log.NewTMFmtLogger(ioutil.Discard), withMessage) +} + +func TestTMFmtLoggerConcurrency(t *testing.T) { + t.Parallel() + testConcurrency(t, log.NewTMFmtLogger(ioutil.Discard), 10000) +} + +func benchmarkRunnerKitlog(b *testing.B, logger kitlog.Logger, f func(kitlog.Logger)) { + lc := kitlog.With(logger, "common_key", "common_value") + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + f(lc) + } +} + +var ( + baseMessage = func(logger kitlog.Logger) { logger.Log("foo_key", "foo_value") } + withMessage = func(logger kitlog.Logger) { kitlog.With(logger, "a", "b").Log("d", "f") } +) + +// These test are designed to be run with the race detector. + +func testConcurrency(t *testing.T, logger kitlog.Logger, total int) { + n := int(math.Sqrt(float64(total))) + share := total / n + + errC := make(chan error, n) + + for i := 0; i < n; i++ { + go func() { + errC <- spam(logger, share) + }() + } + + for i := 0; i < n; i++ { + err := <-errC + if err != nil { + t.Fatalf("concurrent logging error: %v", err) + } + } +} + +func spam(logger kitlog.Logger, count int) error { + for i := 0; i < count; i++ { + err := logger.Log("key", i) + if err != nil { + return err + } + } + return nil +} + +type mymap map[int]int + +func (m mymap) String() string { return "special_behavior" } diff --git a/libs/log/tracing_logger.go b/libs/log/tracing_logger.go new file mode 100644 index 000000000..d2a6ff44e --- /dev/null +++ b/libs/log/tracing_logger.go @@ -0,0 +1,76 @@ +package log + +import ( + "fmt" + + "github.com/pkg/errors" +) + +// NewTracingLogger enables tracing by wrapping all errors (if they +// implement stackTracer interface) in tracedError. +// +// All errors returned by https://github.com/pkg/errors implement stackTracer +// interface. +// +// For debugging purposes only as it doubles the amount of allocations. +func NewTracingLogger(next Logger) Logger { + return &tracingLogger{ + next: next, + } +} + +type stackTracer interface { + error + StackTrace() errors.StackTrace +} + +type tracingLogger struct { + next Logger +} + +func (l *tracingLogger) Info(msg string, keyvals ...interface{}) { + l.next.Info(msg, formatErrors(keyvals)...) +} + +func (l *tracingLogger) Debug(msg string, keyvals ...interface{}) { + l.next.Debug(msg, formatErrors(keyvals)...) +} + +func (l *tracingLogger) Error(msg string, keyvals ...interface{}) { + l.next.Error(msg, formatErrors(keyvals)...) +} + +func (l *tracingLogger) With(keyvals ...interface{}) Logger { + return &tracingLogger{next: l.next.With(formatErrors(keyvals)...)} +} + +func formatErrors(keyvals []interface{}) []interface{} { + newKeyvals := make([]interface{}, len(keyvals)) + copy(newKeyvals, keyvals) + for i := 0; i < len(newKeyvals)-1; i += 2 { + if err, ok := newKeyvals[i+1].(stackTracer); ok { + newKeyvals[i+1] = tracedError{err} + } + } + return newKeyvals +} + +// tracedError wraps a stackTracer and just makes the Error() result +// always return a full stack trace. +type tracedError struct { + wrapped stackTracer +} + +var _ stackTracer = tracedError{} + +func (t tracedError) StackTrace() errors.StackTrace { + return t.wrapped.StackTrace() +} + +func (t tracedError) Cause() error { + return t.wrapped +} + +func (t tracedError) Error() string { + return fmt.Sprintf("%+v", t.wrapped) +} diff --git a/libs/log/tracing_logger_test.go b/libs/log/tracing_logger_test.go new file mode 100644 index 000000000..1abc6440f --- /dev/null +++ b/libs/log/tracing_logger_test.go @@ -0,0 +1,41 @@ +package log_test + +import ( + "bytes" + stderr "errors" + "fmt" + "strings" + "testing" + + "github.com/pkg/errors" + "github.com/tendermint/tendermint/libs/log" +) + +func TestTracingLogger(t *testing.T) { + var buf bytes.Buffer + + logger := log.NewTMJSONLogger(&buf) + + logger1 := log.NewTracingLogger(logger) + err1 := errors.New("Courage is grace under pressure.") + err2 := errors.New("It does not matter how slowly you go, so long as you do not stop.") + logger1.With("err1", err1).Info("foo", "err2", err2) + have := strings.Replace(strings.Replace(strings.TrimSpace(buf.String()), "\\n", "", -1), "\\t", "", -1) + if want := strings.Replace(strings.Replace(`{"_msg":"foo","err1":"`+fmt.Sprintf("%+v", err1)+`","err2":"`+fmt.Sprintf("%+v", err2)+`","level":"info"}`, "\t", "", -1), "\n", "", -1); want != have { + t.Errorf("\nwant '%s'\nhave '%s'", want, have) + } + + buf.Reset() + + logger.With("err1", stderr.New("Opportunities don't happen. You create them.")).Info("foo", "err2", stderr.New("Once you choose hope, anything's possible.")) + if want, have := `{"_msg":"foo","err1":"Opportunities don't happen. You create them.","err2":"Once you choose hope, anything's possible.","level":"info"}`, strings.TrimSpace(buf.String()); want != have { + t.Errorf("\nwant '%s'\nhave '%s'", want, have) + } + + buf.Reset() + + logger.With("user", "Sam").With("context", "value").Info("foo", "bar", "baz") + if want, have := `{"_msg":"foo","bar":"baz","context":"value","level":"info","user":"Sam"}`, strings.TrimSpace(buf.String()); want != have { + t.Errorf("\nwant '%s'\nhave '%s'", want, have) + } +} diff --git a/libs/merge.sh b/libs/merge.sh new file mode 100644 index 000000000..7ff0ed94e --- /dev/null +++ b/libs/merge.sh @@ -0,0 +1,54 @@ +#! /bin/bash +set -e + +# NOTE: go-alert depends on go-common + +REPOS=("autofile" "clist" "db" "events" "flowrate" "logger" "process") + +mkdir common +git mv *.go common +git mv LICENSE common + +git commit -m "move all files to common/ to begin repo merge" + +for repo in "${REPOS[@]}"; do + # add and fetch the repo + git remote add -f "$repo" "https://github.com/tendermint/go-${repo}" + + # merge master and move into subdir + git merge "$repo/master" --no-edit + + if [[ "$repo" != "flowrate" ]]; then + mkdir "$repo" + git mv *.go "$repo/" + fi + + set +e # these might not exist + git mv *.md "$repo/" + git mv README "$repo/README.md" + git mv Makefile "$repo/Makefile" + git rm LICENSE + set -e + + # commit + git commit -m "merge go-${repo}" + + git remote rm "$repo" +done + +go get github.com/ebuchman/got +got replace "tendermint/go-common" "tendermint/go-common/common" +for repo in "${REPOS[@]}"; do + + if [[ "$repo" != "flowrate" ]]; then + got replace "tendermint/go-${repo}" "tendermint/go-common/${repo}" + else + got replace "tendermint/go-${repo}/flowrate" "tendermint/go-common/flowrate" + fi +done + +git add -u +git commit -m "update import paths" + +# TODO: change any paths in non-Go files +# TODO: add license diff --git a/libs/pubsub/example_test.go b/libs/pubsub/example_test.go index 260521cd9..4e4634de5 100644 --- a/libs/pubsub/example_test.go +++ b/libs/pubsub/example_test.go @@ -6,7 +6,7 @@ import ( "github.com/stretchr/testify/require" - "github.com/tendermint/tmlibs/log" + "github.com/tendermint/tendermint/libs/log" "github.com/tendermint/tendermint/libs/pubsub" "github.com/tendermint/tendermint/libs/pubsub/query" diff --git a/libs/pubsub/pubsub.go b/libs/pubsub/pubsub.go index 776e0653b..4280ca1ea 100644 --- a/libs/pubsub/pubsub.go +++ b/libs/pubsub/pubsub.go @@ -16,7 +16,7 @@ import ( "errors" "sync" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) type operation int diff --git a/libs/pubsub/pubsub_test.go b/libs/pubsub/pubsub_test.go index fd6c11cf4..5e9931e40 100644 --- a/libs/pubsub/pubsub_test.go +++ b/libs/pubsub/pubsub_test.go @@ -10,7 +10,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/tendermint/tmlibs/log" + "github.com/tendermint/tendermint/libs/log" "github.com/tendermint/tendermint/libs/pubsub" "github.com/tendermint/tendermint/libs/pubsub/query" diff --git a/libs/test.sh b/libs/test.sh new file mode 100755 index 000000000..ecf17fc45 --- /dev/null +++ b/libs/test.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -e + +# run the linter +# make metalinter_test + +# setup certs +make gen_certs + +# run the unit tests with coverage +echo "" > coverage.txt +for d in $(go list ./... | grep -v vendor); do + go test -race -coverprofile=profile.out -covermode=atomic "$d" + if [ -f profile.out ]; then + cat profile.out >> coverage.txt + rm profile.out + fi +done + +# cleanup certs +make clean_certs diff --git a/libs/test/assert.go b/libs/test/assert.go new file mode 100644 index 000000000..a6ffed0ce --- /dev/null +++ b/libs/test/assert.go @@ -0,0 +1,14 @@ +package test + +import ( + "testing" +) + +func AssertPanics(t *testing.T, msg string, f func()) { + defer func() { + if err := recover(); err == nil { + t.Errorf("Should have panic'd, but didn't: %v", msg) + } + }() + f() +} diff --git a/libs/test/mutate.go b/libs/test/mutate.go new file mode 100644 index 000000000..3bbbbd217 --- /dev/null +++ b/libs/test/mutate.go @@ -0,0 +1,28 @@ +package test + +import ( + cmn "github.com/tendermint/tendermint/libs/common" +) + +// Contract: !bytes.Equal(input, output) && len(input) >= len(output) +func MutateByteSlice(bytez []byte) []byte { + // If bytez is empty, panic + if len(bytez) == 0 { + panic("Cannot mutate an empty bytez") + } + + // Copy bytez + mBytez := make([]byte, len(bytez)) + copy(mBytez, bytez) + bytez = mBytez + + // Try a random mutation + switch cmn.RandInt() % 2 { + case 0: // Mutate a single byte + bytez[cmn.RandInt()%len(bytez)] += byte(cmn.RandInt()%255 + 1) + case 1: // Remove an arbitrary byte + pos := cmn.RandInt() % len(bytez) + bytez = append(bytez[:pos], bytez[pos+1:]...) + } + return bytez +} diff --git a/libs/version/version.go b/libs/version/version.go new file mode 100644 index 000000000..6e73a937d --- /dev/null +++ b/libs/version/version.go @@ -0,0 +1,3 @@ +package version + +const Version = "0.9.0" diff --git a/lite/client/main_test.go b/lite/client/main_test.go index 972d37310..49b194366 100644 --- a/lite/client/main_test.go +++ b/lite/client/main_test.go @@ -4,7 +4,7 @@ import ( "os" "testing" - "github.com/tendermint/abci/example/kvstore" + "github.com/tendermint/tendermint/abci/example/kvstore" nm "github.com/tendermint/tendermint/node" rpctest "github.com/tendermint/tendermint/rpc/test" diff --git a/lite/files/commit_test.go b/lite/files/commit_test.go index e0235ba29..2891e5809 100644 --- a/lite/files/commit_test.go +++ b/lite/files/commit_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" "github.com/tendermint/tendermint/lite" ) diff --git a/lite/files/wire.go b/lite/files/wire.go index 99f98931e..3a207744a 100644 --- a/lite/files/wire.go +++ b/lite/files/wire.go @@ -2,7 +2,7 @@ package files import ( "github.com/tendermint/go-amino" - "github.com/tendermint/go-crypto" + "github.com/tendermint/tendermint/crypto" ) var cdc = amino.NewCodec() diff --git a/lite/helpers.go b/lite/helpers.go index 714675afb..695f6fb9b 100644 --- a/lite/helpers.go +++ b/lite/helpers.go @@ -3,7 +3,7 @@ package lite import ( "time" - crypto "github.com/tendermint/go-crypto" + crypto "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/types" ) @@ -103,7 +103,13 @@ func makeVote(header *types.Header, vals *types.ValidatorSet, key crypto.PrivKey } // Sign it signBytes := vote.SignBytes(header.ChainID) - vote.Signature = key.Sign(signBytes) + // TODO Consider reworking makeVote API to return an error + sig, err := key.Sign(signBytes) + if err != nil { + panic(err) + } + vote.Signature = sig + return vote } diff --git a/lite/proxy/proxy.go b/lite/proxy/proxy.go index fe10399dc..0294ddf68 100644 --- a/lite/proxy/proxy.go +++ b/lite/proxy/proxy.go @@ -3,8 +3,8 @@ package proxy import ( "net/http" - "github.com/tendermint/go-amino" - "github.com/tendermint/tmlibs/log" + amino "github.com/tendermint/go-amino" + "github.com/tendermint/tendermint/libs/log" rpcclient "github.com/tendermint/tendermint/rpc/client" "github.com/tendermint/tendermint/rpc/core" @@ -38,7 +38,8 @@ func StartProxy(c rpcclient.Client, listenAddr string, logger log.Logger) error core.SetLogger(logger) mux.HandleFunc(wsEndpoint, wm.WebsocketHandler) - _, err = rpc.StartHTTPServer(listenAddr, mux, logger) + // TODO: limit max number of open connections rpc.Config{MaxOpenConnections: X} + _, err = rpc.StartHTTPServer(listenAddr, mux, logger, rpc.Config{}) return err } diff --git a/lite/proxy/query.go b/lite/proxy/query.go index 9c9557f8f..0ca5be174 100644 --- a/lite/proxy/query.go +++ b/lite/proxy/query.go @@ -3,7 +3,7 @@ package proxy import ( "github.com/pkg/errors" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" "github.com/tendermint/tendermint/lite" "github.com/tendermint/tendermint/lite/client" diff --git a/lite/proxy/query_test.go b/lite/proxy/query_test.go index d6dcccc96..38a43af2b 100644 --- a/lite/proxy/query_test.go +++ b/lite/proxy/query_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/tendermint/abci/example/kvstore" + "github.com/tendermint/tendermint/abci/example/kvstore" "github.com/tendermint/tendermint/lite" certclient "github.com/tendermint/tendermint/lite/client" diff --git a/lite/proxy/wrapper.go b/lite/proxy/wrapper.go index 5fb12a40a..f0eb6b41e 100644 --- a/lite/proxy/wrapper.go +++ b/lite/proxy/wrapper.go @@ -1,7 +1,7 @@ package proxy import ( - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" "github.com/tendermint/tendermint/lite" certclient "github.com/tendermint/tendermint/lite/client" diff --git a/mempool/mempool.go b/mempool/mempool.go index 935dfaac7..06852c9af 100644 --- a/mempool/mempool.go +++ b/mempool/mempool.go @@ -10,11 +10,11 @@ import ( "github.com/pkg/errors" - abci "github.com/tendermint/abci/types" - auto "github.com/tendermint/tmlibs/autofile" - "github.com/tendermint/tmlibs/clist" - cmn "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tmlibs/log" + abci "github.com/tendermint/tendermint/abci/types" + auto "github.com/tendermint/tendermint/libs/autofile" + "github.com/tendermint/tendermint/libs/clist" + cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/libs/log" cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/proxy" @@ -57,6 +57,11 @@ var ( ErrMempoolIsFull = errors.New("Mempool is full") ) +// TxID is the hex encoded hash of the bytes as a types.Tx. +func TxID(tx []byte) string { + return fmt.Sprintf("%X", types.Tx(tx).Hash()) +} + // Mempool is an ordered in-memory pool for transactions before they are proposed in a consensus // round. Transaction validity is checked using the CheckTx abci message before the transaction is // added to the pool. The Mempool uses a concurrent list structure for storing transactions that @@ -83,10 +88,20 @@ type Mempool struct { wal *auto.AutoFile logger log.Logger + + metrics *Metrics } +// MempoolOption sets an optional parameter on the Mempool. +type MempoolOption func(*Mempool) + // NewMempool returns a new Mempool with the given configuration and connection to an application. -func NewMempool(config *cfg.MempoolConfig, proxyAppConn proxy.AppConnMempool, height int64) *Mempool { +func NewMempool( + config *cfg.MempoolConfig, + proxyAppConn proxy.AppConnMempool, + height int64, + options ...MempoolOption, +) *Mempool { mempool := &Mempool{ config: config, proxyAppConn: proxyAppConn, @@ -97,6 +112,7 @@ func NewMempool(config *cfg.MempoolConfig, proxyAppConn proxy.AppConnMempool, he recheckCursor: nil, recheckEnd: nil, logger: log.NewNopLogger(), + metrics: NopMetrics(), } if config.CacheSize > 0 { mempool.cache = newMapTxCache(config.CacheSize) @@ -104,6 +120,9 @@ func NewMempool(config *cfg.MempoolConfig, proxyAppConn proxy.AppConnMempool, he mempool.cache = nopTxCache{} } proxyAppConn.SetResponseCallback(mempool.resCb) + for _, option := range options { + option(mempool) + } return mempool } @@ -119,6 +138,11 @@ func (mem *Mempool) SetLogger(l log.Logger) { mem.logger = l } +// WithMetrics sets the metrics. +func WithMetrics(metrics *Metrics) MempoolOption { + return func(mem *Mempool) { mem.metrics = metrics } +} + // CloseWAL closes and discards the underlying WAL file. // Any further writes will not be relayed to disk. func (mem *Mempool) CloseWAL() bool { @@ -254,6 +278,7 @@ func (mem *Mempool) resCb(req *abci.Request, res *abci.Response) { } else { mem.resCbRecheck(req, res) } + mem.metrics.Size.Set(float64(mem.Size())) } func (mem *Mempool) resCbNormal(req *abci.Request, res *abci.Response) { @@ -268,11 +293,11 @@ func (mem *Mempool) resCbNormal(req *abci.Request, res *abci.Response) { tx: tx, } mem.txs.PushBack(memTx) - mem.logger.Info("Added good transaction", "tx", fmt.Sprintf("%X", types.Tx(tx).Hash()), "res", r) + mem.logger.Info("Added good transaction", "tx", TxID(tx), "res", r, "total", mem.Size()) mem.notifyTxsAvailable() } else { // ignore bad transaction - mem.logger.Info("Rejected bad transaction", "tx", fmt.Sprintf("%X", types.Tx(tx).Hash()), "res", r) + mem.logger.Info("Rejected bad transaction", "tx", TxID(tx), "res", r) // remove from cache (it might be good later) mem.cache.Remove(tx) @@ -397,6 +422,7 @@ func (mem *Mempool) Update(height int64, txs types.Txs) error { // mem.recheckCursor re-scans mem.txs and possibly removes some txs. // Before mem.Reap(), we should wait for mem.recheckCursor to be nil. } + mem.metrics.Size.Set(float64(mem.Size())) return nil } diff --git a/mempool/mempool_test.go b/mempool/mempool_test.go index a67adf6d3..1a91de4f9 100644 --- a/mempool/mempool_test.go +++ b/mempool/mempool_test.go @@ -11,11 +11,11 @@ import ( "testing" "time" - "github.com/tendermint/abci/example/counter" - "github.com/tendermint/abci/example/kvstore" - abci "github.com/tendermint/abci/types" - cmn "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tmlibs/log" + "github.com/tendermint/tendermint/abci/example/counter" + "github.com/tendermint/tendermint/abci/example/kvstore" + abci "github.com/tendermint/tendermint/abci/types" + cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/libs/log" cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/proxy" diff --git a/mempool/metrics.go b/mempool/metrics.go new file mode 100644 index 000000000..f381678cb --- /dev/null +++ b/mempool/metrics.go @@ -0,0 +1,34 @@ +package mempool + +import ( + "github.com/go-kit/kit/metrics" + "github.com/go-kit/kit/metrics/discard" + + prometheus "github.com/go-kit/kit/metrics/prometheus" + stdprometheus "github.com/prometheus/client_golang/prometheus" +) + +// Metrics contains metrics exposed by this package. +// see MetricsProvider for descriptions. +type Metrics struct { + // Size of the mempool. + Size metrics.Gauge +} + +// PrometheusMetrics returns Metrics build using Prometheus client library. +func PrometheusMetrics() *Metrics { + return &Metrics{ + Size: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Subsystem: "mempool", + Name: "size", + Help: "Size of the mempool (number of uncommitted transactions).", + }, []string{}), + } +} + +// NopMetrics returns no-op Metrics. +func NopMetrics() *Metrics { + return &Metrics{ + Size: discard.NewGauge(), + } +} diff --git a/mempool/reactor.go b/mempool/reactor.go index 5d1f4e793..e63ff58e8 100644 --- a/mempool/reactor.go +++ b/mempool/reactor.go @@ -5,10 +5,10 @@ import ( "reflect" "time" - abci "github.com/tendermint/abci/types" amino "github.com/tendermint/go-amino" - "github.com/tendermint/tmlibs/clist" - "github.com/tendermint/tmlibs/log" + abci "github.com/tendermint/tendermint/abci/types" + "github.com/tendermint/tendermint/libs/clist" + "github.com/tendermint/tendermint/libs/log" cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/p2p" @@ -90,7 +90,7 @@ func (memR *MempoolReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) { case *TxMessage: err := memR.Mempool.CheckTx(msg.Tx, nil) if err != nil { - memR.Logger.Info("Could not check tx", "tx", msg.Tx, "err", err) + memR.Logger.Info("Could not check tx", "tx", TxID(msg.Tx), "err", err) } // broadcasting happens from go routines per peer default: diff --git a/mempool/reactor_test.go b/mempool/reactor_test.go index 0a6d09153..b4362032a 100644 --- a/mempool/reactor_test.go +++ b/mempool/reactor_test.go @@ -12,8 +12,8 @@ import ( "github.com/go-kit/kit/log/term" - "github.com/tendermint/abci/example/kvstore" - "github.com/tendermint/tmlibs/log" + "github.com/tendermint/tendermint/abci/example/kvstore" + "github.com/tendermint/tendermint/libs/log" cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/p2p" diff --git a/networks/local/README.md b/networks/local/README.md index 554abdf4b..09a0b12cb 100644 --- a/networks/local/README.md +++ b/networks/local/README.md @@ -2,7 +2,7 @@ ## Requirements -- [Install tendermint](/docs/install.rst) +- [Install tendermint](/docs/install.md) - [Install docker](https://docs.docker.com/engine/installation/) - [Install docker-compose](https://docs.docker.com/compose/install/) diff --git a/networks/remote/README.md b/networks/remote/README.md index 090f6da16..2094fcc98 100644 --- a/networks/remote/README.md +++ b/networks/remote/README.md @@ -1,3 +1,3 @@ # Remote Cluster with Terraform and Ansible -See the [docs](/docs/terraform-and-ansible.rst) +See the [docs](/docs/terraform-and-ansible.md) diff --git a/node/id.go b/node/id.go index fa391f946..5100597c6 100644 --- a/node/id.go +++ b/node/id.go @@ -3,7 +3,7 @@ package node import ( "time" - "github.com/tendermint/go-crypto" + "github.com/tendermint/tendermint/crypto" ) type NodeID struct { diff --git a/node/node.go b/node/node.go index efeb17ee0..0780891ef 100644 --- a/node/node.go +++ b/node/node.go @@ -2,21 +2,24 @@ package node import ( "bytes" + "context" "errors" "fmt" "net" "net/http" - abci "github.com/tendermint/abci/types" + "github.com/prometheus/client_golang/prometheus/promhttp" + amino "github.com/tendermint/go-amino" - crypto "github.com/tendermint/go-crypto" - cmn "github.com/tendermint/tmlibs/common" - dbm "github.com/tendermint/tmlibs/db" - "github.com/tendermint/tmlibs/log" + abci "github.com/tendermint/tendermint/abci/types" + cmn "github.com/tendermint/tendermint/libs/common" + dbm "github.com/tendermint/tendermint/libs/db" + "github.com/tendermint/tendermint/libs/log" bc "github.com/tendermint/tendermint/blockchain" cfg "github.com/tendermint/tendermint/config" cs "github.com/tendermint/tendermint/consensus" + "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/evidence" mempl "github.com/tendermint/tendermint/mempool" "github.com/tendermint/tendermint/p2p" @@ -81,10 +84,25 @@ func DefaultNewNode(config *cfg.Config, logger log.Logger) (*Node, error) { proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir()), DefaultGenesisDocProviderFunc(config), DefaultDBProvider, + DefaultMetricsProvider, logger, ) } +// MetricsProvider returns a consensus, p2p and mempool Metrics. +type MetricsProvider func() (*cs.Metrics, *p2p.Metrics, *mempl.Metrics) + +// DefaultMetricsProvider returns consensus, p2p and mempool Metrics build +// using Prometheus client library. +func DefaultMetricsProvider() (*cs.Metrics, *p2p.Metrics, *mempl.Metrics) { + return cs.PrometheusMetrics(), p2p.PrometheusMetrics(), mempl.PrometheusMetrics() +} + +// NopMetricsProvider returns consensus, p2p and mempool Metrics as no-op. +func NopMetricsProvider() (*cs.Metrics, *p2p.Metrics, *mempl.Metrics) { + return cs.NopMetrics(), p2p.NopMetrics(), mempl.NopMetrics() +} + //------------------------------------------------------------------------------ // Node is the highest level interface to a full Tendermint node. @@ -114,6 +132,7 @@ type Node struct { rpcListeners []net.Listener // rpc servers txIndexer txindex.TxIndexer indexerService *txindex.IndexerService + prometheusSrv *http.Server } // NewNode returns a new, ready to go, Tendermint Node. @@ -122,6 +141,7 @@ func NewNode(config *cfg.Config, clientCreator proxy.ClientCreator, genesisDocProvider GenesisDocProvider, dbProvider DBProvider, + metricsProvider MetricsProvider, logger log.Logger) (*Node, error) { // Get BlockStore @@ -208,11 +228,28 @@ func NewNode(config *cfg.Config, consensusLogger.Info("This node is not a validator", "addr", privValidator.GetAddress(), "pubKey", privValidator.GetPubKey()) } + // metrics + var ( + csMetrics *cs.Metrics + p2pMetrics *p2p.Metrics + memplMetrics *mempl.Metrics + ) + if config.Instrumentation.Prometheus { + csMetrics, p2pMetrics, memplMetrics = metricsProvider() + } else { + csMetrics, p2pMetrics, memplMetrics = NopMetricsProvider() + } + // Make MempoolReactor mempoolLogger := logger.With("module", "mempool") - mempool := mempl.NewMempool(config.Mempool, proxyApp.Mempool(), state.LastBlockHeight) - mempool.InitWAL() // no need to have the mempool wal during tests + mempool := mempl.NewMempool( + config.Mempool, + proxyApp.Mempool(), + state.LastBlockHeight, + mempl.WithMetrics(memplMetrics), + ) mempool.SetLogger(mempoolLogger) + mempool.InitWAL() // no need to have the mempool wal during tests mempoolReactor := mempl.NewMempoolReactor(config.Mempool, mempool) mempoolReactor.SetLogger(mempoolLogger) @@ -241,8 +278,15 @@ func NewNode(config *cfg.Config, bcReactor.SetLogger(logger.With("module", "blockchain")) // Make ConsensusReactor - consensusState := cs.NewConsensusState(config.Consensus, state.Copy(), - blockExec, blockStore, mempool, evidencePool) + consensusState := cs.NewConsensusState( + config.Consensus, + state.Copy(), + blockExec, + blockStore, + mempool, + evidencePool, + cs.WithMetrics(csMetrics), + ) consensusState.SetLogger(consensusLogger) if privValidator != nil { consensusState.SetPrivValidator(privValidator) @@ -252,7 +296,7 @@ func NewNode(config *cfg.Config, p2pLogger := logger.With("module", "p2p") - sw := p2p.NewSwitch(config.P2P) + sw := p2p.NewSwitch(config.P2P, p2p.WithMetrics(p2pMetrics)) sw.SetLogger(p2pLogger) sw.AddReactor("MEMPOOL", mempoolReactor) sw.AddReactor("BLOCKCHAIN", bcReactor) @@ -382,8 +426,11 @@ func (n *Node) OnStart() error { } // Create & add listener - protocol, address := cmn.ProtocolAndAddress(n.config.P2P.ListenAddress) - l := p2p.NewDefaultListener(protocol, address, n.config.P2P.SkipUPNP, n.Logger.With("module", "p2p")) + l := p2p.NewDefaultListener( + n.config.P2P.ListenAddress, + n.config.P2P.ExternalAddress, + n.config.P2P.UPNP, + n.Logger.With("module", "p2p")) n.sw.AddListener(l) // Generate node PrivKey @@ -411,6 +458,10 @@ func (n *Node) OnStart() error { n.rpcListeners = listeners } + if n.config.Instrumentation.Prometheus { + n.prometheusSrv = n.startPrometheusServer(n.config.Instrumentation.PrometheusListenAddr) + } + // Start the switch (the P2P server). err = n.sw.Start() if err != nil { @@ -452,6 +503,13 @@ func (n *Node) OnStop() { n.Logger.Error("Error stopping priv validator socket client", "err", err) } } + + if n.prometheusSrv != nil { + if err := n.prometheusSrv.Shutdown(context.Background()); err != nil { + // Error from closing listeners, or context timeout: + n.Logger.Error("Prometheus HTTP server Shutdown", "err", err) + } + } } // RunForever waits for an interrupt signal and stops the node. @@ -507,7 +565,12 @@ func (n *Node) startRPC() ([]net.Listener, error) { wm.SetLogger(rpcLogger.With("protocol", "websocket")) mux.HandleFunc("/websocket", wm.WebsocketHandler) rpcserver.RegisterRPCFuncs(mux, rpccore.Routes, coreCodec, rpcLogger) - listener, err := rpcserver.StartHTTPServer(listenAddr, mux, rpcLogger) + listener, err := rpcserver.StartHTTPServer( + listenAddr, + mux, + rpcLogger, + rpcserver.Config{MaxOpenConnections: n.config.RPC.MaxOpenConnections}, + ) if err != nil { return nil, err } @@ -517,7 +580,12 @@ func (n *Node) startRPC() ([]net.Listener, error) { // we expose a simplified api over grpc for convenience to app devs grpcListenAddr := n.config.RPC.GRPCListenAddress if grpcListenAddr != "" { - listener, err := grpccore.StartGRPCServer(grpcListenAddr) + listener, err := grpccore.StartGRPCServer( + grpcListenAddr, + grpccore.Config{ + MaxOpenConnections: n.config.RPC.GRPCMaxOpenConnections, + }, + ) if err != nil { return nil, err } @@ -527,6 +595,22 @@ func (n *Node) startRPC() ([]net.Listener, error) { return listeners, nil } +// startPrometheusServer starts a Prometheus HTTP server, listening for metrics +// collectors on addr. +func (n *Node) startPrometheusServer(addr string) *http.Server { + srv := &http.Server{ + Addr: addr, + Handler: promhttp.Handler(), + } + go func() { + if err := srv.ListenAndServe(); err != http.ErrServerClosed { + // Error starting or closing listener: + n.Logger.Error("Prometheus HTTP server ListenAndServe", "err", err) + } + }() + return srv +} + // Switch returns the Node's Switch. func (n *Node) Switch() *p2p.Switch { return n.sw @@ -615,7 +699,7 @@ func (n *Node) makeNodeInfo(nodeID p2p.ID) p2p.NodeInfo { } p2pListener := n.sw.Listeners()[0] - p2pHost := p2pListener.ExternalAddress().IP.String() + p2pHost := p2pListener.ExternalAddressHost() p2pPort := p2pListener.ExternalAddress().Port nodeInfo.ListenAddr = cmn.Fmt("%v:%v", p2pHost, p2pPort) diff --git a/node/node_test.go b/node/node_test.go index cdabdbb3f..80f6f02c2 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/assert" - "github.com/tendermint/tmlibs/log" + "github.com/tendermint/tendermint/libs/log" cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/types" diff --git a/node/wire.go b/node/wire.go index a0d7677df..8b3ae8950 100644 --- a/node/wire.go +++ b/node/wire.go @@ -2,7 +2,7 @@ package node import ( amino "github.com/tendermint/go-amino" - crypto "github.com/tendermint/go-crypto" + crypto "github.com/tendermint/tendermint/crypto" ) var cdc = amino.NewCodec() diff --git a/p2p/base_reactor.go b/p2p/base_reactor.go index 83c8efa4b..da1296da0 100644 --- a/p2p/base_reactor.go +++ b/p2p/base_reactor.go @@ -2,7 +2,7 @@ package p2p import ( "github.com/tendermint/tendermint/p2p/conn" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) type Reactor interface { diff --git a/p2p/conn/connection.go b/p2p/conn/connection.go index 5c7f19cf7..9672e0117 100644 --- a/p2p/conn/connection.go +++ b/p2p/conn/connection.go @@ -12,14 +12,13 @@ import ( "time" amino "github.com/tendermint/go-amino" - cmn "github.com/tendermint/tmlibs/common" - flow "github.com/tendermint/tmlibs/flowrate" - "github.com/tendermint/tmlibs/log" + cmn "github.com/tendermint/tendermint/libs/common" + flow "github.com/tendermint/tendermint/libs/flowrate" + "github.com/tendermint/tendermint/libs/log" ) const ( - maxPacketMsgPayloadSizeDefault = 1024 // NOTE: Must be below 16,384 bytes for 14 below. - maxPacketMsgOverheadSize = 14 // NOTE: See connection_test for derivation. + defaultMaxPacketMsgPayloadSize = 1024 numBatchPacketMsgs = 10 minReadBufferSize = 1024 @@ -96,6 +95,8 @@ type MConnection struct { chStatsTimer *cmn.RepeatTimer // update channel stats periodically created time.Time // time of creation + + _maxPacketMsgSize int } // MConnConfig is a MConnection configuration. @@ -116,16 +117,12 @@ type MConnConfig struct { PongTimeout time.Duration `mapstructure:"pong_timeout"` } -func (cfg *MConnConfig) maxPacketMsgTotalSize() int { - return cfg.MaxPacketMsgPayloadSize + maxPacketMsgOverheadSize -} - // DefaultMConnConfig returns the default config. func DefaultMConnConfig() MConnConfig { return MConnConfig{ SendRate: defaultSendRate, RecvRate: defaultRecvRate, - MaxPacketMsgPayloadSize: maxPacketMsgPayloadSizeDefault, + MaxPacketMsgPayloadSize: defaultMaxPacketMsgPayloadSize, FlushThrottle: defaultFlushThrottle, PingInterval: defaultPingInterval, PongTimeout: defaultPongTimeout, @@ -175,6 +172,9 @@ func NewMConnectionWithConfig(conn net.Conn, chDescs []*ChannelDescriptor, onRec mconn.BaseService = *cmn.NewBaseService(nil, "MConnection", mconn) + // maxPacketMsgSize() is a bit heavy, so call just once + mconn._maxPacketMsgSize = mconn.maxPacketMsgSize() + return mconn } @@ -399,7 +399,7 @@ func (c *MConnection) sendSomePacketMsgs() bool { // Block until .sendMonitor says we can write. // Once we're ready we send more than we asked for, // but amortized it should even out. - c.sendMonitor.Limit(c.config.maxPacketMsgTotalSize(), atomic.LoadInt64(&c.config.SendRate), true) + c.sendMonitor.Limit(c._maxPacketMsgSize, atomic.LoadInt64(&c.config.SendRate), true) // Now send some PacketMsgs. for i := 0; i < numBatchPacketMsgs; i++ { @@ -457,7 +457,7 @@ func (c *MConnection) recvRoutine() { FOR_LOOP: for { // Block until .recvMonitor says we can read. - c.recvMonitor.Limit(c.config.maxPacketMsgTotalSize(), atomic.LoadInt64(&c.config.RecvRate), true) + c.recvMonitor.Limit(c._maxPacketMsgSize, atomic.LoadInt64(&c.config.RecvRate), true) // Peek into bufConnReader for debugging /* @@ -477,7 +477,7 @@ FOR_LOOP: var packet Packet var _n int64 var err error - _n, err = cdc.UnmarshalBinaryReader(c.bufConnReader, &packet, int64(c.config.maxPacketMsgTotalSize())) + _n, err = cdc.UnmarshalBinaryReader(c.bufConnReader, &packet, int64(c._maxPacketMsgSize)) c.recvMonitor.Update(int(_n)) if err != nil { if c.IsRunning() { @@ -550,6 +550,16 @@ func (c *MConnection) stopPongTimer() { } } +// maxPacketMsgSize returns a maximum size of PacketMsg, including the overhead +// of amino encoding. +func (c *MConnection) maxPacketMsgSize() int { + return len(cdc.MustMarshalBinary(PacketMsg{ + ChannelID: 0x01, + EOF: 1, + Bytes: make([]byte, c.config.MaxPacketMsgPayloadSize), + })) + 10 // leave room for changes in amino +} + type ConnectionStatus struct { Duration time.Duration SendMonitor flow.Status diff --git a/p2p/conn/connection_test.go b/p2p/conn/connection_test.go index ade8e8e9f..19e05fbc7 100644 --- a/p2p/conn/connection_test.go +++ b/p2p/conn/connection_test.go @@ -11,9 +11,11 @@ import ( "github.com/stretchr/testify/require" amino "github.com/tendermint/go-amino" - "github.com/tendermint/tmlibs/log" + "github.com/tendermint/tendermint/libs/log" ) +const maxPingPongPacketSize = 1024 // bytes + func createTestMConnection(conn net.Conn) *MConnection { onReceive := func(chID byte, msgBytes []byte) { } @@ -138,8 +140,7 @@ func TestMConnectionPongTimeoutResultsInError(t *testing.T) { go func() { // read ping var pkt PacketPing - const maxPacketPingSize = 1024 - _, err = cdc.UnmarshalBinaryReader(server, &pkt, maxPacketPingSize) + _, err = cdc.UnmarshalBinaryReader(server, &pkt, maxPingPongPacketSize) assert.Nil(t, err) serverGotPing <- struct{}{} }() @@ -186,7 +187,7 @@ func TestMConnectionMultiplePongsInTheBeginning(t *testing.T) { go func() { // read ping (one byte) var packet, err = Packet(nil), error(nil) - _, err = cdc.UnmarshalBinaryReader(server, &packet, 1024) + _, err = cdc.UnmarshalBinaryReader(server, &packet, maxPingPongPacketSize) require.Nil(t, err) serverGotPing <- struct{}{} // respond with pong @@ -229,15 +230,15 @@ func TestMConnectionMultiplePings(t *testing.T) { _, err = server.Write(cdc.MustMarshalBinary(PacketPing{})) require.Nil(t, err) var pkt PacketPong - _, err = cdc.UnmarshalBinaryReader(server, &pkt, 1024) + _, err = cdc.UnmarshalBinaryReader(server, &pkt, maxPingPongPacketSize) require.Nil(t, err) _, err = server.Write(cdc.MustMarshalBinary(PacketPing{})) require.Nil(t, err) - _, err = cdc.UnmarshalBinaryReader(server, &pkt, 1024) + _, err = cdc.UnmarshalBinaryReader(server, &pkt, maxPingPongPacketSize) require.Nil(t, err) _, err = server.Write(cdc.MustMarshalBinary(PacketPing{})) require.Nil(t, err) - _, err = cdc.UnmarshalBinaryReader(server, &pkt, 1024) + _, err = cdc.UnmarshalBinaryReader(server, &pkt, maxPingPongPacketSize) require.Nil(t, err) assert.True(t, mconn.IsRunning()) @@ -269,7 +270,7 @@ func TestMConnectionPingPongs(t *testing.T) { go func() { // read ping var pkt PacketPing - _, err = cdc.UnmarshalBinaryReader(server, &pkt, 1024) + _, err = cdc.UnmarshalBinaryReader(server, &pkt, maxPingPongPacketSize) require.Nil(t, err) serverGotPing <- struct{}{} // respond with pong @@ -279,7 +280,7 @@ func TestMConnectionPingPongs(t *testing.T) { time.Sleep(mconn.config.PingInterval) // read ping - _, err = cdc.UnmarshalBinaryReader(server, &pkt, 1024) + _, err = cdc.UnmarshalBinaryReader(server, &pkt, maxPingPongPacketSize) require.Nil(t, err) // respond with pong _, err = server.Write(cdc.MustMarshalBinary(PacketPong{})) @@ -422,15 +423,6 @@ func TestMConnectionReadErrorLongMessage(t *testing.T) { // send msg thats just right var err error var buf = new(bytes.Buffer) - // - Uvarint length of MustMarshalBinary(packet) = 1 or 2 bytes - // (as long as it's less than 16,384 bytes) - // - Prefix bytes = 4 bytes - // - ChannelID field key + byte = 2 bytes - // - EOF field key + byte = 2 bytes - // - Bytes field key = 1 bytes - // - Uvarint length of MustMarshalBinary(bytes) = 1 or 2 bytes - // - Struct terminator = 1 byte - // = up to 14 bytes overhead for the packet. var packet = PacketMsg{ ChannelID: 0x01, EOF: 1, @@ -448,7 +440,7 @@ func TestMConnectionReadErrorLongMessage(t *testing.T) { packet = PacketMsg{ ChannelID: 0x01, EOF: 1, - Bytes: make([]byte, mconnClient.config.MaxPacketMsgPayloadSize+1), + Bytes: make([]byte, mconnClient.config.MaxPacketMsgPayloadSize+100), } _, err = cdc.MarshalBinaryWriter(buf, packet) assert.Nil(t, err) diff --git a/p2p/conn/secret_connection.go b/p2p/conn/secret_connection.go index 2a507f882..a2cbe008d 100644 --- a/p2p/conn/secret_connection.go +++ b/p2p/conn/secret_connection.go @@ -20,8 +20,8 @@ import ( "golang.org/x/crypto/nacl/secretbox" "golang.org/x/crypto/ripemd160" - "github.com/tendermint/go-crypto" - cmn "github.com/tendermint/tmlibs/common" + "github.com/tendermint/tendermint/crypto" + cmn "github.com/tendermint/tendermint/libs/common" ) // 4 + 1024 == 1028 total frame size @@ -267,7 +267,11 @@ func genChallenge(loPubKey, hiPubKey *[32]byte) (challenge *[32]byte) { } func signChallenge(challenge *[32]byte, locPrivKey crypto.PrivKey) (signature crypto.Signature) { - signature = locPrivKey.Sign(challenge[:]) + signature, err := locPrivKey.Sign(challenge[:]) + // TODO(ismail): let signChallenge return an error instead + if err != nil { + panic(err) + } return } diff --git a/p2p/conn/secret_connection_test.go b/p2p/conn/secret_connection_test.go index 8dfc2bef0..7274dfaf7 100644 --- a/p2p/conn/secret_connection_test.go +++ b/p2p/conn/secret_connection_test.go @@ -7,8 +7,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - crypto "github.com/tendermint/go-crypto" - cmn "github.com/tendermint/tmlibs/common" + crypto "github.com/tendermint/tendermint/crypto" + cmn "github.com/tendermint/tendermint/libs/common" ) type kvstoreConn struct { diff --git a/p2p/conn/wire.go b/p2p/conn/wire.go index 02d67f6fb..3182fde38 100644 --- a/p2p/conn/wire.go +++ b/p2p/conn/wire.go @@ -2,7 +2,7 @@ package conn import ( "github.com/tendermint/go-amino" - "github.com/tendermint/go-crypto" + "github.com/tendermint/tendermint/crypto" ) var cdc *amino.Codec = amino.NewCodec() diff --git a/p2p/dummy/peer.go b/p2p/dummy/peer.go index fc2242366..d18a9f99d 100644 --- a/p2p/dummy/peer.go +++ b/p2p/dummy/peer.go @@ -5,7 +5,7 @@ import ( p2p "github.com/tendermint/tendermint/p2p" tmconn "github.com/tendermint/tendermint/p2p/conn" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) type peer struct { diff --git a/p2p/fuzz.go b/p2p/fuzz.go index 8d00ba40d..80e4fed6a 100644 --- a/p2p/fuzz.go +++ b/p2p/fuzz.go @@ -6,7 +6,7 @@ import ( "time" "github.com/tendermint/tendermint/config" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) // FuzzedConnection wraps any net.Conn and depending on the mode either delays diff --git a/p2p/key.go b/p2p/key.go index 73103ebd4..9548d34f0 100644 --- a/p2p/key.go +++ b/p2p/key.go @@ -6,8 +6,8 @@ import ( "fmt" "io/ioutil" - crypto "github.com/tendermint/go-crypto" - cmn "github.com/tendermint/tmlibs/common" + crypto "github.com/tendermint/tendermint/crypto" + cmn "github.com/tendermint/tendermint/libs/common" ) // ID is a hex-encoded crypto.Address diff --git a/p2p/key_test.go b/p2p/key_test.go index c2e1f3e0e..51e1c0787 100644 --- a/p2p/key_test.go +++ b/p2p/key_test.go @@ -7,7 +7,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) func TestLoadOrGenNodeKey(t *testing.T) { diff --git a/p2p/listener.go b/p2p/listener.go index e698765cd..3509ec69c 100644 --- a/p2p/listener.go +++ b/p2p/listener.go @@ -4,22 +4,30 @@ import ( "fmt" "net" "strconv" + "strings" "time" + cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/libs/log" "github.com/tendermint/tendermint/p2p/upnp" - cmn "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tmlibs/log" ) +// Listener is a network listener for stream-oriented protocols, providing +// convenient methods to get listener's internal and external addresses. +// Clients are supposed to read incoming connections from a channel, returned +// by Connections() method. type Listener interface { Connections() <-chan net.Conn InternalAddress() *NetAddress ExternalAddress() *NetAddress + ExternalAddressHost() string String() string Stop() error } -// Implements Listener +// DefaultListener is a cmn.Service, running net.Listener underneath. +// Optionally, UPnP is used upon calling NewDefaultListener to resolve external +// address. type DefaultListener struct { cmn.BaseService @@ -29,6 +37,8 @@ type DefaultListener struct { connections chan net.Conn } +var _ Listener = (*DefaultListener)(nil) + const ( numBufferedConnections = 10 defaultExternalPort = 8770 @@ -47,9 +57,16 @@ func splitHostPort(addr string) (host string, port int) { return host, port } -// skipUPNP: If true, does not try getUPNPExternalAddress() -func NewDefaultListener(protocol string, lAddr string, skipUPNP bool, logger log.Logger) Listener { - // Local listen IP & port +// NewDefaultListener creates a new DefaultListener on lAddr, optionally trying +// to determine external address using UPnP. +func NewDefaultListener( + fullListenAddrString string, + externalAddrString string, + useUPnP bool, + logger log.Logger) Listener { + + // Split protocol, address, and port. + protocol, lAddr := cmn.ProtocolAndAddress(fullListenAddrString) lAddrIP, lAddrPort := splitHostPort(lAddr) // Create listener @@ -77,17 +94,28 @@ func NewDefaultListener(protocol string, lAddr string, skipUPNP bool, logger log panic(err) } - // Determine external address... + inAddrAny := lAddrIP == "" || lAddrIP == "0.0.0.0" + + // Determine external address. var extAddr *NetAddress - if !skipUPNP { - // If the lAddrIP is INADDR_ANY, try UPnP - if lAddrIP == "" || lAddrIP == "0.0.0.0" { - extAddr = getUPNPExternalAddress(lAddrPort, listenerPort, logger) + + if externalAddrString != "" { + var err error + extAddr, err = NewNetAddressStringWithOptionalID(externalAddrString) + if err != nil { + panic(fmt.Sprintf("Error in ExternalAddress: %v", err)) } } - // Otherwise just use the local address... + + // If the lAddrIP is INADDR_ANY, try UPnP. + if extAddr == nil && useUPnP && inAddrAny { + extAddr = getUPNPExternalAddress(lAddrPort, listenerPort, logger) + } + + // Otherwise just use the local address. if extAddr == nil { - extAddr = getNaiveExternalAddress(listenerPort, false, logger) + defaultToIPv4 := inAddrAny + extAddr = getNaiveExternalAddress(defaultToIPv4, listenerPort, false, logger) } if extAddr == nil { panic("Could not determine external address!") @@ -107,6 +135,8 @@ func NewDefaultListener(protocol string, lAddr string, skipUPNP bool, logger log return dl } +// OnStart implements cmn.Service by spinning a goroutine, listening for new +// connections. func (l *DefaultListener) OnStart() error { if err := l.BaseService.OnStart(); err != nil { return err @@ -115,6 +145,7 @@ func (l *DefaultListener) OnStart() error { return nil } +// OnStop implements cmn.Service by closing the listener. func (l *DefaultListener) OnStop() { l.BaseService.OnStop() l.listener.Close() // nolint: errcheck @@ -145,24 +176,33 @@ func (l *DefaultListener) listenRoutine() { } } -// A channel of inbound connections. +// Connections returns a channel of inbound connections. // It gets closed when the listener closes. func (l *DefaultListener) Connections() <-chan net.Conn { return l.connections } +// InternalAddress returns the internal NetAddress (address used for +// listening). func (l *DefaultListener) InternalAddress() *NetAddress { return l.intAddr } +// ExternalAddress returns the external NetAddress (publicly available, +// determined using either UPnP or local resolver). func (l *DefaultListener) ExternalAddress() *NetAddress { return l.extAddr } -// NOTE: The returned listener is already Accept()'ing. -// So it's not suitable to pass into http.Serve(). -func (l *DefaultListener) NetListener() net.Listener { - return l.listener +// ExternalAddressHost returns the external NetAddress IP string. If an IP is +// IPv6, it's wrapped in brackets ("[2001:db8:1f70::999:de8:7648:6e8]"). +func (l *DefaultListener) ExternalAddressHost() string { + ip := l.ExternalAddress().IP + if isIpv6(ip) { + // Means it's ipv6, so format it with brackets + return "[" + ip.String() + "]" + } + return ip.String() } func (l *DefaultListener) String() string { @@ -201,8 +241,20 @@ func getUPNPExternalAddress(externalPort, internalPort int, logger log.Logger) * return NewNetAddressIPPort(ext, uint16(externalPort)) } +func isIpv6(ip net.IP) bool { + v4 := ip.To4() + if v4 != nil { + return false + } + + ipString := ip.String() + + // Extra check just to be sure it's IPv6 + return (strings.Contains(ipString, ":") && !strings.Contains(ipString, ".")) +} + // TODO: use syscalls: see issue #712 -func getNaiveExternalAddress(port int, settleForLocal bool, logger log.Logger) *NetAddress { +func getNaiveExternalAddress(defaultToIPv4 bool, port int, settleForLocal bool, logger log.Logger) *NetAddress { addrs, err := net.InterfaceAddrs() if err != nil { panic(cmn.Fmt("Could not fetch interface addresses: %v", err)) @@ -213,14 +265,20 @@ func getNaiveExternalAddress(port int, settleForLocal bool, logger log.Logger) * if !ok { continue } - v4 := ipnet.IP.To4() - if v4 == nil || (!settleForLocal && v4[0] == 127) { + if defaultToIPv4 || !isIpv6(ipnet.IP) { + v4 := ipnet.IP.To4() + if v4 == nil || (!settleForLocal && v4[0] == 127) { + // loopback + continue + } + } else if !settleForLocal && ipnet.IP.IsLoopback() { + // IPv6, check for loopback continue - } // loopback + } return NewNetAddressIPPort(ipnet.IP, uint16(port)) } // try again, but settle for local logger.Info("Node may not be connected to internet. Settling for local address") - return getNaiveExternalAddress(port, true, logger) + return getNaiveExternalAddress(defaultToIPv4, port, true, logger) } diff --git a/p2p/listener_test.go b/p2p/listener_test.go index 92018e0aa..f87b5d6f5 100644 --- a/p2p/listener_test.go +++ b/p2p/listener_test.go @@ -2,14 +2,17 @@ package p2p import ( "bytes" + "net" + "strings" "testing" - "github.com/tendermint/tmlibs/log" + "github.com/stretchr/testify/require" + "github.com/tendermint/tendermint/libs/log" ) func TestListener(t *testing.T) { // Create a listener - l := NewDefaultListener("tcp", ":8001", true, log.TestingLogger()) + l := NewDefaultListener("tcp://:8001", "", false, log.TestingLogger()) // Dial the listener lAddr := l.ExternalAddress() @@ -45,3 +48,32 @@ func TestListener(t *testing.T) { // Close the server, no longer needed. l.Stop() } + +func TestExternalAddress(t *testing.T) { + { + // Create a listener with no external addr. Should default + // to local ipv4. + l := NewDefaultListener("tcp://:8001", "", false, log.TestingLogger()) + lAddr := l.ExternalAddress().String() + _, _, err := net.SplitHostPort(lAddr) + require.Nil(t, err) + spl := strings.Split(lAddr, ".") + require.Equal(t, len(spl), 4) + l.Stop() + } + + { + // Create a listener with set external ipv4 addr. + setExAddr := "8.8.8.8:8080" + l := NewDefaultListener("tcp://:8001", setExAddr, false, log.TestingLogger()) + lAddr := l.ExternalAddress().String() + require.Equal(t, lAddr, setExAddr) + l.Stop() + } + + { + // Invalid external addr causes panic + setExAddr := "awrlsckjnal:8080" + require.Panics(t, func() { NewDefaultListener("tcp://:8001", setExAddr, false, log.TestingLogger()) }) + } +} diff --git a/p2p/metrics.go b/p2p/metrics.go new file mode 100644 index 000000000..ab876ee7c --- /dev/null +++ b/p2p/metrics.go @@ -0,0 +1,33 @@ +package p2p + +import ( + "github.com/go-kit/kit/metrics" + "github.com/go-kit/kit/metrics/discard" + + prometheus "github.com/go-kit/kit/metrics/prometheus" + stdprometheus "github.com/prometheus/client_golang/prometheus" +) + +// Metrics contains metrics exposed by this package. +type Metrics struct { + // Number of peers. + Peers metrics.Gauge +} + +// PrometheusMetrics returns Metrics build using Prometheus client library. +func PrometheusMetrics() *Metrics { + return &Metrics{ + Peers: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Subsystem: "p2p", + Name: "peers", + Help: "Number of peers.", + }, []string{}), + } +} + +// NopMetrics returns no-op Metrics. +func NopMetrics() *Metrics { + return &Metrics{ + Peers: discard.NewGauge(), + } +} diff --git a/p2p/netaddress.go b/p2p/netaddress.go index 3e0d99d69..ebac8cc82 100644 --- a/p2p/netaddress.go +++ b/p2p/netaddress.go @@ -13,7 +13,7 @@ import ( "strings" "time" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) // NetAddress defines information about a peer on the network diff --git a/p2p/node_info.go b/p2p/node_info.go index 60383bc5e..5e8160a3b 100644 --- a/p2p/node_info.go +++ b/p2p/node_info.go @@ -2,7 +2,7 @@ package p2p import ( "fmt" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" "strings" ) diff --git a/p2p/peer.go b/p2p/peer.go index da69fe74f..5c615275b 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -6,9 +6,9 @@ import ( "sync/atomic" "time" - crypto "github.com/tendermint/go-crypto" - cmn "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tmlibs/log" + crypto "github.com/tendermint/tendermint/crypto" + cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/libs/log" "github.com/tendermint/tendermint/config" tmconn "github.com/tendermint/tendermint/p2p/conn" diff --git a/p2p/peer_set.go b/p2p/peer_set.go index e048cf4e3..257856156 100644 --- a/p2p/peer_set.go +++ b/p2p/peer_set.go @@ -55,8 +55,8 @@ func (ps *PeerSet) Add(peer Peer) error { return nil } -// Has returns true iff the PeerSet contains -// the peer referred to by this peerKey. +// Has returns true if the set contains the peer referred to by this +// peerKey, otherwise false. func (ps *PeerSet) Has(peerKey ID) bool { ps.mtx.Lock() _, ok := ps.lookup[peerKey] @@ -64,8 +64,8 @@ func (ps *PeerSet) Has(peerKey ID) bool { return ok } -// HasIP returns true if the PeerSet contains the peer referred to by this IP -// address. +// HasIP returns true if the set contains the peer referred to by this IP +// address, otherwise false. func (ps *PeerSet) HasIP(peerIP net.IP) bool { ps.mtx.Lock() defer ps.mtx.Unlock() @@ -85,7 +85,8 @@ func (ps *PeerSet) hasIP(peerIP net.IP) bool { return false } -// Get looks up a peer by the provided peerKey. +// Get looks up a peer by the provided peerKey. Returns nil if peer is not +// found. func (ps *PeerSet) Get(peerKey ID) Peer { ps.mtx.Lock() defer ps.mtx.Unlock() diff --git a/p2p/peer_set_test.go b/p2p/peer_set_test.go index 1675812fe..aa63ef949 100644 --- a/p2p/peer_set_test.go +++ b/p2p/peer_set_test.go @@ -8,8 +8,8 @@ import ( "github.com/stretchr/testify/assert" - crypto "github.com/tendermint/go-crypto" - cmn "github.com/tendermint/tmlibs/common" + crypto "github.com/tendermint/tendermint/crypto" + cmn "github.com/tendermint/tendermint/libs/common" ) // Returns an empty kvstore peer diff --git a/p2p/peer_test.go b/p2p/peer_test.go index 3a477199d..281b218d9 100644 --- a/p2p/peer_test.go +++ b/p2p/peer_test.go @@ -9,9 +9,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - crypto "github.com/tendermint/go-crypto" - cmn "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tmlibs/log" + crypto "github.com/tendermint/tendermint/crypto" + cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/libs/log" "github.com/tendermint/tendermint/config" tmconn "github.com/tendermint/tendermint/p2p/conn" diff --git a/p2p/pex/addrbook.go b/p2p/pex/addrbook.go index dc51761fe..421aa135a 100644 --- a/p2p/pex/addrbook.go +++ b/p2p/pex/addrbook.go @@ -12,9 +12,9 @@ import ( "sync" "time" - crypto "github.com/tendermint/go-crypto" + crypto "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/p2p" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) const ( diff --git a/p2p/pex/addrbook_test.go b/p2p/pex/addrbook_test.go index 2e2604286..dd983f76f 100644 --- a/p2p/pex/addrbook_test.go +++ b/p2p/pex/addrbook_test.go @@ -10,8 +10,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/tendermint/tendermint/p2p" - cmn "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tmlibs/log" + cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/libs/log" ) func createTempFileName(prefix string) string { diff --git a/p2p/pex/file.go b/p2p/pex/file.go index 38142dd9d..3237e1253 100644 --- a/p2p/pex/file.go +++ b/p2p/pex/file.go @@ -4,7 +4,7 @@ import ( "encoding/json" "os" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) /* Loading & Saving */ diff --git a/p2p/pex/pex_reactor.go b/p2p/pex/pex_reactor.go index 27ed422c5..2d93783df 100644 --- a/p2p/pex/pex_reactor.go +++ b/p2p/pex/pex_reactor.go @@ -8,7 +8,7 @@ import ( "time" amino "github.com/tendermint/go-amino" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/p2p/conn" @@ -77,10 +77,10 @@ type PEXReactor struct { attemptsToDial sync.Map // address (string) -> {number of attempts (int), last time dialed (time.Time)} } -func (pexR *PEXReactor) minReceiveRequestInterval() time.Duration { +func (r *PEXReactor) minReceiveRequestInterval() time.Duration { // NOTE: must be less than ensurePeersPeriod, otherwise we'll request // peers too quickly from others and they'll think we're bad! - return pexR.ensurePeersPeriod / 3 + return r.ensurePeersPeriod / 3 } // PEXReactorConfig holds reactor specific configuration data. @@ -628,7 +628,9 @@ func (r *PEXReactor) crawlPeers() { } // Ask for more addresses peer := r.Switch.Peers().Get(pi.Addr.ID) - r.RequestAddrs(peer) + if peer != nil { + r.RequestAddrs(peer) + } } } diff --git a/p2p/pex/pex_reactor_test.go b/p2p/pex/pex_reactor_test.go index 645e93171..6d6e91c38 100644 --- a/p2p/pex/pex_reactor_test.go +++ b/p2p/pex/pex_reactor_test.go @@ -12,9 +12,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - crypto "github.com/tendermint/go-crypto" - cmn "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tmlibs/log" + crypto "github.com/tendermint/tendermint/crypto" + cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/libs/log" "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/p2p" @@ -109,7 +109,7 @@ func TestPEXReactorRunning(t *testing.T) { addOtherNodeAddrToAddrBook(2, 1) for i, sw := range switches { - sw.AddListener(p2p.NewDefaultListener("tcp", sw.NodeInfo().ListenAddr, true, logger.With("pex", i))) + sw.AddListener(p2p.NewDefaultListener("tcp://"+sw.NodeInfo().ListenAddr, "", false, logger.With("pex", i))) err := sw.Start() // start switch and reactors require.Nil(t, err) @@ -229,12 +229,7 @@ func TestPEXReactorUsesSeedsIfNeeded(t *testing.T) { }, ) seed.AddListener( - p2p.NewDefaultListener( - "tcp", - seed.NodeInfo().ListenAddr, - true, - log.TestingLogger(), - ), + p2p.NewDefaultListener("tcp://"+seed.NodeInfo().ListenAddr, "", false, log.TestingLogger()), ) require.Nil(t, seed.Start()) defer seed.Stop() diff --git a/p2p/switch.go b/p2p/switch.go index f1ceee5c6..d1e2ef23b 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -9,7 +9,7 @@ import ( "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/p2p/conn" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) const ( @@ -73,10 +73,15 @@ type Switch struct { mConfig conn.MConnConfig rng *cmn.Rand // seed for randomizing dial times and orders + + metrics *Metrics } +// SwitchOption sets an optional parameter on the Switch. +type SwitchOption func(*Switch) + // NewSwitch creates a new Switch with the given config. -func NewSwitch(cfg *config.P2PConfig) *Switch { +func NewSwitch(cfg *config.P2PConfig, options ...SwitchOption) *Switch { sw := &Switch{ config: cfg, reactors: make(map[string]Reactor), @@ -85,6 +90,7 @@ func NewSwitch(cfg *config.P2PConfig) *Switch { peers: NewPeerSet(), dialing: cmn.NewCMap(), reconnecting: cmn.NewCMap(), + metrics: NopMetrics(), } // Ensure we have a completely undeterministic PRNG. @@ -99,9 +105,19 @@ func NewSwitch(cfg *config.P2PConfig) *Switch { sw.mConfig = mConfig sw.BaseService = *cmn.NewBaseService(nil, "P2P Switch", sw) + + for _, option := range options { + option(sw) + } + return sw } +// WithMetrics sets the metrics. +func WithMetrics(metrics *Metrics) SwitchOption { + return func(sw *Switch) { sw.metrics = metrics } +} + //--------------------------------------------------------------------- // Switch setup @@ -279,6 +295,7 @@ func (sw *Switch) StopPeerGracefully(peer Peer) { func (sw *Switch) stopAndRemovePeer(peer Peer, reason interface{}) { sw.peers.Remove(peer) + sw.metrics.Peers.Add(float64(-1)) peer.Stop() for _, reactor := range sw.reactors { reactor.RemovePeer(peer, reason) @@ -623,6 +640,7 @@ func (sw *Switch) addPeer(pc peerConn) error { if err := sw.peers.Add(peer); err != nil { return err } + sw.metrics.Peers.Add(float64(1)) sw.Logger.Info("Added peer", "peer", peer) return nil diff --git a/p2p/switch_test.go b/p2p/switch_test.go index 6157f45c6..97539112e 100644 --- a/p2p/switch_test.go +++ b/p2p/switch_test.go @@ -11,8 +11,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - crypto "github.com/tendermint/go-crypto" - "github.com/tendermint/tmlibs/log" + crypto "github.com/tendermint/tendermint/crypto" + "github.com/tendermint/tendermint/libs/log" "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/p2p/conn" diff --git a/p2p/test_util.go b/p2p/test_util.go index 0d2ba6c5e..467532f0f 100644 --- a/p2p/test_util.go +++ b/p2p/test_util.go @@ -4,9 +4,9 @@ import ( "fmt" "net" - crypto "github.com/tendermint/go-crypto" - cmn "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tmlibs/log" + crypto "github.com/tendermint/tendermint/crypto" + cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/libs/log" "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/p2p/conn" diff --git a/p2p/trust/metric.go b/p2p/trust/metric.go index 5770b4208..c0175a93f 100644 --- a/p2p/trust/metric.go +++ b/p2p/trust/metric.go @@ -8,7 +8,7 @@ import ( "sync" "time" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) //--------------------------------------------------------------------------------------- @@ -256,12 +256,13 @@ func (tm *TrustMetric) SetTicker(ticker MetricTicker) { // Copy returns a new trust metric with members containing the same values func (tm *TrustMetric) Copy() *TrustMetric { - tm.mtx.Lock() - defer tm.mtx.Unlock() if tm == nil { return nil } + tm.mtx.Lock() + defer tm.mtx.Unlock() + return &TrustMetric{ proportionalWeight: tm.proportionalWeight, integralWeight: tm.integralWeight, diff --git a/p2p/trust/metric_test.go b/p2p/trust/metric_test.go index 98ea99ab4..f690ce557 100644 --- a/p2p/trust/metric_test.go +++ b/p2p/trust/metric_test.go @@ -56,6 +56,14 @@ func TestTrustMetricConfig(t *testing.T) { tm.Wait() } +func TestTrustMetricCopyNilPointer(t *testing.T) { + var tm *TrustMetric + + ctm := tm.Copy() + + assert.Nil(t, ctm) +} + // XXX: This test fails non-deterministically func _TestTrustMetricStopPause(t *testing.T) { // The TestTicker will provide manual control over diff --git a/p2p/trust/store.go b/p2p/trust/store.go index bbb4592a4..31f659a43 100644 --- a/p2p/trust/store.go +++ b/p2p/trust/store.go @@ -8,8 +8,8 @@ import ( "sync" "time" - cmn "github.com/tendermint/tmlibs/common" - dbm "github.com/tendermint/tmlibs/db" + cmn "github.com/tendermint/tendermint/libs/common" + dbm "github.com/tendermint/tendermint/libs/db" ) const defaultStorePeriodicSaveInterval = 1 * time.Minute diff --git a/p2p/trust/store_test.go b/p2p/trust/store_test.go index 4e5553961..e1bea8636 100644 --- a/p2p/trust/store_test.go +++ b/p2p/trust/store_test.go @@ -10,8 +10,8 @@ import ( "testing" "github.com/stretchr/testify/assert" - dbm "github.com/tendermint/tmlibs/db" - "github.com/tendermint/tmlibs/log" + dbm "github.com/tendermint/tendermint/libs/db" + "github.com/tendermint/tendermint/libs/log" ) func TestTrustMetricStoreSaveLoad(t *testing.T) { diff --git a/p2p/upnp/probe.go b/p2p/upnp/probe.go index 55479415f..2de5e7905 100644 --- a/p2p/upnp/probe.go +++ b/p2p/upnp/probe.go @@ -5,8 +5,8 @@ import ( "net" "time" - cmn "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tmlibs/log" + cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/libs/log" ) type UPNPCapabilities struct { diff --git a/p2p/wire.go b/p2p/wire.go index a90ac851c..b7ae41253 100644 --- a/p2p/wire.go +++ b/p2p/wire.go @@ -2,7 +2,7 @@ package p2p import ( "github.com/tendermint/go-amino" - "github.com/tendermint/go-crypto" + "github.com/tendermint/tendermint/crypto" ) var cdc = amino.NewCodec() diff --git a/privval/priv_validator.go b/privval/priv_validator.go index 2bb5ef323..2abcf5590 100644 --- a/privval/priv_validator.go +++ b/privval/priv_validator.go @@ -8,9 +8,9 @@ import ( "sync" "time" - "github.com/tendermint/go-crypto" + "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/types" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) // TODO: type ? @@ -222,7 +222,10 @@ func (pv *FilePV) signVote(chainID string, vote *types.Vote) error { } // It passed the checks. Sign the vote - sig := pv.PrivKey.Sign(signBytes) + sig, err := pv.PrivKey.Sign(signBytes) + if err != nil { + return err + } pv.saveSigned(height, round, step, signBytes, sig) vote.Signature = sig return nil @@ -258,7 +261,10 @@ func (pv *FilePV) signProposal(chainID string, proposal *types.Proposal) error { } // It passed the checks. Sign the proposal - sig := pv.PrivKey.Sign(signBytes) + sig, err := pv.PrivKey.Sign(signBytes) + if err != nil { + return err + } pv.saveSigned(height, round, step, signBytes, sig) proposal.Signature = sig return nil @@ -281,7 +287,11 @@ func (pv *FilePV) saveSigned(height int64, round int, step int8, func (pv *FilePV) SignHeartbeat(chainID string, heartbeat *types.Heartbeat) error { pv.mtx.Lock() defer pv.mtx.Unlock() - heartbeat.Signature = pv.PrivKey.Sign(heartbeat.SignBytes(chainID)) + sig, err := pv.PrivKey.Sign(heartbeat.SignBytes(chainID)) + if err != nil { + return err + } + heartbeat.Signature = sig return nil } diff --git a/privval/priv_validator_test.go b/privval/priv_validator_test.go index c7212d594..5889c0d68 100644 --- a/privval/priv_validator_test.go +++ b/privval/priv_validator_test.go @@ -9,9 +9,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/tendermint/go-crypto" + "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/types" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) func TestGenLoadValidator(t *testing.T) { @@ -60,14 +60,14 @@ func TestUnmarshalValidator(t *testing.T) { serialized := fmt.Sprintf(`{ "address": "%s", "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "%s" }, - "last_height": 0, - "last_round": 0, + "last_height": "0", + "last_round": "0", "last_step": 0, "priv_key": { - "type": "954568A3288910", + "type": "tendermint/PrivKeyEd25519", "value": "%s" } }`, addr, pubB64, privB64) diff --git a/privval/socket.go b/privval/socket.go index 9f59a8152..1e8a3807b 100644 --- a/privval/socket.go +++ b/privval/socket.go @@ -8,9 +8,9 @@ import ( "time" "github.com/tendermint/go-amino" - "github.com/tendermint/go-crypto" - cmn "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tmlibs/log" + "github.com/tendermint/tendermint/crypto" + cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/libs/log" p2pconn "github.com/tendermint/tendermint/p2p/conn" "github.com/tendermint/tendermint/types" diff --git a/privval/socket_test.go b/privval/socket_test.go index 197c2508e..7bcacd6e1 100644 --- a/privval/socket_test.go +++ b/privval/socket_test.go @@ -9,9 +9,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/tendermint/go-crypto" - cmn "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tmlibs/log" + "github.com/tendermint/tendermint/crypto" + cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/libs/log" p2pconn "github.com/tendermint/tendermint/p2p/conn" "github.com/tendermint/tendermint/types" @@ -119,7 +119,7 @@ func TestSocketPVAcceptDeadline(t *testing.T) { SocketPVAcceptDeadline(time.Millisecond)(sc) - assert.Equal(t, sc.Start().(cmn.Error).Cause(), ErrConnWaitTimeout) + assert.Equal(t, sc.Start().(cmn.Error).Data(), ErrConnWaitTimeout) } func TestSocketPVDeadline(t *testing.T) { @@ -165,7 +165,7 @@ func TestSocketPVDeadline(t *testing.T) { time.Sleep(20 * time.Microsecond) _, err := sc.getPubKey() - assert.Equal(t, err.(cmn.Error).Cause(), ErrConnTimeout) + assert.Equal(t, err.(cmn.Error).Data(), ErrConnTimeout) } func TestSocketPVWait(t *testing.T) { @@ -178,7 +178,7 @@ func TestSocketPVWait(t *testing.T) { SocketPVConnWait(time.Millisecond)(sc) - assert.Equal(t, sc.Start().(cmn.Error).Cause(), ErrConnWaitTimeout) + assert.Equal(t, sc.Start().(cmn.Error).Data(), ErrConnWaitTimeout) } func TestRemoteSignerRetry(t *testing.T) { @@ -221,7 +221,7 @@ func TestRemoteSignerRetry(t *testing.T) { RemoteSignerConnDeadline(time.Millisecond)(rs) RemoteSignerConnRetries(retries)(rs) - assert.Equal(t, rs.Start().(cmn.Error).Cause(), ErrDialRetryMax) + assert.Equal(t, rs.Start().(cmn.Error).Data(), ErrDialRetryMax) select { case attempts := <-attemptc: diff --git a/privval/wire.go b/privval/wire.go index 688910834..c42ba40d6 100644 --- a/privval/wire.go +++ b/privval/wire.go @@ -2,7 +2,7 @@ package privval import ( "github.com/tendermint/go-amino" - "github.com/tendermint/go-crypto" + "github.com/tendermint/tendermint/crypto" ) var cdc = amino.NewCodec() diff --git a/proxy/app_conn.go b/proxy/app_conn.go index 2319fed82..2f792671e 100644 --- a/proxy/app_conn.go +++ b/proxy/app_conn.go @@ -1,8 +1,8 @@ package proxy import ( - abcicli "github.com/tendermint/abci/client" - "github.com/tendermint/abci/types" + abcicli "github.com/tendermint/tendermint/abci/client" + "github.com/tendermint/tendermint/abci/types" ) //---------------------------------------------------------------------------------------- diff --git a/proxy/app_conn_test.go b/proxy/app_conn_test.go index 7eb3831cd..3c556d4f0 100644 --- a/proxy/app_conn_test.go +++ b/proxy/app_conn_test.go @@ -4,12 +4,12 @@ import ( "strings" "testing" - abcicli "github.com/tendermint/abci/client" - "github.com/tendermint/abci/example/kvstore" - "github.com/tendermint/abci/server" - "github.com/tendermint/abci/types" - cmn "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tmlibs/log" + abcicli "github.com/tendermint/tendermint/abci/client" + "github.com/tendermint/tendermint/abci/example/kvstore" + "github.com/tendermint/tendermint/abci/server" + "github.com/tendermint/tendermint/abci/types" + cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/libs/log" ) //---------------------------------------- diff --git a/proxy/client.go b/proxy/client.go index 6c987368a..87f4e716d 100644 --- a/proxy/client.go +++ b/proxy/client.go @@ -5,9 +5,9 @@ import ( "github.com/pkg/errors" - abcicli "github.com/tendermint/abci/client" - "github.com/tendermint/abci/example/kvstore" - "github.com/tendermint/abci/types" + abcicli "github.com/tendermint/tendermint/abci/client" + "github.com/tendermint/tendermint/abci/example/kvstore" + "github.com/tendermint/tendermint/abci/types" ) // NewABCIClient returns newly connected client diff --git a/proxy/multi_app_conn.go b/proxy/multi_app_conn.go index 5d89ef195..279fa42ee 100644 --- a/proxy/multi_app_conn.go +++ b/proxy/multi_app_conn.go @@ -3,7 +3,7 @@ package proxy import ( "github.com/pkg/errors" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) //----------------------------- diff --git a/rpc/client/event_test.go b/rpc/client/event_test.go index 2254c1d1e..79c452fc9 100644 --- a/rpc/client/event_test.go +++ b/rpc/client/event_test.go @@ -7,10 +7,10 @@ import ( "github.com/stretchr/testify/require" - abci "github.com/tendermint/abci/types" + abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/rpc/client" "github.com/tendermint/tendermint/types" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) var waitForEventTimeout = 5 * time.Second diff --git a/rpc/client/httpclient.go b/rpc/client/httpclient.go index 1414edce3..4b85bf01d 100644 --- a/rpc/client/httpclient.go +++ b/rpc/client/httpclient.go @@ -11,7 +11,7 @@ import ( ctypes "github.com/tendermint/tendermint/rpc/core/types" rpcclient "github.com/tendermint/tendermint/rpc/lib/client" "github.com/tendermint/tendermint/types" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) /* @@ -29,12 +29,13 @@ type HTTP struct { *WSEvents } -// New takes a remote endpoint in the form tcp://: +// NewHTTP takes a remote endpoint in the form tcp://: // and the websocket path (which always seems to be "/websocket") func NewHTTP(remote, wsEndpoint string) *HTTP { rc := rpcclient.NewJSONRPCClient(remote) cdc := rc.Codec() ctypes.RegisterAmino(cdc) + rc.SetCodec(cdc) return &HTTP{ rpc: rc, diff --git a/rpc/client/interface.go b/rpc/client/interface.go index afe2d8fa0..f939c855b 100644 --- a/rpc/client/interface.go +++ b/rpc/client/interface.go @@ -23,7 +23,7 @@ implementation. import ( ctypes "github.com/tendermint/tendermint/rpc/core/types" "github.com/tendermint/tendermint/types" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) // ABCIClient groups together the functionality that principally diff --git a/rpc/client/localclient.go b/rpc/client/localclient.go index d89ec3b22..df3daf907 100644 --- a/rpc/client/localclient.go +++ b/rpc/client/localclient.go @@ -8,7 +8,7 @@ import ( "github.com/tendermint/tendermint/rpc/core" ctypes "github.com/tendermint/tendermint/rpc/core/types" "github.com/tendermint/tendermint/types" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) /* diff --git a/rpc/client/main_test.go b/rpc/client/main_test.go index 82b5a0195..1e911bbe6 100644 --- a/rpc/client/main_test.go +++ b/rpc/client/main_test.go @@ -4,7 +4,7 @@ import ( "os" "testing" - "github.com/tendermint/abci/example/kvstore" + "github.com/tendermint/tendermint/abci/example/kvstore" nm "github.com/tendermint/tendermint/node" rpctest "github.com/tendermint/tendermint/rpc/test" ) diff --git a/rpc/client/mock/abci.go b/rpc/client/mock/abci.go index 7f4c45df3..c8ca060c6 100644 --- a/rpc/client/mock/abci.go +++ b/rpc/client/mock/abci.go @@ -1,12 +1,12 @@ package mock import ( - abci "github.com/tendermint/abci/types" + abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/rpc/client" ctypes "github.com/tendermint/tendermint/rpc/core/types" "github.com/tendermint/tendermint/types" "github.com/tendermint/tendermint/version" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) // ABCIApp will send all abci related request to the named app, diff --git a/rpc/client/mock/abci_test.go b/rpc/client/mock/abci_test.go index 564f0129a..bcf443cf0 100644 --- a/rpc/client/mock/abci_test.go +++ b/rpc/client/mock/abci_test.go @@ -9,13 +9,13 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/tendermint/abci/example/kvstore" - abci "github.com/tendermint/abci/types" + "github.com/tendermint/tendermint/abci/example/kvstore" + abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/rpc/client" "github.com/tendermint/tendermint/rpc/client/mock" ctypes "github.com/tendermint/tendermint/rpc/core/types" "github.com/tendermint/tendermint/types" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) func TestABCIMock(t *testing.T) { diff --git a/rpc/client/mock/client.go b/rpc/client/mock/client.go index 6af9abb27..955df6277 100644 --- a/rpc/client/mock/client.go +++ b/rpc/client/mock/client.go @@ -20,7 +20,7 @@ import ( "github.com/tendermint/tendermint/rpc/core" ctypes "github.com/tendermint/tendermint/rpc/core/types" "github.com/tendermint/tendermint/types" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) // Client wraps arbitrary implementations of the various interfaces. diff --git a/rpc/client/mock/status_test.go b/rpc/client/mock/status_test.go index dafd35080..8e3c15061 100644 --- a/rpc/client/mock/status_test.go +++ b/rpc/client/mock/status_test.go @@ -8,7 +8,7 @@ import ( "github.com/tendermint/tendermint/rpc/client/mock" ctypes "github.com/tendermint/tendermint/rpc/core/types" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) func TestStatus(t *testing.T) { diff --git a/rpc/client/rpc_test.go b/rpc/client/rpc_test.go index 13109f786..e7e9042a7 100644 --- a/rpc/client/rpc_test.go +++ b/rpc/client/rpc_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - abci "github.com/tendermint/abci/types" + abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/rpc/client" rpctest "github.com/tendermint/tendermint/rpc/test" diff --git a/rpc/core/abci.go b/rpc/core/abci.go index 067108c44..a5eede3fc 100644 --- a/rpc/core/abci.go +++ b/rpc/core/abci.go @@ -1,10 +1,10 @@ package core import ( - abci "github.com/tendermint/abci/types" + abci "github.com/tendermint/tendermint/abci/types" ctypes "github.com/tendermint/tendermint/rpc/core/types" "github.com/tendermint/tendermint/version" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) // Query the application for some information. diff --git a/rpc/core/blocks.go b/rpc/core/blocks.go index a5ad5b4cb..0e8873152 100644 --- a/rpc/core/blocks.go +++ b/rpc/core/blocks.go @@ -6,7 +6,7 @@ import ( ctypes "github.com/tendermint/tendermint/rpc/core/types" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) // Get block headers for minHeight <= height <= maxHeight. diff --git a/rpc/core/consensus.go b/rpc/core/consensus.go index dad64b6b6..c026cd91f 100644 --- a/rpc/core/consensus.go +++ b/rpc/core/consensus.go @@ -87,7 +87,7 @@ func Validators(heightPtr *int64) (*ctypes.ResultValidators, error) { // { // "address": "B5B3D40BE53982AD294EF99FF5A34C0C3E5A3244", // "pub_key": { -// "type": "AC26791624DE60", +// "type": "tendermint/PubKeyEd25519", // "value": "SBctdhRBcXtBgdI/8a/alTsUhGXqGs9k5ylV1u5iKHg=" // }, // "voting_power": 10, @@ -97,7 +97,7 @@ func Validators(heightPtr *int64) (*ctypes.ResultValidators, error) { // "proposer": { // "address": "B5B3D40BE53982AD294EF99FF5A34C0C3E5A3244", // "pub_key": { -// "type": "AC26791624DE60", +// "type": "tendermint/PubKeyEd25519", // "value": "SBctdhRBcXtBgdI/8a/alTsUhGXqGs9k5ylV1u5iKHg=" // }, // "voting_power": 10, @@ -133,7 +133,7 @@ func Validators(heightPtr *int64) (*ctypes.ResultValidators, error) { // { // "address": "B5B3D40BE53982AD294EF99FF5A34C0C3E5A3244", // "pub_key": { -// "type": "AC26791624DE60", +// "type": "tendermint/PubKeyEd25519", // "value": "SBctdhRBcXtBgdI/8a/alTsUhGXqGs9k5ylV1u5iKHg=" // }, // "voting_power": 10, @@ -143,7 +143,7 @@ func Validators(heightPtr *int64) (*ctypes.ResultValidators, error) { // "proposer": { // "address": "B5B3D40BE53982AD294EF99FF5A34C0C3E5A3244", // "pub_key": { -// "type": "AC26791624DE60", +// "type": "tendermint/PubKeyEd25519", // "value": "SBctdhRBcXtBgdI/8a/alTsUhGXqGs9k5ylV1u5iKHg=" // }, // "voting_power": 10, diff --git a/rpc/core/mempool.go b/rpc/core/mempool.go index 515ada87c..ecc41ce12 100644 --- a/rpc/core/mempool.go +++ b/rpc/core/mempool.go @@ -7,10 +7,10 @@ import ( "github.com/pkg/errors" - abci "github.com/tendermint/abci/types" + abci "github.com/tendermint/tendermint/abci/types" ctypes "github.com/tendermint/tendermint/rpc/core/types" "github.com/tendermint/tendermint/types" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) //----------------------------------------------------------------------------- diff --git a/rpc/core/pipe.go b/rpc/core/pipe.go index 7a042362f..128b3e9a7 100644 --- a/rpc/core/pipe.go +++ b/rpc/core/pipe.go @@ -3,15 +3,15 @@ package core import ( "time" - crypto "github.com/tendermint/go-crypto" "github.com/tendermint/tendermint/consensus" + crypto "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/proxy" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/state/txindex" "github.com/tendermint/tendermint/types" - dbm "github.com/tendermint/tmlibs/db" - "github.com/tendermint/tmlibs/log" + dbm "github.com/tendermint/tendermint/libs/db" + "github.com/tendermint/tendermint/libs/log" ) const ( diff --git a/rpc/core/status.go b/rpc/core/status.go index 044c12899..63e62b2c7 100644 --- a/rpc/core/status.go +++ b/rpc/core/status.go @@ -7,7 +7,7 @@ import ( ctypes "github.com/tendermint/tendermint/rpc/core/types" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) // Get Tendermint status including node info, pubkey, latest block @@ -50,12 +50,12 @@ import ( // "latest_app_hash": "0000000000000000", // "latest_block_height": 231, // "latest_block_time": "2018-04-27T23:18:08.459766485-04:00", -// "syncing": false +// "catching_up": false // }, // "validator_info": { // "address": "5875562FF0FFDECC895C20E32FC14988952E99E7", // "pub_key": { -// "type": "AC26791624DE60", +// "type": "tendermint/PubKeyEd25519", // "value": "PpDJRUrLG2RgFqYYjawfn/AcAgacSXpLFrmfYYQnuzE=" // }, // "voting_power": 10 @@ -92,7 +92,7 @@ func Status() (*ctypes.ResultStatus, error) { LatestAppHash: latestAppHash, LatestBlockHeight: latestHeight, LatestBlockTime: latestBlockTime, - Syncing: consensusReactor.FastSync(), + CatchingUp: consensusReactor.FastSync(), }, ValidatorInfo: ctypes.ValidatorInfo{ Address: pubKey.Address(), diff --git a/rpc/core/tx.go b/rpc/core/tx.go index 2fa7825fd..f53d82f14 100644 --- a/rpc/core/tx.go +++ b/rpc/core/tx.go @@ -3,7 +3,7 @@ package core import ( "fmt" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" tmquery "github.com/tendermint/tendermint/libs/pubsub/query" ctypes "github.com/tendermint/tendermint/rpc/core/types" diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index 5b001d7d3..4fec416ed 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -5,9 +5,9 @@ import ( "strings" "time" - abci "github.com/tendermint/abci/types" - crypto "github.com/tendermint/go-crypto" - cmn "github.com/tendermint/tmlibs/common" + abci "github.com/tendermint/tendermint/abci/types" + crypto "github.com/tendermint/tendermint/crypto" + cmn "github.com/tendermint/tendermint/libs/common" "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/state" @@ -65,7 +65,7 @@ type SyncInfo struct { LatestAppHash cmn.HexBytes `json:"latest_app_hash"` LatestBlockHeight int64 `json:"latest_block_height"` LatestBlockTime time.Time `json:"latest_block_time"` - Syncing bool `json:"syncing"` + CatchingUp bool `json:"catching_up"` } // Info about the node's validator diff --git a/rpc/core/types/wire.go b/rpc/core/types/wire.go index 6648364b1..d3a31dc35 100644 --- a/rpc/core/types/wire.go +++ b/rpc/core/types/wire.go @@ -2,7 +2,7 @@ package core_types import ( "github.com/tendermint/go-amino" - "github.com/tendermint/go-crypto" + "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/types" ) diff --git a/rpc/grpc/api.go b/rpc/grpc/api.go index c0a920046..0b840e3e9 100644 --- a/rpc/grpc/api.go +++ b/rpc/grpc/api.go @@ -3,7 +3,7 @@ package core_grpc import ( "context" - abci "github.com/tendermint/abci/types" + abci "github.com/tendermint/tendermint/abci/types" core "github.com/tendermint/tendermint/rpc/core" ) diff --git a/rpc/grpc/client_server.go b/rpc/grpc/client_server.go index 80d736f57..c88989685 100644 --- a/rpc/grpc/client_server.go +++ b/rpc/grpc/client_server.go @@ -6,13 +6,21 @@ import ( "strings" "time" + "golang.org/x/net/netutil" "google.golang.org/grpc" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) -// Start the grpcServer in a go routine -func StartGRPCServer(protoAddr string) (net.Listener, error) { +// Config is an gRPC server configuration. +type Config struct { + MaxOpenConnections int +} + +// StartGRPCServer starts a new gRPC BroadcastAPIServer, listening on +// protoAddr, in a goroutine. Returns a listener and an error, if it fails to +// parse an address. +func StartGRPCServer(protoAddr string, config Config) (net.Listener, error) { parts := strings.SplitN(protoAddr, "://", 2) if len(parts) != 2 { return nil, fmt.Errorf("Invalid listen address for grpc server (did you forget a tcp:// prefix?) : %s", protoAddr) @@ -22,6 +30,9 @@ func StartGRPCServer(protoAddr string) (net.Listener, error) { if err != nil { return nil, err } + if config.MaxOpenConnections > 0 { + ln = netutil.LimitListener(ln, config.MaxOpenConnections) + } grpcServer := grpc.NewServer() RegisterBroadcastAPIServer(grpcServer, &broadcastAPI{}) @@ -30,7 +41,8 @@ func StartGRPCServer(protoAddr string) (net.Listener, error) { return ln, nil } -// Start the client by dialing the server +// StartGRPCClient dials the gRPC server using protoAddr and returns a new +// BroadcastAPIClient. func StartGRPCClient(protoAddr string) BroadcastAPIClient { conn, err := grpc.Dial(protoAddr, grpc.WithInsecure(), grpc.WithDialer(dialerFunc)) if err != nil { diff --git a/rpc/grpc/grpc_test.go b/rpc/grpc/grpc_test.go index 20b3ab7b7..fe979c549 100644 --- a/rpc/grpc/grpc_test.go +++ b/rpc/grpc/grpc_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/require" - "github.com/tendermint/abci/example/kvstore" + "github.com/tendermint/tendermint/abci/example/kvstore" "github.com/tendermint/tendermint/rpc/grpc" "github.com/tendermint/tendermint/rpc/test" ) diff --git a/rpc/grpc/types.pb.go b/rpc/grpc/types.pb.go index cf7a5ec71..be16b711a 100644 --- a/rpc/grpc/types.pb.go +++ b/rpc/grpc/types.pb.go @@ -19,7 +19,7 @@ package core_grpc import proto "github.com/golang/protobuf/proto" import fmt "fmt" import math "math" -import types "github.com/tendermint/abci/types" +import types "github.com/tendermint/tendermint/abci/types" import ( "context" diff --git a/rpc/grpc/types.proto b/rpc/grpc/types.proto index 354625948..d7980d5e0 100644 --- a/rpc/grpc/types.proto +++ b/rpc/grpc/types.proto @@ -1,7 +1,7 @@ syntax = "proto3"; package core_grpc; -import "github.com/tendermint/abci/types/types.proto"; +import "github.com/tendermint/tendermint/abci/types/types.proto"; //---------------------------------------- // Message types diff --git a/rpc/lib/client/args_test.go b/rpc/lib/client/args_test.go index 4442ac2bc..cb7a56bd5 100644 --- a/rpc/lib/client/args_test.go +++ b/rpc/lib/client/args_test.go @@ -26,7 +26,7 @@ func TestArgToJSON(t *testing.T) { }{ {[]byte("1234"), "0x31323334"}, {Tx("654"), "0x363534"}, - {Foo{7, "hello"}, `{"Bar":7,"Baz":"hello"}`}, + {Foo{7, "hello"}, `{"Bar":"7","Baz":"hello"}`}, } cdc := amino.NewCodec() diff --git a/rpc/lib/client/http_client.go b/rpc/lib/client/http_client.go index e26d8f274..bd440289b 100644 --- a/rpc/lib/client/http_client.go +++ b/rpc/lib/client/http_client.go @@ -17,6 +17,14 @@ import ( types "github.com/tendermint/tendermint/rpc/lib/types" ) +const ( + protoHTTP = "http" + protoHTTPS = "https" + protoWSS = "wss" + protoWS = "ws" + protoTCP = "tcp" +) + // HTTPClient is a common interface for JSONRPCClient and URIClient. type HTTPClient interface { Call(method string, params map[string]interface{}, result interface{}) (interface{}, error) @@ -25,29 +33,37 @@ type HTTPClient interface { } // TODO: Deprecate support for IP:PORT or /path/to/socket -func makeHTTPDialer(remoteAddr string) (string, func(string, string) (net.Conn, error)) { +func makeHTTPDialer(remoteAddr string) (string, string, func(string, string) (net.Conn, error)) { + // protocol to use for http operations, to support both http and https + clientProtocol := protoHTTP + parts := strings.SplitN(remoteAddr, "://", 2) var protocol, address string if len(parts) == 1 { // default to tcp if nothing specified - protocol, address = "tcp", remoteAddr + protocol, address = protoTCP, remoteAddr } else if len(parts) == 2 { protocol, address = parts[0], parts[1] } else { // return a invalid message msg := fmt.Sprintf("Invalid addr: %s", remoteAddr) - return msg, func(_ string, _ string) (net.Conn, error) { + return clientProtocol, msg, func(_ string, _ string) (net.Conn, error) { return nil, errors.New(msg) } } - // accept http as an alias for tcp - if protocol == "http" { - protocol = "tcp" + + // accept http as an alias for tcp and set the client protocol + switch protocol { + case protoHTTP, protoHTTPS: + clientProtocol = protocol + protocol = protoTCP + case protoWS, protoWSS: + clientProtocol = protocol } // replace / with . for http requests (kvstore domain) trimmedAddress := strings.Replace(address, "/", ".", -1) - return trimmedAddress, func(proto, addr string) (net.Conn, error) { + return clientProtocol, trimmedAddress, func(proto, addr string) (net.Conn, error) { return net.Dial(protocol, address) } } @@ -55,8 +71,8 @@ func makeHTTPDialer(remoteAddr string) (string, func(string, string) (net.Conn, // We overwrite the http.Client.Dial so we can do http over tcp or unix. // remoteAddr should be fully featured (eg. with tcp:// or unix://) func makeHTTPClient(remoteAddr string) (string, *http.Client) { - address, dialer := makeHTTPDialer(remoteAddr) - return "http://" + address, &http.Client{ + protocol, address, dialer := makeHTTPDialer(remoteAddr) + return protocol + "://" + address, &http.Client{ Transport: &http.Transport{ Dial: dialer, }, diff --git a/rpc/lib/client/integration_test.go b/rpc/lib/client/integration_test.go index d3d993374..93a32388f 100644 --- a/rpc/lib/client/integration_test.go +++ b/rpc/lib/client/integration_test.go @@ -14,7 +14,7 @@ import ( "time" "github.com/stretchr/testify/require" - "github.com/tendermint/tmlibs/log" + "github.com/tendermint/tendermint/libs/log" ) func TestWSClientReconnectWithJitter(t *testing.T) { diff --git a/rpc/lib/client/ws_client.go b/rpc/lib/client/ws_client.go index a95ce17d2..9a07c8676 100644 --- a/rpc/lib/client/ws_client.go +++ b/rpc/lib/client/ws_client.go @@ -15,7 +15,7 @@ import ( "github.com/tendermint/go-amino" types "github.com/tendermint/tendermint/rpc/lib/types" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) const ( @@ -70,13 +70,21 @@ type WSClient struct { // Send pings to server with this period. Must be less than readWait. If 0, no pings will be sent. pingPeriod time.Duration + + // Support both ws and wss protocols + protocol string } // NewWSClient returns a new client. See the commentary on the func(*WSClient) // functions for a detailed description of how to configure ping period and // pong wait time. The endpoint argument must begin with a `/`. func NewWSClient(remoteAddr, endpoint string, options ...func(*WSClient)) *WSClient { - addr, dialer := makeHTTPDialer(remoteAddr) + protocol, addr, dialer := makeHTTPDialer(remoteAddr) + // default to ws protocol, unless wss is explicitly specified + if protocol != "wss" { + protocol = "ws" + } + c := &WSClient{ cdc: amino.NewCodec(), Address: addr, @@ -88,6 +96,7 @@ func NewWSClient(remoteAddr, endpoint string, options ...func(*WSClient)) *WSCli readWait: defaultReadWait, writeWait: defaultWriteWait, pingPeriod: defaultPingPeriod, + protocol: protocol, } c.BaseService = *cmn.NewBaseService(nil, "WSClient", c) for _, option := range options { @@ -242,7 +251,7 @@ func (c *WSClient) dial() error { Proxy: http.ProxyFromEnvironment, } rHeader := http.Header{} - conn, _, err := dialer.Dial("ws://"+c.Address+c.Endpoint, rHeader) + conn, _, err := dialer.Dial(c.protocol+"://"+c.Address+c.Endpoint, rHeader) if err != nil { return err } diff --git a/rpc/lib/client/ws_client_test.go b/rpc/lib/client/ws_client_test.go index 73f671609..e902fe21a 100644 --- a/rpc/lib/client/ws_client_test.go +++ b/rpc/lib/client/ws_client_test.go @@ -12,7 +12,7 @@ import ( "github.com/gorilla/websocket" "github.com/stretchr/testify/require" - "github.com/tendermint/tmlibs/log" + "github.com/tendermint/tendermint/libs/log" types "github.com/tendermint/tendermint/rpc/lib/types" ) diff --git a/rpc/lib/rpc_test.go b/rpc/lib/rpc_test.go index f34b09f68..31839dcab 100644 --- a/rpc/lib/rpc_test.go +++ b/rpc/lib/rpc_test.go @@ -17,9 +17,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/tendermint/go-amino" - cmn "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tmlibs/log" + amino "github.com/tendermint/go-amino" + cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/libs/log" client "github.com/tendermint/tendermint/rpc/lib/client" server "github.com/tendermint/tendermint/rpc/lib/server" @@ -123,7 +123,7 @@ func setup() { wm.SetLogger(tcpLogger) mux.HandleFunc(websocketEndpoint, wm.WebsocketHandler) go func() { - _, err := server.StartHTTPServer(tcpAddr, mux, tcpLogger) + _, err := server.StartHTTPServer(tcpAddr, mux, tcpLogger, server.Config{}) if err != nil { panic(err) } @@ -136,7 +136,7 @@ func setup() { wm.SetLogger(unixLogger) mux2.HandleFunc(websocketEndpoint, wm.WebsocketHandler) go func() { - _, err := server.StartHTTPServer(unixAddr, mux2, unixLogger) + _, err := server.StartHTTPServer(unixAddr, mux2, unixLogger, server.Config{}) if err != nil { panic(err) } @@ -274,18 +274,18 @@ func TestServersAndClientsBasic(t *testing.T) { serverAddrs := [...]string{tcpAddr, unixAddr} for _, addr := range serverAddrs { cl1 := client.NewURIClient(addr) - fmt.Printf("=== testing server on %s using %v client", addr, cl1) + fmt.Printf("=== testing server on %s using URI client", addr) testWithHTTPClient(t, cl1) cl2 := client.NewJSONRPCClient(addr) - fmt.Printf("=== testing server on %s using %v client", addr, cl2) + fmt.Printf("=== testing server on %s using JSONRPC client", addr) testWithHTTPClient(t, cl2) cl3 := client.NewWSClient(addr, websocketEndpoint) cl3.SetLogger(log.TestingLogger()) err := cl3.Start() require.Nil(t, err) - fmt.Printf("=== testing server on %s using %v client", addr, cl3) + fmt.Printf("=== testing server on %s using WS client", addr) testWithWSClient(t, cl3) cl3.Stop() } diff --git a/rpc/lib/server/handlers.go b/rpc/lib/server/handlers.go index 6cc03012a..1bfe52536 100644 --- a/rpc/lib/server/handlers.go +++ b/rpc/lib/server/handlers.go @@ -19,8 +19,8 @@ import ( amino "github.com/tendermint/go-amino" types "github.com/tendermint/tendermint/rpc/lib/types" - cmn "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tmlibs/log" + cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/libs/log" ) // RegisterRPCFuncs adds a route for each function in the funcMap, as well as general jsonrpc and websocket handlers for all functions. @@ -406,7 +406,13 @@ type wsConnection struct { // description of how to configure ping period and pong wait time. NOTE: if the // write buffer is full, pongs may be dropped, which may cause clients to // disconnect. see https://github.com/gorilla/websocket/issues/97 -func NewWSConnection(baseConn *websocket.Conn, funcMap map[string]*RPCFunc, cdc *amino.Codec, options ...func(*wsConnection)) *wsConnection { +func NewWSConnection( + baseConn *websocket.Conn, + funcMap map[string]*RPCFunc, + cdc *amino.Codec, + options ...func(*wsConnection), +) *wsConnection { + baseConn.SetReadLimit(maxBodyBytes) wsc := &wsConnection{ remoteAddr: baseConn.RemoteAddr().String(), baseConn: baseConn, diff --git a/rpc/lib/server/handlers_test.go b/rpc/lib/server/handlers_test.go index b1ea46754..3471eb791 100644 --- a/rpc/lib/server/handlers_test.go +++ b/rpc/lib/server/handlers_test.go @@ -16,7 +16,7 @@ import ( amino "github.com/tendermint/go-amino" rs "github.com/tendermint/tendermint/rpc/lib/server" types "github.com/tendermint/tendermint/rpc/lib/types" - "github.com/tendermint/tmlibs/log" + "github.com/tendermint/tendermint/libs/log" ) ////////////////////////////////////////////////////////////////////////////// @@ -55,13 +55,13 @@ func TestRPCParams(t *testing.T) { {`{"jsonrpc": "2.0", "method": "y", "id": "0"}`, "Method not found"}, {`{"method": "c", "id": "0", "params": a}`, "invalid character"}, {`{"method": "c", "id": "0", "params": ["a"]}`, "got 1"}, - {`{"method": "c", "id": "0", "params": ["a", "b"]}`, "of type int"}, + {`{"method": "c", "id": "0", "params": ["a", "b"]}`, "invalid character"}, {`{"method": "c", "id": "0", "params": [1, 1]}`, "of type string"}, // good {`{"jsonrpc": "2.0", "method": "c", "id": "0", "params": null}`, ""}, {`{"method": "c", "id": "0", "params": {}}`, ""}, - {`{"method": "c", "id": "0", "params": ["a", 10]}`, ""}, + {`{"method": "c", "id": "0", "params": ["a", "10"]}`, ""}, } for i, tt := range tests { diff --git a/rpc/lib/server/http_server.go b/rpc/lib/server/http_server.go index 3f54c61ef..5d816ef22 100644 --- a/rpc/lib/server/http_server.go +++ b/rpc/lib/server/http_server.go @@ -12,16 +12,38 @@ import ( "time" "github.com/pkg/errors" + "golang.org/x/net/netutil" types "github.com/tendermint/tendermint/rpc/lib/types" - "github.com/tendermint/tmlibs/log" + "github.com/tendermint/tendermint/libs/log" ) -func StartHTTPServer(listenAddr string, handler http.Handler, logger log.Logger) (listener net.Listener, err error) { +// Config is an RPC server configuration. +type Config struct { + MaxOpenConnections int +} + +const ( + // maxBodyBytes controls the maximum number of bytes the + // server will read parsing the request body. + maxBodyBytes = int64(1000000) // 1MB +) + +// StartHTTPServer starts an HTTP server on listenAddr with the given handler. +// It wraps handler with RecoverAndLogHandler. +func StartHTTPServer( + listenAddr string, + handler http.Handler, + logger log.Logger, + config Config, +) (listener net.Listener, err error) { var proto, addr string parts := strings.SplitN(listenAddr, "://", 2) if len(parts) != 2 { - return nil, errors.Errorf("Invalid listening address %s (use fully formed addresses, including the tcp:// or unix:// prefix)", listenAddr) + return nil, errors.Errorf( + "Invalid listening address %s (use fully formed addresses, including the tcp:// or unix:// prefix)", + listenAddr, + ) } proto, addr = parts[0], parts[1] @@ -30,35 +52,60 @@ func StartHTTPServer(listenAddr string, handler http.Handler, logger log.Logger) if err != nil { return nil, errors.Errorf("Failed to listen on %v: %v", listenAddr, err) } + if config.MaxOpenConnections > 0 { + listener = netutil.LimitListener(listener, config.MaxOpenConnections) + } go func() { err := http.Serve( listener, - RecoverAndLogHandler(handler, logger), + RecoverAndLogHandler(maxBytesHandler{h: handler, n: maxBodyBytes}, logger), ) logger.Error("RPC HTTP server stopped", "err", err) }() return listener, nil } -func StartHTTPAndTLSServer(listenAddr string, handler http.Handler, certFile, keyFile string, logger log.Logger) (listener net.Listener, err error) { +// StartHTTPAndTLSServer starts an HTTPS server on listenAddr with the given +// handler. +// It wraps handler with RecoverAndLogHandler. +func StartHTTPAndTLSServer( + listenAddr string, + handler http.Handler, + certFile, keyFile string, + logger log.Logger, + config Config, +) (listener net.Listener, err error) { var proto, addr string parts := strings.SplitN(listenAddr, "://", 2) if len(parts) != 2 { - return nil, errors.Errorf("Invalid listening address %s (use fully formed addresses, including the tcp:// or unix:// prefix)", listenAddr) + return nil, errors.Errorf( + "Invalid listening address %s (use fully formed addresses, including the tcp:// or unix:// prefix)", + listenAddr, + ) } proto, addr = parts[0], parts[1] - logger.Info(fmt.Sprintf("Starting RPC HTTPS server on %s (cert: %q, key: %q)", listenAddr, certFile, keyFile)) + logger.Info( + fmt.Sprintf( + "Starting RPC HTTPS server on %s (cert: %q, key: %q)", + listenAddr, + certFile, + keyFile, + ), + ) listener, err = net.Listen(proto, addr) if err != nil { return nil, errors.Errorf("Failed to listen on %v: %v", listenAddr, err) } + if config.MaxOpenConnections > 0 { + listener = netutil.LimitListener(listener, config.MaxOpenConnections) + } go func() { err := http.ServeTLS( listener, - RecoverAndLogHandler(handler, logger), + RecoverAndLogHandler(maxBytesHandler{h: handler, n: maxBodyBytes}, logger), certFile, keyFile, ) @@ -67,7 +114,11 @@ func StartHTTPAndTLSServer(listenAddr string, handler http.Handler, certFile, ke return listener, nil } -func WriteRPCResponseHTTPError(w http.ResponseWriter, httpCode int, res types.RPCResponse) { +func WriteRPCResponseHTTPError( + w http.ResponseWriter, + httpCode int, + res types.RPCResponse, +) { jsonBytes, err := json.MarshalIndent(res, "", " ") if err != nil { panic(err) @@ -117,7 +168,10 @@ func RecoverAndLogHandler(handler http.Handler, logger log.Logger) http.Handler WriteRPCResponseHTTP(rww, res) } else { // For the rest, - logger.Error("Panic in RPC HTTP handler", "err", e, "stack", string(debug.Stack())) + logger.Error( + "Panic in RPC HTTP handler", "err", e, "stack", + string(debug.Stack()), + ) rww.WriteHeader(http.StatusInternalServerError) WriteRPCResponseHTTP(rww, types.RPCInternalError("", e.(error))) } @@ -154,3 +208,13 @@ func (w *ResponseWriterWrapper) WriteHeader(status int) { func (w *ResponseWriterWrapper) Hijack() (net.Conn, *bufio.ReadWriter, error) { return w.ResponseWriter.(http.Hijacker).Hijack() } + +type maxBytesHandler struct { + h http.Handler + n int64 +} + +func (h maxBytesHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + r.Body = http.MaxBytesReader(w, r.Body, h.n) + h.h.ServeHTTP(w, r) +} diff --git a/rpc/lib/server/http_server_test.go b/rpc/lib/server/http_server_test.go new file mode 100644 index 000000000..3cbe0d906 --- /dev/null +++ b/rpc/lib/server/http_server_test.go @@ -0,0 +1,62 @@ +package rpcserver + +import ( + "fmt" + "io" + "io/ioutil" + "net/http" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/tendermint/tendermint/libs/log" +) + +func TestMaxOpenConnections(t *testing.T) { + const max = 5 // max simultaneous connections + + // Start the server. + var open int32 + mux := http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + if n := atomic.AddInt32(&open, 1); n > int32(max) { + t.Errorf("%d open connections, want <= %d", n, max) + } + defer atomic.AddInt32(&open, -1) + time.Sleep(10 * time.Millisecond) + fmt.Fprint(w, "some body") + }) + l, err := StartHTTPServer("tcp://127.0.0.1:0", mux, log.TestingLogger(), Config{MaxOpenConnections: max}) + if err != nil { + t.Fatal(err) + } + defer l.Close() + + // Make N GET calls to the server. + attempts := max * 2 + var wg sync.WaitGroup + var failed int32 + for i := 0; i < attempts; i++ { + wg.Add(1) + go func() { + defer wg.Done() + c := http.Client{Timeout: 3 * time.Second} + r, err := c.Get("http://" + l.Addr().String()) + if err != nil { + t.Log(err) + atomic.AddInt32(&failed, 1) + return + } + defer r.Body.Close() + io.Copy(ioutil.Discard, r.Body) + }() + } + wg.Wait() + + // We expect some Gets to fail as the server's accept queue is filled, + // but most should succeed. + if int(failed) >= attempts/2 { + t.Errorf("%d requests failed within %d attempts", failed, attempts) + } +} diff --git a/rpc/lib/server/parse_test.go b/rpc/lib/server/parse_test.go index f4323ef5d..f71316906 100644 --- a/rpc/lib/server/parse_test.go +++ b/rpc/lib/server/parse_test.go @@ -6,8 +6,8 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/tendermint/go-amino" - cmn "github.com/tendermint/tmlibs/common" + amino "github.com/tendermint/go-amino" + cmn "github.com/tendermint/tendermint/libs/common" ) func TestParseJSONMap(t *testing.T) { @@ -148,8 +148,8 @@ func TestParseRPC(t *testing.T) { fail bool }{ // should parse - {`[7, "flew"]`, 7, "flew", false}, - {`{"name": "john", "height": 22}`, 22, "john", false}, + {`["7", "flew"]`, 7, "flew", false}, + {`{"name": "john", "height": "22"}`, 22, "john", false}, // defaults {`{"name": "solo", "unused": "stuff"}`, 0, "solo", false}, // should fail - wrong types/length diff --git a/rpc/lib/test/main.go b/rpc/lib/test/main.go index 604cbd3d8..cb9560e12 100644 --- a/rpc/lib/test/main.go +++ b/rpc/lib/test/main.go @@ -5,10 +5,10 @@ import ( "net/http" "os" - "github.com/tendermint/go-amino" + amino "github.com/tendermint/go-amino" rpcserver "github.com/tendermint/tendermint/rpc/lib/server" - cmn "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tmlibs/log" + cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/libs/log" ) var routes = map[string]*rpcserver.RPCFunc{ @@ -28,7 +28,7 @@ func main() { cdc := amino.NewCodec() logger := log.NewTMLogger(log.NewSyncWriter(os.Stdout)) rpcserver.RegisterRPCFuncs(mux, routes, cdc, logger) - _, err := rpcserver.StartHTTPServer("0.0.0.0:8008", mux, logger) + _, err := rpcserver.StartHTTPServer("0.0.0.0:8008", mux, logger, rpcserver.Config{}) if err != nil { cmn.Exit(err.Error()) } diff --git a/rpc/test/helpers.go b/rpc/test/helpers.go index 5d18299bc..915911818 100644 --- a/rpc/test/helpers.go +++ b/rpc/test/helpers.go @@ -8,10 +8,10 @@ import ( "strings" "time" - "github.com/tendermint/tmlibs/log" + "github.com/tendermint/tendermint/libs/log" - abci "github.com/tendermint/abci/types" - cmn "github.com/tendermint/tmlibs/common" + abci "github.com/tendermint/tendermint/abci/types" + cmn "github.com/tendermint/tendermint/libs/common" cfg "github.com/tendermint/tendermint/config" nm "github.com/tendermint/tendermint/node" @@ -122,7 +122,9 @@ func NewTendermint(app abci.Application) *nm.Node { papp := proxy.NewLocalClientCreator(app) node, err := nm.NewNode(config, pv, papp, nm.DefaultGenesisDocProviderFunc(config), - nm.DefaultDBProvider, logger) + nm.DefaultDBProvider, + nm.DefaultMetricsProvider, + logger) if err != nil { panic(err) } diff --git a/scripts/wire2amino.go b/scripts/wire2amino.go index a942ceaad..867c5735a 100644 --- a/scripts/wire2amino.go +++ b/scripts/wire2amino.go @@ -9,8 +9,8 @@ import ( "time" "github.com/tendermint/go-amino" - crypto "github.com/tendermint/go-crypto" - cmn "github.com/tendermint/tmlibs/common" + crypto "github.com/tendermint/tendermint/crypto" + cmn "github.com/tendermint/tendermint/libs/common" "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/privval" diff --git a/state/errors.go b/state/errors.go index afb5737d7..d40c7e141 100644 --- a/state/errors.go +++ b/state/errors.go @@ -1,7 +1,7 @@ package state import ( - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) type ( diff --git a/state/execution.go b/state/execution.go index e6b94429e..1c0af17a8 100644 --- a/state/execution.go +++ b/state/execution.go @@ -4,11 +4,11 @@ import ( "fmt" fail "github.com/ebuchman/fail-test" - abci "github.com/tendermint/abci/types" + abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/proxy" "github.com/tendermint/tendermint/types" - dbm "github.com/tendermint/tmlibs/db" - "github.com/tendermint/tmlibs/log" + dbm "github.com/tendermint/tendermint/libs/db" + "github.com/tendermint/tendermint/libs/log" ) //----------------------------------------------------------------------------- diff --git a/state/execution_test.go b/state/execution_test.go index b520b0c12..9c0635dcc 100644 --- a/state/execution_test.go +++ b/state/execution_test.go @@ -8,12 +8,12 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/tendermint/abci/example/kvstore" - abci "github.com/tendermint/abci/types" - crypto "github.com/tendermint/go-crypto" - cmn "github.com/tendermint/tmlibs/common" - dbm "github.com/tendermint/tmlibs/db" - "github.com/tendermint/tmlibs/log" + "github.com/tendermint/tendermint/abci/example/kvstore" + abci "github.com/tendermint/tendermint/abci/types" + crypto "github.com/tendermint/tendermint/crypto" + cmn "github.com/tendermint/tendermint/libs/common" + dbm "github.com/tendermint/tendermint/libs/db" + "github.com/tendermint/tendermint/libs/log" "github.com/tendermint/tendermint/proxy" "github.com/tendermint/tendermint/types" diff --git a/state/services.go b/state/services.go index bef286b20..bf0b1a6f4 100644 --- a/state/services.go +++ b/state/services.go @@ -1,7 +1,7 @@ package state import ( - abci "github.com/tendermint/abci/types" + abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/types" ) diff --git a/state/state_test.go b/state/state_test.go index 464456ca5..bf0c910fa 100644 --- a/state/state_test.go +++ b/state/state_test.go @@ -7,10 +7,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - abci "github.com/tendermint/abci/types" - crypto "github.com/tendermint/go-crypto" - cmn "github.com/tendermint/tmlibs/common" - dbm "github.com/tendermint/tmlibs/db" + abci "github.com/tendermint/tendermint/abci/types" + crypto "github.com/tendermint/tendermint/crypto" + cmn "github.com/tendermint/tendermint/libs/common" + dbm "github.com/tendermint/tendermint/libs/db" cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/types" diff --git a/state/store.go b/state/store.go index 2164d699d..8db0689de 100644 --- a/state/store.go +++ b/state/store.go @@ -3,10 +3,10 @@ package state import ( "fmt" - abci "github.com/tendermint/abci/types" + abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/types" - cmn "github.com/tendermint/tmlibs/common" - dbm "github.com/tendermint/tmlibs/db" + cmn "github.com/tendermint/tendermint/libs/common" + dbm "github.com/tendermint/tendermint/libs/db" ) //------------------------------------------------------------------------ diff --git a/state/txindex/indexer_service.go b/state/txindex/indexer_service.go index 264be1fd8..088252f5e 100644 --- a/state/txindex/indexer_service.go +++ b/state/txindex/indexer_service.go @@ -3,7 +3,7 @@ package txindex import ( "context" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" "github.com/tendermint/tendermint/types" ) diff --git a/state/txindex/kv/kv.go b/state/txindex/kv/kv.go index 718a55d15..707325929 100644 --- a/state/txindex/kv/kv.go +++ b/state/txindex/kv/kv.go @@ -10,8 +10,8 @@ import ( "time" "github.com/pkg/errors" - cmn "github.com/tendermint/tmlibs/common" - dbm "github.com/tendermint/tmlibs/db" + cmn "github.com/tendermint/tendermint/libs/common" + dbm "github.com/tendermint/tendermint/libs/db" "github.com/tendermint/tendermint/libs/pubsub/query" "github.com/tendermint/tendermint/state/txindex" diff --git a/state/txindex/kv/kv_test.go b/state/txindex/kv/kv_test.go index af35ec411..1272f4a73 100644 --- a/state/txindex/kv/kv_test.go +++ b/state/txindex/kv/kv_test.go @@ -8,9 +8,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - abci "github.com/tendermint/abci/types" - cmn "github.com/tendermint/tmlibs/common" - db "github.com/tendermint/tmlibs/db" + abci "github.com/tendermint/tendermint/abci/types" + cmn "github.com/tendermint/tendermint/libs/common" + db "github.com/tendermint/tendermint/libs/db" "github.com/tendermint/tendermint/libs/pubsub/query" "github.com/tendermint/tendermint/state/txindex" diff --git a/state/validation.go b/state/validation.go index 84a4cc824..c36339203 100644 --- a/state/validation.go +++ b/state/validation.go @@ -6,7 +6,7 @@ import ( "fmt" "github.com/tendermint/tendermint/types" - dbm "github.com/tendermint/tmlibs/db" + dbm "github.com/tendermint/tendermint/libs/db" ) //----------------------------------------------------- diff --git a/state/validation_test.go b/state/validation_test.go index b4695b077..362a40737 100644 --- a/state/validation_test.go +++ b/state/validation_test.go @@ -4,8 +4,8 @@ import ( "testing" "github.com/stretchr/testify/require" - dbm "github.com/tendermint/tmlibs/db" - "github.com/tendermint/tmlibs/log" + dbm "github.com/tendermint/tendermint/libs/db" + "github.com/tendermint/tendermint/libs/log" ) func TestValidateBlock(t *testing.T) { diff --git a/state/wire.go b/state/wire.go index 3e8b544d9..af743c7b8 100644 --- a/state/wire.go +++ b/state/wire.go @@ -2,7 +2,7 @@ package state import ( "github.com/tendermint/go-amino" - "github.com/tendermint/go-crypto" + "github.com/tendermint/tendermint/crypto" ) var cdc = amino.NewCodec() diff --git a/test/p2p/atomic_broadcast/test.sh b/test/p2p/atomic_broadcast/test.sh index 4e3cc1d86..f066707d3 100644 --- a/test/p2p/atomic_broadcast/test.sh +++ b/test/p2p/atomic_broadcast/test.sh @@ -26,11 +26,11 @@ for i in $(seq 1 "$N"); do echo "" # we need to wait another block to get the new app_hash - h1=$(curl -s "$addr/status" | jq .result.sync_info.latest_block_height) + h1=$(curl -s "$addr/status" | jq .result.sync_info.latest_block_height | jq fromjson) h2=$h1 while [ "$h2" == "$h1" ]; do sleep 1 - h2=$(curl -s "$addr/status" | jq .result.sync_info.latest_block_height) + h2=$(curl -s "$addr/status" | jq .result.sync_info.latest_block_height | jq fromjson) done # wait for all other peers to get to this height @@ -39,10 +39,10 @@ for i in $(seq 1 "$N"); do if [[ "$i" != "$j" ]]; then addrJ=$(test/p2p/ip.sh "$j"):26657 - h=$(curl -s "$addrJ/status" | jq .result.sync_info.latest_block_height) + h=$(curl -s "$addrJ/status" | jq .result.sync_info.latest_block_height | jq fromjson) while [ "$h" -lt "$minHeight" ]; do sleep 1 - h=$(curl -s "$addrJ/status" | jq .result.sync_info.latest_block_height) + h=$(curl -s "$addrJ/status" | jq .result.sync_info.latest_block_height | jq fromjson) done fi done diff --git a/test/p2p/basic/test.sh b/test/p2p/basic/test.sh index 6b356db2d..caf665122 100755 --- a/test/p2p/basic/test.sh +++ b/test/p2p/basic/test.sh @@ -54,12 +54,12 @@ for i in `seq 1 $N`; do done # - assert block height is greater than 1 - BLOCK_HEIGHT=`curl -s $addr/status | jq .result.sync_info.latest_block_height` + BLOCK_HEIGHT=`curl -s $addr/status | jq .result.sync_info.latest_block_height | jq fromjson` COUNT=0 while [ "$BLOCK_HEIGHT" -le 1 ]; do echo "Waiting for node $i to commit a block ..." sleep 1 - BLOCK_HEIGHT=`curl -s $addr/status | jq .result.sync_info.latest_block_height` + BLOCK_HEIGHT=`curl -s $addr/status | jq .result.sync_info.latest_block_height | jq fromjson` COUNT=$((COUNT+1)) if [ "$COUNT" -gt "$MAX_SLEEP" ]; then echo "Waited too long for node $i to commit a block" diff --git a/test/p2p/data/mach1/core/config/genesis.json b/test/p2p/data/mach1/core/config/genesis.json index 19577db00..515c10714 100644 --- a/test/p2p/data/mach1/core/config/genesis.json +++ b/test/p2p/data/mach1/core/config/genesis.json @@ -4,34 +4,34 @@ "validators": [ { "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "vokz3/FgDAJuNHGPF4Wkzeq5DDVpizlOOLaUeukd4RY=" }, - "power": 1, + "power": "1", "name": "mach1" }, { "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "bcU0RlMjEmWH0qKpO1nWibcXBzsd6WiiWm7xPVlTGK0=" }, - "power": 1, + "power": "1", "name": "mach2" }, { "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "rmesaX0TWqC0YB6lfqqz/r9Lqk8inEWlmMKYWxL80aE=" }, - "power": 1, + "power": "1", "name": "mach3" }, { "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "nryPWM7UtG3NWrirpZHdJTzXy1A3Jz/aMrwLZGHE79k=" }, - "power": 1, + "power": "1", "name": "mach4" } ], diff --git a/test/p2p/data/mach1/core/config/node_key.json b/test/p2p/data/mach1/core/config/node_key.json index c6d65008f..4fa960850 100644 --- a/test/p2p/data/mach1/core/config/node_key.json +++ b/test/p2p/data/mach1/core/config/node_key.json @@ -1,6 +1,6 @@ { "priv_key": { - "type": "954568A3288910", + "type": "tendermint/PrivKeyEd25519", "value": "BpYtFp8xSrudBa5aBLRuSPD72PGDAUm0dJORDL3Kd5YJbluUzRefVFrjwoHZv1yeDj2P9xkEi2L3hJCUz/qFkQ==" } } diff --git a/test/p2p/data/mach1/core/config/priv_validator.json b/test/p2p/data/mach1/core/config/priv_validator.json index 08c7c503b..ea2a01f5c 100644 --- a/test/p2p/data/mach1/core/config/priv_validator.json +++ b/test/p2p/data/mach1/core/config/priv_validator.json @@ -1,14 +1,14 @@ { - "address": "7E9D1FB08EDBAFCF116638D4C8FAFAEE2ABE1AAA", + "address": "AE47BBD4B3ACD80BFE17F6E0C66C5B8492A81AE4", "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "vokz3/FgDAJuNHGPF4Wkzeq5DDVpizlOOLaUeukd4RY=" }, - "last_height": 0, - "last_round": 0, + "last_height": "0", + "last_round": "0", "last_step": 0, "priv_key": { - "type": "954568A3288910", + "type": "tendermint/PrivKeyEd25519", "value": "VHqgfHqM4WxcsqQMbCbRWwoylgQQqfHqblC2NvGrOJq+iTPf8WAMAm40cY8XhaTN6rkMNWmLOU44tpR66R3hFg==" } } diff --git a/test/p2p/data/mach2/core/config/genesis.json b/test/p2p/data/mach2/core/config/genesis.json index 19577db00..515c10714 100644 --- a/test/p2p/data/mach2/core/config/genesis.json +++ b/test/p2p/data/mach2/core/config/genesis.json @@ -4,34 +4,34 @@ "validators": [ { "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "vokz3/FgDAJuNHGPF4Wkzeq5DDVpizlOOLaUeukd4RY=" }, - "power": 1, + "power": "1", "name": "mach1" }, { "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "bcU0RlMjEmWH0qKpO1nWibcXBzsd6WiiWm7xPVlTGK0=" }, - "power": 1, + "power": "1", "name": "mach2" }, { "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "rmesaX0TWqC0YB6lfqqz/r9Lqk8inEWlmMKYWxL80aE=" }, - "power": 1, + "power": "1", "name": "mach3" }, { "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "nryPWM7UtG3NWrirpZHdJTzXy1A3Jz/aMrwLZGHE79k=" }, - "power": 1, + "power": "1", "name": "mach4" } ], diff --git a/test/p2p/data/mach2/core/config/node_key.json b/test/p2p/data/mach2/core/config/node_key.json index 146a1328d..6eb151106 100644 --- a/test/p2p/data/mach2/core/config/node_key.json +++ b/test/p2p/data/mach2/core/config/node_key.json @@ -1,6 +1,6 @@ { "priv_key": { - "type": "954568A3288910", + "type": "tendermint/PrivKeyEd25519", "value": "uM6LDVE4wQIIUmq9rc6RxzX8zEGG4G4Jcuw15klzQopF68YfJM4bkbPSavurEcJ4nvBMusKBg2GcARFrZqnFKA==" } } diff --git a/test/p2p/data/mach2/core/config/priv_validator.json b/test/p2p/data/mach2/core/config/priv_validator.json index 8e813dff4..6e0cd7f8f 100644 --- a/test/p2p/data/mach2/core/config/priv_validator.json +++ b/test/p2p/data/mach2/core/config/priv_validator.json @@ -1,14 +1,14 @@ { - "address": "8893D14FE09F1157E39CD34B98036048D51B4985", + "address": "5D61EE46CCE91F579086522D7FD8CEC3F854E946", "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "bcU0RlMjEmWH0qKpO1nWibcXBzsd6WiiWm7xPVlTGK0=" }, - "last_height": 0, - "last_round": 0, + "last_height": "0", + "last_round": "0", "last_step": 0, "priv_key": { - "type": "954568A3288910", + "type": "tendermint/PrivKeyEd25519", "value": "0EeInmBQL8MSnQq38zSxg47Z7R7Nmcu5a3GtWr9agUNtxTRGUyMSZYfSoqk7WdaJtxcHOx3paKJabvE9WVMYrQ==" } } diff --git a/test/p2p/data/mach3/core/config/genesis.json b/test/p2p/data/mach3/core/config/genesis.json index 19577db00..515c10714 100644 --- a/test/p2p/data/mach3/core/config/genesis.json +++ b/test/p2p/data/mach3/core/config/genesis.json @@ -4,34 +4,34 @@ "validators": [ { "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "vokz3/FgDAJuNHGPF4Wkzeq5DDVpizlOOLaUeukd4RY=" }, - "power": 1, + "power": "1", "name": "mach1" }, { "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "bcU0RlMjEmWH0qKpO1nWibcXBzsd6WiiWm7xPVlTGK0=" }, - "power": 1, + "power": "1", "name": "mach2" }, { "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "rmesaX0TWqC0YB6lfqqz/r9Lqk8inEWlmMKYWxL80aE=" }, - "power": 1, + "power": "1", "name": "mach3" }, { "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "nryPWM7UtG3NWrirpZHdJTzXy1A3Jz/aMrwLZGHE79k=" }, - "power": 1, + "power": "1", "name": "mach4" } ], diff --git a/test/p2p/data/mach3/core/config/node_key.json b/test/p2p/data/mach3/core/config/node_key.json index 82689b8e0..0885bcf9c 100644 --- a/test/p2p/data/mach3/core/config/node_key.json +++ b/test/p2p/data/mach3/core/config/node_key.json @@ -1,6 +1,6 @@ { "priv_key": { - "type": "954568A3288910", + "type": "tendermint/PrivKeyEd25519", "value": "kT3orG0YkipT9rAZbvAjtGk/7Pu1ZeCE8LSUF2jz2uiSs1rdlUVi/gccRlvCRLKvrtSicOyEkmk0FHPOGS3mgg==" } } diff --git a/test/p2p/data/mach3/core/config/priv_validator.json b/test/p2p/data/mach3/core/config/priv_validator.json index 84c98b98f..ec68ca7bb 100644 --- a/test/p2p/data/mach3/core/config/priv_validator.json +++ b/test/p2p/data/mach3/core/config/priv_validator.json @@ -1,14 +1,14 @@ { - "address": "7C747D7E002932B3864E3FBE9AC04287043F66A0", + "address": "705F9DA2CC7D7AF5F4519455ED99622E40E439A1", "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "rmesaX0TWqC0YB6lfqqz/r9Lqk8inEWlmMKYWxL80aE=" }, - "last_height": 0, - "last_round": 0, + "last_height": "0", + "last_round": "0", "last_step": 0, "priv_key": { - "type": "954568A3288910", + "type": "tendermint/PrivKeyEd25519", "value": "waTkfzSfxfVW9Kmie6d2uUQkwxK6ps9u5EuGc0jXw/KuZ6xpfRNaoLRgHqV+qrP+v0uqTyKcRaWYwphbEvzRoQ==" } } diff --git a/test/p2p/data/mach4/core/config/genesis.json b/test/p2p/data/mach4/core/config/genesis.json index 19577db00..515c10714 100644 --- a/test/p2p/data/mach4/core/config/genesis.json +++ b/test/p2p/data/mach4/core/config/genesis.json @@ -4,34 +4,34 @@ "validators": [ { "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "vokz3/FgDAJuNHGPF4Wkzeq5DDVpizlOOLaUeukd4RY=" }, - "power": 1, + "power": "1", "name": "mach1" }, { "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "bcU0RlMjEmWH0qKpO1nWibcXBzsd6WiiWm7xPVlTGK0=" }, - "power": 1, + "power": "1", "name": "mach2" }, { "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "rmesaX0TWqC0YB6lfqqz/r9Lqk8inEWlmMKYWxL80aE=" }, - "power": 1, + "power": "1", "name": "mach3" }, { "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "nryPWM7UtG3NWrirpZHdJTzXy1A3Jz/aMrwLZGHE79k=" }, - "power": 1, + "power": "1", "name": "mach4" } ], diff --git a/test/p2p/data/mach4/core/config/node_key.json b/test/p2p/data/mach4/core/config/node_key.json index a0c8d3917..d6a5d79c2 100644 --- a/test/p2p/data/mach4/core/config/node_key.json +++ b/test/p2p/data/mach4/core/config/node_key.json @@ -1,6 +1,6 @@ { "priv_key": { - "type": "954568A3288910", + "type": "tendermint/PrivKeyEd25519", "value": "QIIm8/QEEawiJi3Zozv+J9b+1CufCEkGs3lxGMlRy4L4FVIXCoXJTwYIrotZtwoMqLYEqQV1hbKKJmFA3GFelw==" } } diff --git a/test/p2p/data/mach4/core/config/priv_validator.json b/test/p2p/data/mach4/core/config/priv_validator.json index 4f88045b5..468550ea8 100644 --- a/test/p2p/data/mach4/core/config/priv_validator.json +++ b/test/p2p/data/mach4/core/config/priv_validator.json @@ -1,14 +1,14 @@ { - "address": "CEBEFE3CA1363D425643EF63FC179E77A50A1E9A", + "address": "D1054266EC9EEA511ED9A76DEFD520BBE1B5E850", "pub_key": { - "type": "AC26791624DE60", + "type": "tendermint/PubKeyEd25519", "value": "nryPWM7UtG3NWrirpZHdJTzXy1A3Jz/aMrwLZGHE79k=" }, - "last_height": 0, - "last_round": 0, + "last_height": "0", + "last_round": "0", "last_step": 0, "priv_key": { - "type": "954568A3288910", + "type": "tendermint/PrivKeyEd25519", "value": "xMw+0o8CDC29qYvNvwjDztNwRw508l6TjV0pXo49KwyevI9YztS0bc1auKulkd0lPNfLUDcnP9oyvAtkYcTv2Q==" } } diff --git a/test/p2p/fast_sync/check_peer.sh b/test/p2p/fast_sync/check_peer.sh index e69f977f7..d5d3fc2b5 100644 --- a/test/p2p/fast_sync/check_peer.sh +++ b/test/p2p/fast_sync/check_peer.sh @@ -15,7 +15,7 @@ peerID=$(( $(($ID % 4)) + 1 )) # 1->2 ... 3->4 ... 4->1 peer_addr=$(test/p2p/ip.sh $peerID):26657 # get another peer's height -h1=`curl -s $peer_addr/status | jq .result.sync_info.latest_block_height` +h1=`curl -s $peer_addr/status | jq .result.sync_info.latest_block_height | jq fromjson` # get another peer's state root1=`curl -s $peer_addr/status | jq .result.sync_info.latest_app_hash` @@ -29,7 +29,7 @@ set +o pipefail h2="0" while [[ "$h2" -lt "$(($h1+3))" ]]; do sleep 1 - h2=`curl -s $addr/status | jq .result.sync_info.latest_block_height` + h2=`curl -s $addr/status | jq .result.sync_info.latest_block_height | jq fromjson` echo "... $h2" done diff --git a/test/test_cover.sh b/test/test_cover.sh index 59ce15b03..5f2dea3ee 100644 --- a/test/test_cover.sh +++ b/test/test_cover.sh @@ -6,7 +6,7 @@ set -e echo "mode: atomic" > coverage.txt for pkg in ${PKGS[@]}; do - go test -v -timeout 30m -race -coverprofile=profile.out -covermode=atomic "$pkg" + go test -timeout 5m -race -coverprofile=profile.out -covermode=atomic "$pkg" if [ -f profile.out ]; then tail -n +2 profile.out >> coverage.txt; rm profile.out diff --git a/types/block.go b/types/block.go index 3004672c8..bc018ee89 100644 --- a/types/block.go +++ b/types/block.go @@ -8,9 +8,9 @@ import ( "sync" "time" - cmn "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tmlibs/merkle" - "golang.org/x/crypto/ripemd160" + "github.com/tendermint/tendermint/crypto/merkle" + "github.com/tendermint/tendermint/crypto/tmhash" + cmn "github.com/tendermint/tendermint/libs/common" ) // Block defines the atomic unit of a Tendermint blockchain. @@ -135,6 +135,15 @@ func (b *Block) HashesTo(hash []byte) bool { return bytes.Equal(b.Hash(), hash) } +// Size returns size of the block in bytes. +func (b *Block) Size() int { + bz, err := cdc.MarshalBinaryBare(b) + if err != nil { + return 0 + } + return len(bz) +} + // String returns a string representation of the block func (b *Block) String() string { return b.StringIndented("") @@ -455,7 +464,7 @@ func (data *Data) StringIndented(indent string) string { txStrings[i] = fmt.Sprintf("... (%v total)", len(data.Txs)) break } - txStrings[i] = fmt.Sprintf("Tx:%v", tx) + txStrings[i] = fmt.Sprintf("%X (%d bytes)", tx.Hash(), len(tx)) } return fmt.Sprintf(`Data{ %s %v @@ -495,7 +504,7 @@ func (data *EvidenceData) StringIndented(indent string) string { } evStrings[i] = fmt.Sprintf("Evidence:%v", ev) } - return fmt.Sprintf(`Data{ + return fmt.Sprintf(`EvidenceData{ %s %v %s}#%v`, indent, strings.Join(evStrings, "\n"+indent+" "), @@ -543,7 +552,7 @@ type hasher struct { } func (h hasher) Hash() []byte { - hasher := ripemd160.New() + hasher := tmhash.New() if h.item != nil && !cmn.IsTypedNil(h.item) && !cmn.IsEmpty(h.item) { bz, err := cdc.MarshalBinaryBare(h.item) if err != nil { diff --git a/types/block_test.go b/types/block_test.go index 3c2942cd5..0948e7b21 100644 --- a/types/block_test.go +++ b/types/block_test.go @@ -6,8 +6,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - crypto "github.com/tendermint/go-crypto" - cmn "github.com/tendermint/tmlibs/common" + crypto "github.com/tendermint/tendermint/crypto" + cmn "github.com/tendermint/tendermint/libs/common" ) func TestValidateBlock(t *testing.T) { diff --git a/types/canonical_json.go b/types/canonical_json.go index 95ade9c67..189a8a7a2 100644 --- a/types/canonical_json.go +++ b/types/canonical_json.go @@ -3,14 +3,13 @@ package types import ( "time" - "github.com/tendermint/go-amino" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) // Canonical json is amino's json for structs with fields in alphabetical order // TimeFormat is used for generating the sigs -const TimeFormat = amino.RFC3339Millis +const TimeFormat = "2006-01-02T15:04:05.000Z" type CanonicalJSONBlockID struct { Hash cmn.HexBytes `json:"hash,omitempty"` diff --git a/types/event_bus.go b/types/event_bus.go index cb4b17d51..54fc60c7b 100644 --- a/types/event_bus.go +++ b/types/event_bus.go @@ -5,8 +5,8 @@ import ( "fmt" tmpubsub "github.com/tendermint/tendermint/libs/pubsub" - cmn "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tmlibs/log" + cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/libs/log" ) const defaultCapacity = 0 diff --git a/types/event_bus_test.go b/types/event_bus_test.go index d9b0995b9..81903004d 100644 --- a/types/event_bus_test.go +++ b/types/event_bus_test.go @@ -10,10 +10,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - abci "github.com/tendermint/abci/types" + abci "github.com/tendermint/tendermint/abci/types" tmpubsub "github.com/tendermint/tendermint/libs/pubsub" tmquery "github.com/tendermint/tendermint/libs/pubsub/query" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) func TestEventBusPublishEventTx(t *testing.T) { diff --git a/types/evidence.go b/types/evidence.go index 10907869e..266375ec3 100644 --- a/types/evidence.go +++ b/types/evidence.go @@ -5,8 +5,9 @@ import ( "fmt" "github.com/tendermint/go-amino" - "github.com/tendermint/go-crypto" - "github.com/tendermint/tmlibs/merkle" + + "github.com/tendermint/tendermint/crypto" + "github.com/tendermint/tendermint/crypto/merkle" ) // ErrEvidenceInvalid wraps a piece of evidence and the error denoting how or why it is invalid. @@ -180,7 +181,7 @@ type EvidenceList []Evidence // Hash returns the simple merkle root hash of the EvidenceList. func (evl EvidenceList) Hash() []byte { // Recursive impl. - // Copied from tmlibs/merkle to avoid allocations + // Copied from crypto/merkle to avoid allocations switch len(evl) { case 0: return nil diff --git a/types/genesis.go b/types/genesis.go index aee8e0767..0367c6b2f 100644 --- a/types/genesis.go +++ b/types/genesis.go @@ -5,8 +5,8 @@ import ( "io/ioutil" "time" - "github.com/tendermint/go-crypto" - cmn "github.com/tendermint/tmlibs/common" + "github.com/tendermint/tendermint/crypto" + cmn "github.com/tendermint/tendermint/libs/common" ) //------------------------------------------------------------ diff --git a/types/genesis_test.go b/types/genesis_test.go index bed4b90f5..24398a9a5 100644 --- a/types/genesis_test.go +++ b/types/genesis_test.go @@ -4,7 +4,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/tendermint/go-crypto" + "github.com/tendermint/tendermint/crypto" ) func TestGenesisBad(t *testing.T) { @@ -18,7 +18,7 @@ func TestGenesisBad(t *testing.T) { []byte(`{"chain_id":"mychain","validators":[{}]}`), // missing validators []byte(`{"chain_id":"mychain","validators":null}`), // missing validators []byte(`{"chain_id":"mychain"}`), // missing validators - []byte(`{"validators":[{"pub_key":{"type":"AC26791624DE60","value":"AT/+aaL1eB0477Mud9JMm8Sh8BIvOYlPGC9KkIUmFaE="},"power":10,"name":""}]}`), // missing chain_id + []byte(`{"validators":[{"pub_key":{"type":"tendermint/PubKeyEd25519","value":"AT/+aaL1eB0477Mud9JMm8Sh8BIvOYlPGC9KkIUmFaE="},"power":"10","name":""}]}`), // missing chain_id } for _, testCase := range testCases { @@ -29,7 +29,7 @@ func TestGenesisBad(t *testing.T) { func TestGenesisGood(t *testing.T) { // test a good one by raw json - genDocBytes := []byte(`{"genesis_time":"0001-01-01T00:00:00Z","chain_id":"test-chain-QDKdJr","consensus_params":null,"validators":[{"pub_key":{"type":"AC26791624DE60","value":"AT/+aaL1eB0477Mud9JMm8Sh8BIvOYlPGC9KkIUmFaE="},"power":10,"name":""}],"app_hash":"","app_state":{"account_owner": "Bob"}}`) + genDocBytes := []byte(`{"genesis_time":"0001-01-01T00:00:00Z","chain_id":"test-chain-QDKdJr","consensus_params":null,"validators":[{"pub_key":{"type":"tendermint/PubKeyEd25519","value":"AT/+aaL1eB0477Mud9JMm8Sh8BIvOYlPGC9KkIUmFaE="},"power":"10","name":""}],"app_hash":"","app_state":{"account_owner": "Bob"}}`) _, err := GenesisDocFromJSON(genDocBytes) assert.NoError(t, err, "expected no error for good genDoc json") diff --git a/types/heartbeat.go b/types/heartbeat.go index 097dd22db..cebe2864c 100644 --- a/types/heartbeat.go +++ b/types/heartbeat.go @@ -3,8 +3,8 @@ package types import ( "fmt" - "github.com/tendermint/go-crypto" - cmn "github.com/tendermint/tmlibs/common" + "github.com/tendermint/tendermint/crypto" + cmn "github.com/tendermint/tendermint/libs/common" ) // Heartbeat is a simple vote-like structure so validators can diff --git a/types/heartbeat_test.go b/types/heartbeat_test.go index 3c4536028..174c3ba97 100644 --- a/types/heartbeat_test.go +++ b/types/heartbeat_test.go @@ -5,7 +5,7 @@ import ( "github.com/stretchr/testify/require" - "github.com/tendermint/go-crypto" + "github.com/tendermint/tendermint/crypto" ) func TestHeartbeatCopy(t *testing.T) { @@ -28,7 +28,9 @@ func TestHeartbeatString(t *testing.T) { require.Equal(t, hb.String(), "Heartbeat{1:000000000000 11/02 (0) }") var key crypto.PrivKeyEd25519 - hb.Signature = key.Sign([]byte("Tendermint")) + sig, err := key.Sign([]byte("Tendermint")) + require.NoError(t, err) + hb.Signature = sig require.Equal(t, hb.String(), "Heartbeat{1:000000000000 11/02 (0) /FF41E371B9BF.../}") } @@ -37,11 +39,11 @@ func TestHeartbeatWriteSignBytes(t *testing.T) { hb := &Heartbeat{ValidatorIndex: 1, Height: 10, Round: 1} bz := hb.SignBytes("0xdeadbeef") // XXX HMMMMMMM - require.Equal(t, string(bz), `{"@chain_id":"0xdeadbeef","@type":"heartbeat","height":10,"round":1,"sequence":0,"validator_address":"","validator_index":1}`) + require.Equal(t, string(bz), `{"@chain_id":"0xdeadbeef","@type":"heartbeat","height":"10","round":"1","sequence":"0","validator_address":"","validator_index":"1"}`) plainHb := &Heartbeat{} bz = plainHb.SignBytes("0xdeadbeef") - require.Equal(t, string(bz), `{"@chain_id":"0xdeadbeef","@type":"heartbeat","height":0,"round":0,"sequence":0,"validator_address":"","validator_index":0}`) + require.Equal(t, string(bz), `{"@chain_id":"0xdeadbeef","@type":"heartbeat","height":"0","round":"0","sequence":"0","validator_address":"","validator_index":"0"}`) require.Panics(t, func() { var nilHb *Heartbeat diff --git a/types/params.go b/types/params.go index 2df092d62..3056c82a0 100644 --- a/types/params.go +++ b/types/params.go @@ -1,12 +1,13 @@ package types import ( - abci "github.com/tendermint/abci/types" - cmn "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tmlibs/merkle" + abci "github.com/tendermint/tendermint/abci/types" + "github.com/tendermint/tendermint/crypto/merkle" + cmn "github.com/tendermint/tendermint/libs/common" ) const ( + // MaxBlockSizeBytes is the maximum permitted size of the blocks. MaxBlockSizeBytes = 104857600 // 100MB ) @@ -56,7 +57,7 @@ func DefaultConsensusParams() *ConsensusParams { func DefaultBlockSize() BlockSize { return BlockSize{ MaxBytes: 22020096, // 21MB - MaxTxs: 100000, + MaxTxs: 10000, MaxGas: -1, } } diff --git a/types/part_set.go b/types/part_set.go index 18cfe802c..f6d7f6b6e 100644 --- a/types/part_set.go +++ b/types/part_set.go @@ -7,10 +7,9 @@ import ( "io" "sync" - "golang.org/x/crypto/ripemd160" - - cmn "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tmlibs/merkle" + "github.com/tendermint/tendermint/crypto/merkle" + "github.com/tendermint/tendermint/crypto/tmhash" + cmn "github.com/tendermint/tendermint/libs/common" ) var ( @@ -31,7 +30,7 @@ func (part *Part) Hash() []byte { if part.hash != nil { return part.hash } - hasher := ripemd160.New() + hasher := tmhash.New() hasher.Write(part.Bytes) // nolint: errcheck, gas part.hash = hasher.Sum(nil) return part.hash diff --git a/types/part_set_test.go b/types/part_set_test.go index 545b4d42b..01437f05e 100644 --- a/types/part_set_test.go +++ b/types/part_set_test.go @@ -5,7 +5,7 @@ import ( "io/ioutil" "testing" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) const ( diff --git a/types/priv_validator.go b/types/priv_validator.go index 8759d3f99..85db65a41 100644 --- a/types/priv_validator.go +++ b/types/priv_validator.go @@ -4,7 +4,7 @@ import ( "bytes" "fmt" - "github.com/tendermint/go-crypto" + "github.com/tendermint/tendermint/crypto" ) // PrivValidator defines the functionality of a local Tendermint validator @@ -63,7 +63,10 @@ func (pv *MockPV) GetPubKey() crypto.PubKey { // Implements PrivValidator. func (pv *MockPV) SignVote(chainID string, vote *Vote) error { signBytes := vote.SignBytes(chainID) - sig := pv.privKey.Sign(signBytes) + sig, err := pv.privKey.Sign(signBytes) + if err != nil { + return err + } vote.Signature = sig return nil } @@ -71,14 +74,20 @@ func (pv *MockPV) SignVote(chainID string, vote *Vote) error { // Implements PrivValidator. func (pv *MockPV) SignProposal(chainID string, proposal *Proposal) error { signBytes := proposal.SignBytes(chainID) - sig := pv.privKey.Sign(signBytes) + sig, err := pv.privKey.Sign(signBytes) + if err != nil { + return err + } proposal.Signature = sig return nil } // signHeartbeat signs the heartbeat without any checking. func (pv *MockPV) SignHeartbeat(chainID string, heartbeat *Heartbeat) error { - sig := pv.privKey.Sign(heartbeat.SignBytes(chainID)) + sig, err := pv.privKey.Sign(heartbeat.SignBytes(chainID)) + if err != nil { + return err + } heartbeat.Signature = sig return nil } diff --git a/types/proposal.go b/types/proposal.go index 95008897b..52ce8756e 100644 --- a/types/proposal.go +++ b/types/proposal.go @@ -5,7 +5,7 @@ import ( "fmt" "time" - "github.com/tendermint/go-crypto" + "github.com/tendermint/tendermint/crypto" ) var ( diff --git a/types/proposal_test.go b/types/proposal_test.go index 43fb7c209..8aef870fc 100644 --- a/types/proposal_test.go +++ b/types/proposal_test.go @@ -27,7 +27,11 @@ func TestProposalSignable(t *testing.T) { signBytes := testProposal.SignBytes("test_chain_id") signStr := string(signBytes) - expected := `{"@chain_id":"test_chain_id","@type":"proposal","block_parts_header":{"hash":"626C6F636B7061727473","total":111},"height":12345,"pol_block_id":{},"pol_round":-1,"round":23456,"timestamp":"2018-02-11T07:09:22.765Z"}` + expected := `{"@chain_id":"test_chain_id","@type":"proposal","block_parts_header":{"hash":"626C6F636B7061727473","total":"111"},"height":"12345","pol_block_id":{},"pol_round":"-1","round":"23456","timestamp":"2018-02-11T07:09:22.765Z"}` + if signStr != expected { + t.Errorf("Got unexpected sign string for Proposal. Expected:\n%v\nGot:\n%v", expected, signStr) + } + if signStr != expected { t.Errorf("Got unexpected sign string for Proposal. Expected:\n%v\nGot:\n%v", expected, signStr) } diff --git a/types/protobuf.go b/types/protobuf.go index eb684ae7d..ad7362e03 100644 --- a/types/protobuf.go +++ b/types/protobuf.go @@ -6,8 +6,8 @@ import ( "reflect" "time" - abci "github.com/tendermint/abci/types" - crypto "github.com/tendermint/go-crypto" + abci "github.com/tendermint/tendermint/abci/types" + crypto "github.com/tendermint/tendermint/crypto" ) //------------------------------------------------------- @@ -58,7 +58,7 @@ func (tm2pb) Validator(val *Validator) abci.Validator { } // XXX: panics on nil or unknown pubkey type -// TODO: add cases when new pubkey types are added to go-crypto +// TODO: add cases when new pubkey types are added to crypto func (tm2pb) PubKey(pubKey crypto.PubKey) abci.PubKey { switch pk := pubKey.(type) { case crypto.PubKeyEd25519: @@ -153,7 +153,7 @@ var PB2TM = pb2tm{} type pb2tm struct{} func (pb2tm) PubKey(pubKey abci.PubKey) (crypto.PubKey, error) { - // TODO: define these in go-crypto and use them + // TODO: define these in crypto and use them sizeEd := 32 sizeSecp := 33 switch pubKey.Type { diff --git a/types/protobuf_test.go b/types/protobuf_test.go index 9f3b31873..cd986fd81 100644 --- a/types/protobuf_test.go +++ b/types/protobuf_test.go @@ -4,8 +4,8 @@ import ( "testing" "github.com/stretchr/testify/assert" - abci "github.com/tendermint/abci/types" - crypto "github.com/tendermint/go-crypto" + abci "github.com/tendermint/tendermint/abci/types" + crypto "github.com/tendermint/tendermint/crypto" ) func TestABCIPubKey(t *testing.T) { diff --git a/types/results.go b/types/results.go index 326cee48d..7f8e6093a 100644 --- a/types/results.go +++ b/types/results.go @@ -1,9 +1,9 @@ package types import ( - abci "github.com/tendermint/abci/types" - cmn "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tmlibs/merkle" + abci "github.com/tendermint/tendermint/abci/types" + "github.com/tendermint/tendermint/crypto/merkle" + cmn "github.com/tendermint/tendermint/libs/common" ) //----------------------------------------------------------------------------- @@ -51,6 +51,8 @@ func (a ABCIResults) Bytes() []byte { // Hash returns a merkle hash of all results func (a ABCIResults) Hash() []byte { + // NOTE: we copy the impl of the merkle tree for txs - + // we should be consistent and either do it for both or not. return merkle.SimpleHashFromHashers(a.toHashers()) } diff --git a/types/tx.go b/types/tx.go index e7247693a..489f0b232 100644 --- a/types/tx.go +++ b/types/tx.go @@ -5,20 +5,20 @@ import ( "errors" "fmt" - abci "github.com/tendermint/abci/types" - cmn "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tmlibs/merkle" + abci "github.com/tendermint/tendermint/abci/types" + "github.com/tendermint/tendermint/crypto/merkle" + "github.com/tendermint/tendermint/crypto/tmhash" + cmn "github.com/tendermint/tendermint/libs/common" ) // Tx is an arbitrary byte array. // NOTE: Tx has no types at this level, so when wire encoded it's just length-prefixed. -// Alternatively, it may make sense to add types here and let -// []byte be type 0x1 so we can have versioned txs if need be in the future. +// Might we want types here ? type Tx []byte -// Hash computes the RIPEMD160 hash of the wire encoded transaction. +// Hash computes the TMHASH hash of the wire encoded transaction. func (tx Tx) Hash() []byte { - return aminoHasher(tx).Hash() + return tmhash.Sum(tx) } // String returns the hex-encoded transaction as a string. @@ -32,7 +32,7 @@ type Txs []Tx // Hash returns the simple Merkle root hash of the transactions. func (txs Txs) Hash() []byte { // Recursive impl. - // Copied from tmlibs/merkle to avoid allocations + // Copied from tendermint/crypto/merkle to avoid allocations switch len(txs) { case 0: return nil diff --git a/types/tx_test.go b/types/tx_test.go index 2a93ceb31..67df5c5f3 100644 --- a/types/tx_test.go +++ b/types/tx_test.go @@ -6,8 +6,8 @@ import ( "github.com/stretchr/testify/assert" - cmn "github.com/tendermint/tmlibs/common" - ctest "github.com/tendermint/tmlibs/test" + cmn "github.com/tendermint/tendermint/libs/common" + ctest "github.com/tendermint/tendermint/libs/test" ) func makeTxs(cnt, size int) Txs { diff --git a/types/validator.go b/types/validator.go index 46dc61d07..e43acf09d 100644 --- a/types/validator.go +++ b/types/validator.go @@ -4,8 +4,8 @@ import ( "bytes" "fmt" - "github.com/tendermint/go-crypto" - cmn "github.com/tendermint/tmlibs/common" + "github.com/tendermint/tendermint/crypto" + cmn "github.com/tendermint/tendermint/libs/common" ) // Volatile state for each Validator diff --git a/types/validator_set.go b/types/validator_set.go index f2fac2929..191f8b428 100644 --- a/types/validator_set.go +++ b/types/validator_set.go @@ -7,8 +7,8 @@ import ( "sort" "strings" - cmn "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tmlibs/merkle" + "github.com/tendermint/tendermint/crypto/merkle" + cmn "github.com/tendermint/tendermint/libs/common" ) // ValidatorSet represent a set of *Validator at a given height. diff --git a/types/validator_set_test.go b/types/validator_set_test.go index 8db53a533..61f4dada9 100644 --- a/types/validator_set_test.go +++ b/types/validator_set_test.go @@ -10,8 +10,8 @@ import ( "github.com/stretchr/testify/assert" - crypto "github.com/tendermint/go-crypto" - cmn "github.com/tendermint/tmlibs/common" + crypto "github.com/tendermint/tendermint/crypto" + cmn "github.com/tendermint/tendermint/libs/common" ) func TestCopy(t *testing.T) { @@ -331,7 +331,9 @@ func TestValidatorSetVerifyCommit(t *testing.T) { Type: VoteTypePrecommit, BlockID: blockID, } - vote.Signature = privKey.Sign(vote.SignBytes(chainID)) + sig, err := privKey.Sign(vote.SignBytes(chainID)) + assert.NoError(t, err) + vote.Signature = sig commit := &Commit{ BlockID: blockID, Precommits: []*Vote{vote}, @@ -365,6 +367,6 @@ func TestValidatorSetVerifyCommit(t *testing.T) { } // test a good one - err := vset.VerifyCommit(chainID, blockID, height, commit) + err = vset.VerifyCommit(chainID, blockID, height, commit) assert.Nil(t, err) } diff --git a/types/vote.go b/types/vote.go index e4ead612a..ed4ebd73e 100644 --- a/types/vote.go +++ b/types/vote.go @@ -6,8 +6,8 @@ import ( "fmt" "time" - crypto "github.com/tendermint/go-crypto" - cmn "github.com/tendermint/tmlibs/common" + crypto "github.com/tendermint/tendermint/crypto" + cmn "github.com/tendermint/tendermint/libs/common" ) var ( diff --git a/types/vote_set.go b/types/vote_set.go index a60d95daf..c51681053 100644 --- a/types/vote_set.go +++ b/types/vote_set.go @@ -8,7 +8,7 @@ import ( "github.com/pkg/errors" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) // UNSTABLE diff --git a/types/vote_set_test.go b/types/vote_set_test.go index d424667b1..32ceb7b16 100644 --- a/types/vote_set_test.go +++ b/types/vote_set_test.go @@ -5,9 +5,9 @@ import ( "testing" "time" - crypto "github.com/tendermint/go-crypto" - cmn "github.com/tendermint/tmlibs/common" - tst "github.com/tendermint/tmlibs/test" + crypto "github.com/tendermint/tendermint/crypto" + cmn "github.com/tendermint/tendermint/libs/common" + tst "github.com/tendermint/tendermint/libs/test" ) // NOTE: privValidators are in order diff --git a/types/vote_test.go b/types/vote_test.go index 263d2c351..cbb22aaae 100644 --- a/types/vote_test.go +++ b/types/vote_test.go @@ -43,7 +43,7 @@ func TestVoteSignable(t *testing.T) { signBytes := vote.SignBytes("test_chain_id") signStr := string(signBytes) - expected := `{"@chain_id":"test_chain_id","@type":"vote","block_id":{"hash":"68617368","parts":{"hash":"70617274735F68617368","total":1000000}},"height":12345,"round":2,"timestamp":"2017-12-25T03:00:01.234Z","type":2}` + expected := `{"@chain_id":"test_chain_id","@type":"vote","block_id":{"hash":"68617368","parts":{"hash":"70617274735F68617368","total":"1000000"}},"height":"12345","round":"2","timestamp":"2017-12-25T03:00:01.234Z","type":2}` if signStr != expected { // NOTE: when this fails, you probably want to fix up consensus/replay_test too t.Errorf("Got unexpected sign string for Vote. Expected:\n%v\nGot:\n%v", expected, signStr) diff --git a/types/wire.go b/types/wire.go index bd5c4497d..6342d7eba 100644 --- a/types/wire.go +++ b/types/wire.go @@ -2,7 +2,7 @@ package types import ( "github.com/tendermint/go-amino" - "github.com/tendermint/go-crypto" + "github.com/tendermint/tendermint/crypto" ) var cdc = amino.NewCodec() diff --git a/version/version.go b/version/version.go index df553115a..9be4c9d82 100644 --- a/version/version.go +++ b/version/version.go @@ -3,14 +3,14 @@ package version // Version components const ( Maj = "0" - Min = "21" + Min = "22" Fix = "0" ) var ( // Version is the current version of Tendermint // Must be a string because scripts like dist.sh read this file. - Version = "0.21.0" + Version = "0.22.0" // GitCommit is the current HEAD set using ldflags. GitCommit string