diff --git a/test/jepsen/README.md b/test/jepsen/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/test/jepsen/jepsen/.gitignore b/test/jepsen/jepsen/.gitignore new file mode 100644 index 000000000..15bfa90d5 --- /dev/null +++ b/test/jepsen/jepsen/.gitignore @@ -0,0 +1,17 @@ +/target +/classes +/checkouts +pom.xml +pom.xml.asc +*.jar +*.class +/.lein-* +/.nrepl-port +.hgignore +.hg/ +.lein-repl-history +.lein-deps-sum +.lein-failures +*~ +.*.swp +/store diff --git a/test/jepsen/jepsen/CHANGELOG.md b/test/jepsen/jepsen/CHANGELOG.md new file mode 100644 index 000000000..6f29fa960 --- /dev/null +++ b/test/jepsen/jepsen/CHANGELOG.md @@ -0,0 +1,24 @@ +# Change Log +All notable changes to this project will be documented in this file. This change log follows the conventions of [keepachangelog.com](http://keepachangelog.com/). + +## [Unreleased] +### Changed +- Add a new arity to `make-widget-async` to provide a different widget shape. + +## [0.1.1] - 2017-06-05 +### Changed +- Documentation on how to make the widgets. + +### Removed +- `make-widget-sync` - we're all async, all the time. + +### Fixed +- Fixed widget maker to keep working when daylight savings switches over. + +## 0.1.0 - 2017-06-05 +### Added +- Files from the new template. +- Widget maker public API - `make-widget-sync`. + +[Unreleased]: https://github.com/your-name/jepsen.tendermint/compare/0.1.1...HEAD +[0.1.1]: https://github.com/your-name/jepsen.tendermint/compare/0.1.0...0.1.1 diff --git a/test/jepsen/jepsen/LICENSE b/test/jepsen/jepsen/LICENSE new file mode 100644 index 000000000..7689f30ef --- /dev/null +++ b/test/jepsen/jepsen/LICENSE @@ -0,0 +1,214 @@ +THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC +LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM +CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. + +1. DEFINITIONS + +"Contribution" means: + +a) in the case of the initial Contributor, the initial code and +documentation distributed under this Agreement, and + +b) in the case of each subsequent Contributor: + +i) changes to the Program, and + +ii) additions to the Program; + +where such changes and/or additions to the Program originate from and are +distributed by that particular Contributor. A Contribution 'originates' from +a Contributor if it was added to the Program by such Contributor itself or +anyone acting on such Contributor's behalf. Contributions do not include +additions to the Program which: (i) are separate modules of software +distributed in conjunction with the Program under their own license +agreement, and (ii) are not derivative works of the Program. + +"Contributor" means any person or entity that distributes the Program. + +"Licensed Patents" mean patent claims licensable by a Contributor which are +necessarily infringed by the use or sale of its Contribution alone or when +combined with the Program. + +"Program" means the Contributions distributed in accordance with this +Agreement. + +"Recipient" means anyone who receives the Program under this Agreement, +including all Contributors. + +2. GRANT OF RIGHTS + +a) Subject to the terms of this Agreement, each Contributor hereby grants +Recipient a non-exclusive, worldwide, royalty-free copyright license to +reproduce, prepare derivative works of, publicly display, publicly perform, +distribute and sublicense the Contribution of such Contributor, if any, and +such derivative works, in source code and object code form. + +b) Subject to the terms of this Agreement, each Contributor hereby grants +Recipient a non-exclusive, worldwide, royalty-free patent license under +Licensed Patents to make, use, sell, offer to sell, import and otherwise +transfer the Contribution of such Contributor, if any, in source code and +object code form. This patent license shall apply to the combination of the +Contribution and the Program if, at the time the Contribution is added by the +Contributor, such addition of the Contribution causes such combination to be +covered by the Licensed Patents. The patent license shall not apply to any +other combinations which include the Contribution. No hardware per se is +licensed hereunder. + +c) Recipient understands that although each Contributor grants the licenses +to its Contributions set forth herein, no assurances are provided by any +Contributor that the Program does not infringe the patent or other +intellectual property rights of any other entity. Each Contributor disclaims +any liability to Recipient for claims brought by any other entity based on +infringement of intellectual property rights or otherwise. As a condition to +exercising the rights and licenses granted hereunder, each Recipient hereby +assumes sole responsibility to secure any other intellectual property rights +needed, if any. For example, if a third party patent license is required to +allow Recipient to distribute the Program, it is Recipient's responsibility +to acquire that license before distributing the Program. + +d) Each Contributor represents that to its knowledge it has sufficient +copyright rights in its Contribution, if any, to grant the copyright license +set forth in this Agreement. + +3. REQUIREMENTS + +A Contributor may choose to distribute the Program in object code form under +its own license agreement, provided that: + +a) it complies with the terms and conditions of this Agreement; and + +b) its license agreement: + +i) effectively disclaims on behalf of all Contributors all warranties and +conditions, express and implied, including warranties or conditions of title +and non-infringement, and implied warranties or conditions of merchantability +and fitness for a particular purpose; + +ii) effectively excludes on behalf of all Contributors all liability for +damages, including direct, indirect, special, incidental and consequential +damages, such as lost profits; + +iii) states that any provisions which differ from this Agreement are offered +by that Contributor alone and not by any other party; and + +iv) states that source code for the Program is available from such +Contributor, and informs licensees how to obtain it in a reasonable manner on +or through a medium customarily used for software exchange. + +When the Program is made available in source code form: + +a) it must be made available under this Agreement; and + +b) a copy of this Agreement must be included with each copy of the Program. + +Contributors may not remove or alter any copyright notices contained within +the Program. + +Each Contributor must identify itself as the originator of its Contribution, +if any, in a manner that reasonably allows subsequent Recipients to identify +the originator of the Contribution. + +4. COMMERCIAL DISTRIBUTION + +Commercial distributors of software may accept certain responsibilities with +respect to end users, business partners and the like. While this license is +intended to facilitate the commercial use of the Program, the Contributor who +includes the Program in a commercial product offering should do so in a +manner which does not create potential liability for other Contributors. +Therefore, if a Contributor includes the Program in a commercial product +offering, such Contributor ("Commercial Contributor") hereby agrees to defend +and indemnify every other Contributor ("Indemnified Contributor") against any +losses, damages and costs (collectively "Losses") arising from claims, +lawsuits and other legal actions brought by a third party against the +Indemnified Contributor to the extent caused by the acts or omissions of such +Commercial Contributor in connection with its distribution of the Program in +a commercial product offering. The obligations in this section do not apply +to any claims or Losses relating to any actual or alleged intellectual +property infringement. In order to qualify, an Indemnified Contributor must: +a) promptly notify the Commercial Contributor in writing of such claim, and +b) allow the Commercial Contributor tocontrol, and cooperate with the +Commercial Contributor in, the defense and any related settlement +negotiations. The Indemnified Contributor may participate in any such claim +at its own expense. + +For example, a Contributor might include the Program in a commercial product +offering, Product X. That Contributor is then a Commercial Contributor. If +that Commercial Contributor then makes performance claims, or offers +warranties related to Product X, those performance claims and warranties are +such Commercial Contributor's responsibility alone. Under this section, the +Commercial Contributor would have to defend claims against the other +Contributors related to those performance claims and warranties, and if a +court requires any other Contributor to pay any damages as a result, the +Commercial Contributor must pay those damages. + +5. NO WARRANTY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON +AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER +EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR +CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A +PARTICULAR PURPOSE. Each Recipient is solely responsible for determining the +appropriateness of using and distributing the Program and assumes all risks +associated with its exercise of rights under this Agreement , including but +not limited to the risks and costs of program errors, compliance with +applicable laws, damage to or loss of data, programs or equipment, and +unavailability or interruption of operations. + +6. DISCLAIMER OF LIABILITY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY +CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION +LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE +EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY +OF SUCH DAMAGES. + +7. GENERAL + +If any provision of this Agreement is invalid or unenforceable under +applicable law, it shall not affect the validity or enforceability of the +remainder of the terms of this Agreement, and without further action by the +parties hereto, such provision shall be reformed to the minimum extent +necessary to make such provision valid and enforceable. + +If Recipient institutes patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Program itself +(excluding combinations of the Program with other software or hardware) +infringes such Recipient's patent(s), then such Recipient's rights granted +under Section 2(b) shall terminate as of the date such litigation is filed. + +All Recipient's rights under this Agreement shall terminate if it fails to +comply with any of the material terms or conditions of this Agreement and +does not cure such failure in a reasonable period of time after becoming +aware of such noncompliance. If all Recipient's rights under this Agreement +terminate, Recipient agrees to cease use and distribution of the Program as +soon as reasonably practicable. However, Recipient's obligations under this +Agreement and any licenses granted by Recipient relating to the Program shall +continue and survive. + +Everyone is permitted to copy and distribute copies of this Agreement, but in +order to avoid inconsistency the Agreement is copyrighted and may only be +modified in the following manner. The Agreement Steward reserves the right to +publish new versions (including revisions) of this Agreement from time to +time. No one other than the Agreement Steward has the right to modify this +Agreement. The Eclipse Foundation is the initial Agreement Steward. The +Eclipse Foundation may assign the responsibility to serve as the Agreement +Steward to a suitable separate entity. Each new version of the Agreement will +be given a distinguishing version number. The Program (including +Contributions) may always be distributed subject to the version of the +Agreement under which it was received. In addition, after a new version of +the Agreement is published, Contributor may elect to distribute the Program +(including its Contributions) under the new version. Except as expressly +stated in Sections 2(a) and 2(b) above, Recipient receives no rights or +licenses to the intellectual property of any Contributor under this +Agreement, whether expressly, by implication, estoppel or otherwise. All +rights in the Program not expressly granted under this Agreement are +reserved. + +This Agreement is governed by the laws of the State of New York and the +intellectual property laws of the United States of America. No party to this +Agreement will bring a legal action under this Agreement more than one year +after the cause of action arose. Each party waives its rights to a jury trial +in any resulting litigation. diff --git a/test/jepsen/jepsen/README.md b/test/jepsen/jepsen/README.md new file mode 100644 index 000000000..6467a2086 --- /dev/null +++ b/test/jepsen/jepsen/README.md @@ -0,0 +1,41 @@ +# jepsen.tendermint + +Jepsen tests for the Tendermint distributed consensus system. + +## Building + +- Clone this repo +- Install JDK8 or higher +- Install [Leiningen](https://leiningen.org/) +- Optional: install gnuplot +- In the tendermint repository, run `lein run test` + +To build a fat JAR file that you can run independently of leiningen, run `lein +uberjar`. + +## Usage + +`lein run serve` runs a web server for browsing test results. + +`lein run test` runs a test. Use `lein run test --help` to see options: you'll +likely want to set `--node some.hostname ...` or `--nodes-file some_file`, and +adjust `--username` as desired. + +For instance, to test a shifting cluster membership, with two clients per node, +for 3000 seconds, over a group of compare-and-set registers, you'd run + +``` +lein run test --nemesis changing-validators --concurrency 2n --time-limit 3000 --workload cas-register +``` + +Or to show that a duplicated validator with just shy of 2/3 of the vote can split the blockchain during a partition, leading to the loss of acknowledged writes: + +``` +lein run test --node n1 --node n2 --node n3 --node n4 --node n5 --node n6 --node n7 --dup-validators --super-byzantine-validators --nemesis split-dup-validators --workload set --concurrency 5n --test-count 15 --time-limit 30 +``` + +## License + +Copyright © 2017 Jepsen, LLC + +Distributed under the Apache Public License 2.0. diff --git a/test/jepsen/jepsen/doc/intro.md b/test/jepsen/jepsen/doc/intro.md new file mode 100644 index 000000000..2209978bb --- /dev/null +++ b/test/jepsen/jepsen/doc/intro.md @@ -0,0 +1,3 @@ +# Introduction to jepsen.tendermint + +TODO: write [great documentation](http://jacobian.org/writing/what-to-write/) diff --git a/test/jepsen/jepsen/project.clj b/test/jepsen/jepsen/project.clj new file mode 100644 index 000000000..a311c179f --- /dev/null +++ b/test/jepsen/jepsen/project.clj @@ -0,0 +1,24 @@ +(defproject jepsen.tendermint "0.1.0" + :description "Jepsen tests for the Tendermint Byzantine consensus system" + :url "http://github.com/jepsen-io/tendermint" + :license {:name "Apache License, version 2.0" + :url "https://www.apache.org/licenses/LICENSE-2.0"} + :dependencies [[org.clojure/clojure "1.8.0"] + [org.clojure/core.typed "0.4.0"] + [cheshire "5.7.1"] + [slingshot "0.12.2"] + [clj-http "3.6.1"] + [jepsen "0.1.6"]] + :jvm-opts ["-Xmx6g" + "-XX:+UseConcMarkSweepGC" + "-XX:+UseParNewGC" + "-XX:+CMSParallelRemarkEnabled" + "-XX:+AggressiveOpts" + "-XX:+UseFastAccessorMethods" + "-XX:MaxInlineLevel=32" + "-XX:MaxRecursiveInlineLevel=2" + "-XX:-OmitStackTraceInFastThrow" + "-server"] + :main jepsen.tendermint.cli + :injections [(require 'clojure.core.typed) + (clojure.core.typed/install)]) diff --git a/test/jepsen/jepsen/resources/config.toml b/test/jepsen/jepsen/resources/config.toml new file mode 100644 index 000000000..337a1923a --- /dev/null +++ b/test/jepsen/jepsen/resources/config.toml @@ -0,0 +1,22 @@ +# This is a TOML config file. +# For more information, see https://github.com/toml-lang/toml + +proxy_app = "tcp://127.0.0.1:26658" +moniker = "anonymous" +fast_sync = true +db_backend = "goleveldb" +# log_level = "state:info,consensus:debug,*:error" +log_level = "state:info,consensus:debug,*:error" + +[rpc] +laddr = "tcp://0.0.0.0:26657" + +[p2p] +laddr = "tcp://0.0.0.0:26656" +seeds = "" +flush_throttle_timeout = 10 + +[consensus] +timeout_commit = 10 +skip_timeout_commit = true +peer_gossip_sleep_duration = 10 diff --git a/test/jepsen/jepsen/src/jepsen/tendermint.clj b/test/jepsen/jepsen/src/jepsen/tendermint.clj new file mode 100644 index 000000000..f39281e09 --- /dev/null +++ b/test/jepsen/jepsen/src/jepsen/tendermint.clj @@ -0,0 +1,6 @@ +(ns jepsen.tendermint) + +(defn foo + "I don't do a whole lot." + [x] + (println x "Hello, World!")) diff --git a/test/jepsen/jepsen/src/jepsen/tendermint/cli.clj b/test/jepsen/jepsen/src/jepsen/tendermint/cli.clj new file mode 100644 index 000000000..b1ac349d4 --- /dev/null +++ b/test/jepsen/jepsen/src/jepsen/tendermint/cli.clj @@ -0,0 +1,28 @@ +(ns jepsen.tendermint.cli + "Command line interface to Tendermint tests." + (:require [clojure.pprint :refer [pprint]] + [clojure.tools.cli :as tc] + [jepsen [cli :as jc]] + [jepsen.tendermint [core :as core]])) + +(def opts + "Extra command line opts." + [[nil "--workload WORKLOAD" "Test workload to run; e.g. cas-register, set" + :default :cas-register + :parse-fn keyword] + [nil "--nemesis NEMESIS" "Nemesis to use; e.g. clocks" + :default :none + :parse-fn keyword] + [nil "--dup-validators" "Whether to have multiple validators share the same key."] + [nil "--super-byzantine-validators" "Should byzantine validators have just shy of 2/3 the voting weight?"] + (jc/package-opt "tendermint-url" "https://github.com/tendermint/tendermint/releases/download/v0.33.8/tendermint_v0.33.8_linux_amd64.zip") + (jc/package-opt "merkleeyes-url" "") + (jc/package-opt "abci-url" "")]) + + +(defn -main + [& args] + (jc/run! (merge (jc/serve-cmd) + (jc/single-test-cmd {:test-fn core/test + :opt-spec opts})) + args)) diff --git a/test/jepsen/jepsen/src/jepsen/tendermint/client.clj b/test/jepsen/jepsen/src/jepsen/tendermint/client.clj new file mode 100644 index 000000000..f6d202ccd --- /dev/null +++ b/test/jepsen/jepsen/src/jepsen/tendermint/client.clj @@ -0,0 +1,197 @@ +(ns jepsen.tendermint.client + "Client for merkleeyes." + (:refer-clojure :exclude [read]) + (:require [jepsen.tendermint.gowire :as w] + [clojure.data.fressian :as f] + [clj-http.client :as http] + [cheshire.core :as json] + [clojure.tools.logging :refer [info warn]] + [jepsen.util :refer [map-vals]] + [slingshot.slingshot :refer [throw+]] + [byte-streams :as bs]) + (:import (java.nio ByteBuffer) + (java.util Random) + (java.lang StringBuilder))) + +; General-purpose ABCI operations + +(defn byte-buf->hex + "Convert a byte buffer to a hex string." + [^ByteBuffer buf] + (let [sb (StringBuilder.)] + (loop [] + (if (.hasRemaining buf) + (do (.append sb (format "%02x" (bit-and (.get buf) 0xff))) + (recur)) + (str sb))))) + +(defn hex->byte-buf + "Convert a string of hex digits to a byte buffer" + [s] + (let [n (/ (count s) 2) + a (byte-array n)] + (loop [i 0] + (when (< i n) + (aset a i (unchecked-byte (Integer/parseInt + (subs s (* i 2) (+ (* i 2) 2)) 16))) + (recur (inc i)))) + (ByteBuffer/wrap a))) + +(defn encode-query-param + "Encodes a string or bytebuffer for use as a URL parameter. Converts strings + to quoted strings, e.g. \"foo\" -> \"\\\"foo\\\"\", and byte buffers to 0x... + strings, e.g. \"0xabcd\"." + [x] + (condp instance? x + String (str "\"" x "\"") + ByteBuffer (str "0x" (byte-buf->hex x)))) + +(defn validate-tx-code + "Checks a check_tx or deliver_tx structure for errors, and throws as + necessary. Returns the tx otherwise." + [tx] + (case (:code tx) + 0 tx + 4 (throw+ {:type :unauthorized :log (:log tx)}) + 111 (throw+ {:type :base-unknown-address, :log (:log tx)}) + (throw+ (assoc tx :type :unknown-tx-error)))) + +(def port "HTTP interface port" 26657) + +(def default-http-opts + "clj-http options" + {:socket-timeout 10000 + :conn-timeout 10000 + :accept :json + :as :json + :throw-entire-message? true + :retry-handler (fn [ex try-count http-context] false)}) + +(defn broadcast-tx! + "Broadcast a given transaction to the given node. tx can be a string, in + which case it is encoded as \"...\", or a ByteBuffer, in which case it is + encoded as 0x.... Throws for errors in either check_tx or deliver_tx, and if + no errors are present, returns a result map." + [node tx] + (let [tx (encode-query-param tx) + http-res (http/get (str "http://" node ":" port "/broadcast_tx_commit") + (assoc default-http-opts + :query-params {:tx tx})) + result (-> http-res :body :result)] + ; (info :result result) + (validate-tx-code (:check_tx result)) + (validate-tx-code (:deliver_tx result)) + result)) + +(defn abci-query + "Performs an ABCI query on the given node." + [node path data] + (http/get (str "http://" node ":" port "/abci_query") + (assoc default-http-opts + :query-params {:data (encode-query-param data) + :path (encode-query-param path) + :prove false}))) + +; Merkleeyes-specific paths + +(defn nonce + "A 12 byte random nonce byte buffer" + [] + (let [buf (byte-array 12)] + (.nextBytes (Random.) buf) + (w/fixed-bytes buf))) + +(def tx-types + "A map of transaction type keywords to their magic bytes." + (map-vals w/uint8 + {:set 0x01 + :remove 0x02 + :get 0x03 + :cas 0x04 + :validator-set-change 0x05 + :validator-set-read 0x06 + :validator-set-cas 0x07})) + +(defn tx-type + "Returns the byte for a transaction type keyword" + [type-kw] + (or (get tx-types type-kw) + (throw (IllegalArgumentException. (str "Unknown tx type " type-kw))))) + +(defn tx + "Construct a merkleeyes transaction byte buffer" + [type & args] + (w/write [(nonce) (tx-type type) args])) + +(defn write! + "Ask node to set k to v" + [node k v] + (broadcast-tx! node (tx :set (f/write k) (f/write v)))) + +(defn read + "Perform a transactional read" + [node k] + (-> (broadcast-tx! node (tx :get (f/write k))) + :deliver_tx + :data + hex->byte-buf + f/read)) + +(defn cas! + "Perform a compare-and-set from v to v' of k" + [node k v v'] + (broadcast-tx! node (tx :cas (f/write k) (f/write v) (f/write v')))) + +(defn validator-set + "Reads the current validator set, transactionally." + [node] + (-> (broadcast-tx! node (tx :validator-set-read)) + :deliver_tx + :data + hex->byte-buf + (bs/convert java.io.Reader) + (json/parse-stream true))) + +(defn validator-set-change! + "Change the weight of a validator, given by private key (a hex string), and a + voting power, an integer." + [node validator-key weight] + (-> (broadcast-tx! node (tx :validator-set-change + (hex->byte-buf validator-key) + (w/uint64 weight))))) + +(defn validator-set-cas! + "Change the weight of a validator, iff the current version is as given." + [node version validator-key power] + (-> (broadcast-tx! node (tx :validator-set-cas + (w/uint64 version) + (hex->byte-buf validator-key) + (w/uint64 power))))) + +(defn local-read + "Read by querying a particular node" + [node k] + (let [k (f/write k) + res (-> (abci-query node "/store" k) + :body + :result + :response + :value)] + (if (= res "") + nil + (f/read (hex->byte-buf res))))) + +(defn with-any-node + "Takes a test, a function taking a node as its first argument, and remaining + args to that function. Makes the request to various nodes until one + connects." + [test f & args] + (reduce (fn [_ node] + (try + (reduced (apply f node args)) + (catch java.net.ConnectException e + (condp re-find (.getMessage e) + #"Connection refused" nil + (throw e))))) + nil + (shuffle (:nodes test)))) diff --git a/test/jepsen/jepsen/src/jepsen/tendermint/core.clj b/test/jepsen/jepsen/src/jepsen/tendermint/core.clj new file mode 100644 index 000000000..1f472d823 --- /dev/null +++ b/test/jepsen/jepsen/src/jepsen/tendermint/core.clj @@ -0,0 +1,412 @@ +(ns jepsen.tendermint.core + (:refer-clojure :exclude [test]) + (:require [clojure.tools.logging :refer :all] + [clojure.java.io :as io] + [clojure.string :as str] + [clojure.pprint :refer [pprint]] + [knossos.model :as model] + [slingshot.slingshot :refer [try+]] + [jepsen [checker :as checker] + [cli :as cli] + [client :as client] + [control :as c] + [db :as db] + [generator :as gen] + [independent :as independent] + [nemesis :as nemesis] + [tests :as tests] + [util :as util :refer [timeout with-retry map-vals]]] + [jepsen.checker.timeline :as timeline] + [jepsen.nemesis.time :as nt] + [jepsen.control.util :as cu] + [jepsen.os.debian :as debian] + [cheshire.core :as json] + [jepsen.tendermint [client :as tc] + [db :as td] + [util :refer [base-dir]] + [validator :as tv]] + )) + +(defn r [_ _] {:type :invoke, :f :read, :value nil}) +(defn w [_ _] {:type :invoke, :f :write, :value (rand-int 10)}) +(defn cas [_ _] {:type :invoke, :f :cas, :value [(rand-int 10) (rand-int 10)]}) + +(defn cas-register-client + ([] + (cas-register-client nil)) + ([node] + (reify client/Client + (setup! [_ test node] + (cas-register-client node)) + + (invoke! [_ test op] + (let [[k v] (:value op) + crash (if (= (:f op) :read) + :fail + :info)] + (try+ + (case (:f op) + :read (assoc op + :type :ok + :value (independent/tuple k (tc/read node k))) + :write (do (tc/write! node k v) + (assoc op :type :ok)) + :cas (let [[v v'] v] + (tc/cas! node k v v') + (assoc op :type :ok))) + + (catch [:type :unauthorized] e + (assoc op :type :fail, :error :precondition-failed)) + + (catch [:type :base-unknown-address] e + (assoc op :type :fail, :error :not-found)) + + (catch org.apache.http.NoHttpResponseException e + (assoc op :type crash, :error :no-http-response)) + + (catch java.net.ConnectException e + (condp re-find (.getMessage e) + #"Connection refused" + (assoc op :type :fail, :error :connection-refused) + + (assoc op :type crash, :error [:connect-exception + (.getMessage e)]))) + + (catch java.net.SocketTimeoutException e + (assoc op :type crash, :error :timeout))))) + + (teardown! [_ test])))) + + +(defn set-client + ([] + (set-client nil)) + ([node] + (reify client/Client + (setup! [_ test node] + (set-client node)) + + (invoke! [_ test op] + (let [[k v] (:value op) + crash (if (= (:f op) :read) + :fail + :info)] + (try+ + (case (:f op) + :init (with-retry [tries 0] + (tc/write! node k []) + (assoc op :type :ok) + (catch Exception e + (if (<= 10 tries) + (throw e) + (do (info "Couldn't initialize key" k ":" (.getMessage e) "- retrying") + (Thread/sleep (* 50 (Math/pow 2 tries))) + (retry (inc tries)))))) + :add (let [s (or (vec (tc/read node k)) []) + s' (conj s v)] + (tc/cas! node k s s') + (assoc op :type :ok)) + :read (assoc op + :type :ok + :value (independent/tuple + k + (into (sorted-set) (tc/read node k))))) + + (catch [:type :unauthorized] e + (assoc op :type :fail, :error :precondition-failed)) + + (catch [:type :base-unknown-address] e + (assoc op :type :fail, :error :not-found)) + + (catch org.apache.http.NoHttpResponseException e + (assoc op :type crash, :error :no-http-response)) + + (catch java.net.ConnectException e + (condp re-find (.getMessage e) + #"Connection refused" + (assoc op :type :fail, :error :connection-refused) + + (assoc op :type crash, :error [:connect-exception + (.getMessage e)]))) + + (catch java.net.SocketTimeoutException e + (assoc op :type crash, :error :timeout))))) + + (teardown! [_ test])))) + +(defn peekaboo-dup-validators-grudge + "Takes a test. Returns a function which takes a collection of nodes from that + test, and constructs a network partition (a grudge) which isolates some dups + completely, and leaves one connected to the majority component." + [test] + (fn [nodes] + ; Pick one random node from every group of dups to participate in the + ; main component, and compute the remaining complement for each dup + ; group. + (let [{:keys [groups singles dups]} (tv/dup-groups + @(:validator-config test)) + chosen-ones (map (comp hash-set rand-nth vec) dups) + exiles (map remove chosen-ones dups)] + (nemesis/complete-grudge + (cons ; Main group + (set (concat (apply concat singles) + (apply concat chosen-ones))) + ; Exiles + exiles))))) + +(defn split-dup-validators-grudge + "Takes a test. Returns a function which takes a collection of nodes from that + test, and constructs a network partition (a grudge) which splits the network + into n disjoint components, each having a single duplicate validator and an + equal share of the remaining nodes." + [test] + (fn [nodes] + (let [{:keys [groups singles dups]} (tv/dup-groups + @(:validator-config test)) + n (reduce max (map count dups))] + (->> groups + shuffle + (map shuffle) + (apply concat) + (reduce (fn [[components i] node] + [(update components (mod i n) conj node) + (inc i)]) + [[] 0]) + first + nemesis/complete-grudge)))) + +(defn crash-truncate-nemesis + "A nemesis which kills tendermint, kills merkleeyes, truncates the merkleeyes + log, and restarts the process, on up to `fraction` of the test's nodes." + [test fraction file] + (let [faulty-nodes (take (Math/floor (* fraction (count (:nodes test)))) + (shuffle (:nodes test)))] + (reify client/Client + (setup! [this test _] this) + + (invoke! [this test op] + (info :nemesis-got op) + (case (:f op) + :stop nil + + :crash + (c/on-nodes test faulty-nodes + (fn [test node] + (td/stop-tendermint! test node) + (td/stop-merkleeyes! test node) + (c/su + (c/exec :truncate :-c :-s + (str "-" (rand-int 1048576)) + (str base-dir file))) + (td/start-merkleeyes! test node) + (td/start-tendermint! test node)))) + op) + + (teardown! [this test] + ; Ensure processes start back up by the end + (c/on-nodes test faulty-nodes td/start-merkleeyes!) + (c/on-nodes test faulty-nodes td/start-tendermint!))))) + +(defn crash-nemesis + "A nemesis which kills merkleeyes and tendermint on all nodes." + [] + (nemesis/node-start-stopper identity td/stop! td/start!)) + +(defn changing-validators-nemesis + "A nemesis which takes {:nodes [active-node-set], :transition {...}} values + and applies those transitions to the cluster." + [] + (reify client/Client + (setup! [this test _] this) + + (invoke! [this test op] + (if (= :stop (:f op)) + nil + (do (assert (= :transition (:f op))) + (let [t (:value op)] + ; Before we apply our transition, we have to move to an + ; intermediate config in case it crashes. + (swap! (:validator-config test) #(tv/pre-step % t)) + + (case (:type t) + :add + (tc/with-any-node test + tc/validator-set-cas! + (:version t) + (:data (:pub_key (:validator t))) + (:votes (:validator t))) + + :remove + (tc/with-any-node test + tc/validator-set-cas! + (:version t) + (:data (:pub_key t)) + 0) + + :alter-votes + (tc/with-any-node test + tc/validator-set-cas! + (:version t) + (:data (:pub_key t)) + (:votes t)) + + :create + (c/on-nodes test (list (:node t)) + (fn create [test node] + (td/write-validator! (:validator t)) + (td/start! test node))) + + :destroy + (c/on-nodes test (list (:node t)) + (fn destroy [test node] + (td/stop! test node) + (td/reset-node! test node)))) + + ; After we've executed an operation, we need to update our test + ; state to reflect the new state of things. + (swap! (:validator-config test) #(tv/post-step % t))))) + + (assoc op :value :done)) + + (teardown! [this test]))) + +(defn nemesis + "The generator and nemesis for each nemesis profile" + [test] + (case (:nemesis test) + :changing-validators {:nemesis (changing-validators-nemesis) + :generator (gen/stagger 10 (tv/generator))} + + :peekaboo-dup-validators {:nemesis (nemesis/partitioner + (peekaboo-dup-validators-grudge test)) + :generator (gen/start-stop 0 5)} + + :split-dup-validators {:nemesis (nemesis/partitioner + (split-dup-validators-grudge test)) + :generator (gen/once {:type :info, :f :start})} + + :half-partitions {:nemesis (nemesis/partition-random-halves) + :generator (gen/start-stop 5 30)} + + :ring-partitions {:nemesis (nemesis/partition-majorities-ring) + :generator (gen/start-stop 5 30)} + + :single-partitions {:nemesis (nemesis/partition-random-node) + :generator (gen/start-stop 5 30)} + + :clocks {:nemesis (nt/clock-nemesis) + :generator (gen/stagger 5 (nt/clock-gen))} + + :crash {:nemesis (crash-nemesis) + :generator (gen/start-stop 15 0)} + + :truncate-merkleeyes {:nemesis (crash-truncate-nemesis + test 1/3 "/jepsen/jepsen.db/000001.log") + :generator (->> {:type :info, :f :crash} + (gen/delay 10))} + + :truncate-tendermint {:nemesis (crash-truncate-nemesis + test 1/3 "/data/cs.wal/wal") + :generator (->> {:type :info, :f :crash} + (gen/delay 10))} + + :none {:nemesis nemesis/noop + :generator gen/void})) + +(defn deref-gen + "Sometimes you need to build a generator not *now*, but *later*; e.g. because + it depends on state that won't be available until the generator is actually + invoked. Wrap a derefable returning a generator in this, and it'll be deref'ed + only when asked for ops." + [dgen] + (reify gen/Generator + (op [this test process] + (gen/op @dgen test process)))) + +(defn workload + "Given a test map, computes + + {:generator a generator of client ops + :client a client to execute those ops + :model a model to validate the history + :checker a map of checker names to checkers to run}." + [test] + (let [n (count (:nodes test))] + (case (:workload test) + :cas-register {:client (cas-register-client) + :model (model/cas-register) + :generator (independent/concurrent-generator + (* 2 n) + (range) + (fn [k] + (->> (gen/mix [w cas]) + (gen/reserve n r) + (gen/stagger 1) + (gen/limit 120)))) + :final-generator nil + :checker {:linear (independent/checker + (checker/linearizable))}} + :set + (let [keys (atom [])] + {:client (set-client) + :model nil + :generator (independent/concurrent-generator + n + (range) + (fn [k] + (swap! keys conj k) + (gen/phases + (gen/once {:type :invoke, :f :init}) + (->> (range) + (map (fn [x] + {:type :invoke + :f :add + :value x})) + gen/seq + (gen/stagger 1/2))))) + :final-generator (deref-gen + (delay + (locking keys + (independent/concurrent-generator + n + @keys + (fn [k] + (gen/each (gen/once {:type :invoke + :f :read}))))))) + :checker {:set (independent/checker (checker/set))}})))) + + +(defn test + [opts] + (let [validator-config (atom nil) + test (merge + tests/noop-test + opts + {:name (str "tendermint " (name (:workload opts)) " " + (name (:nemesis opts))) + :os debian/os + :nonserializable-keys [:validator-config] + :validator-config validator-config}) + db (td/db test) + nemesis (nemesis test) + workload (workload test) + checker (checker/compose + (merge {:timeline (independent/checker (timeline/html)) + :perf (checker/perf)} + (:checker workload))) + test (merge test + {:db db + :client (:client workload) + :generator (gen/phases + (->> (:generator workload) + (gen/nemesis (:generator nemesis)) + (gen/time-limit (:time-limit opts))) + (gen/nemesis + (gen/once {:type :info, :f :stop})) + (gen/sleep 30) + (gen/clients + (:final-generator workload))) + :nemesis (:nemesis nemesis) + :model (:model workload) + :checker checker})] + test)) diff --git a/test/jepsen/jepsen/src/jepsen/tendermint/db.clj b/test/jepsen/jepsen/src/jepsen/tendermint/db.clj new file mode 100644 index 000000000..0e493e96d --- /dev/null +++ b/test/jepsen/jepsen/src/jepsen/tendermint/db.clj @@ -0,0 +1,191 @@ +(ns jepsen.tendermint.db + "Supports tendermint operations like installation, creating validators, + starting and stopping nodes, etc." + (:require [clojure.tools.logging :refer :all] + [clojure.java.io :as io] + [clojure.string :as str] + [clojure.pprint :refer [pprint]] + [slingshot.slingshot :refer [try+]] + [jepsen [core :as jepsen] + [control :as c] + [db :as db] + [util :as util :refer [timeout with-retry map-vals]]] + [jepsen.control.util :as cu] + [jepsen.os.debian :as debian] + [jepsen.nemesis.time :as nt] + [cheshire.core :as json] + [jepsen.tendermint [client :as tc] + [util :refer [base-dir]] + [validator :as tv]])) + +(defn install-component! + "Download and install a tendermint component" + [app opts] + (let [opt-name (keyword (str app "-url")) + path (get opts opt-name)] + (cu/install-archive! path (str base-dir "/" app)))) + +(defn write-validator! + "Writes out the given validator structure to priv_validator.json." + [validator] + (c/su + (c/cd base-dir + (c/exec :echo (json/generate-string validator) + :> "priv_validator_key.json") + (info "Wrote priv_validator_key.json")))) + +(defn write-genesis! + "Writes a genesis structure to a JSON file on disk." + [genesis] + (c/su + (c/cd base-dir + (c/exec :echo (json/generate-string genesis) + :> "genesis.json") + (info "Wrote genesis.json")))) + +(defn write-config! + "Writes out a config.toml file to the current node." + [] + (c/su + (c/cd base-dir + (c/exec :echo (slurp (io/resource "config.toml")) + :> "config.toml")))) + +(defn seeds + "Constructs a --seeds command line for a test, so a tendermint node knows + what other nodes to talk to." + [test node] + (->> (:nodes test) + (remove #{node}) + (map (fn [node] (str node ":26656"))) + (str/join ","))) + +(def socket-file (str base-dir "/merkleeyes.sock")) +(def socket + "The socket address we use to communicate with merkleeyes" + (str "unix://" socket-file)) + +(def merkleeyes-logfile (str base-dir "/merkleeyes.log")) +(def tendermint-logfile (str base-dir "/tendermint.log")) +(def merkleeyes-pidfile (str base-dir "/merkleeyes.pid")) +(def tendermint-pidfile (str base-dir "/tendermint.pid")) + +(defn start-tendermint! + "Starts tendermint as a daemon." + [test node] + (c/su + (c/cd base-dir + (cu/start-daemon! + {:logfile tendermint-logfile + :pidfile tendermint-pidfile + :chdir base-dir} + "./tendermint" + :--home base-dir + :node + :--proxy_app socket + :--p2p.seeds (seeds test node)))) + :started) + +(defn start-merkleeyes! + "Starts merkleeyes as a daemon." + [test node] + (c/su + (c/cd base-dir + (cu/start-daemon! + {:logfile merkleeyes-logfile + :pidfile merkleeyes-pidfile + :chdir base-dir} + "./merkleeyes" + :start + :--dbName "jepsen" + :--address socket))) + :started) + +(defn stop-tendermint! [test node] + (c/su (cu/stop-daemon! tendermint-pidfile)) + :stopped) + +(defn stop-merkleeyes! [test node] + (c/su (cu/stop-daemon! merkleeyes-pidfile) + (c/exec :rm :-rf socket-file)) + :stopped) + +(defn start! + [test node] + (start-merkleeyes! test node) + (start-tendermint! test node)) + +(defn stop! + [test node] + (stop-merkleeyes! test node) + (stop-tendermint! test node)) + +(def node-files + "Files required for a validator's state." + (map (partial str base-dir "/") + ["data" + "jepsen" + "priv_validator.json" + "priv_validator.json.bak"])) + +(defn reset-node! + "Wipe data files and identity but preserve binaries." + [test node] + (c/su (c/exec :rm :-rf node-files))) + +(defn db + "A complete Tendermint system. Options: + + :merkleeyes-url Package URLs for Tendermint components + :tendermint-url + :abci-url + + :validator-config An atom which the DB will fill in with the initial + validator config." + [opts] + (reify db/DB + (setup! [_ test node] + (c/su + (install-component! "tendermint" opts) + (install-component! "abci" opts) + (install-component! "merkleeyes" opts) + + (write-config!) + + ; OK we're ready to compute the initial validator config. + (jepsen/synchronize test) + (when (= node (jepsen/primary test)) + (let [validator-config (tv/initial-config test)] + (info :initial-config (with-out-str (pprint validator-config))) + (assert (compare-and-set! (:validator-config opts) + nil + validator-config) + "Initial validator config already established!"))) + + ; Now apply that config. + (jepsen/synchronize test) + (let [vc @(:validator-config opts)] + (write-genesis! (tv/genesis vc)) + (write-validator! (get (:validators vc) + (get-in vc [:nodes node])))) + + (start-merkleeyes! test node) + (start-tendermint! test node) + + (nt/install!) + + (Thread/sleep 1000))) + + (teardown! [_ test node] + (stop-merkleeyes! test node) + (stop-tendermint! test node) + (c/su + (c/exec :rm :-rf base-dir))) + + db/LogFiles + (log-files [_ test node] + [tendermint-logfile + merkleeyes-logfile + (str base-dir "/priv_validator.json") + (str base-dir "/genesis.json") + ]))) diff --git a/test/jepsen/jepsen/src/jepsen/tendermint/gowire.clj b/test/jepsen/jepsen/src/jepsen/tendermint/gowire.clj new file mode 100644 index 000000000..4b04a33a0 --- /dev/null +++ b/test/jepsen/jepsen/src/jepsen/tendermint/gowire.clj @@ -0,0 +1,112 @@ +(ns jepsen.tendermint.gowire + "A verrrrry minimal implementation of Tendermint's custom serialization + format: https://github.com/tendermint/go-wire" + (:import (java.nio ByteBuffer))) + +(defprotocol Writable + (byte-size [w]) + (write! [w ^ByteBuffer b])) + +; Fixed size ints +(defrecord UInt8 [^byte x] + Writable + (byte-size [_] 1) + (write! [_ b] (.put b x) b)) + +(defn uint8 + [b] + (assert (not (neg? b))) + (UInt8. (unchecked-byte b))) + +(defrecord UInt64 [^long x] + Writable + (byte-size [_] 8) + (write! [_ b] (.putLong b x) b)) + +(defn uint64 [^long l] + (assert (not (neg? l))) + (UInt64. l)) + +; Fixed-size byte buffers, which don't have a length header +(defrecord FixedBytes [^ByteBuffer x] + Writable + (byte-size [_] (.remaining x)) + (write! [_ buf] + (.put buf x) + buf)) + +(defn fixed-bytes + [x] + (if (instance? ByteBuffer x) + (FixedBytes. x) + (FixedBytes. (ByteBuffer/wrap x)))) + +; Generic types +(extend-protocol Writable + ; Nil is nothing + nil + (byte-size [n] 0) + (write! [n buf] buf) + + ; Integers work like Longs + Integer + (byte-size [n] (byte-size (long n))) + (write! [n buf] (write! (long n) buf)) + + ; Longs are varints + Long + (byte-size [n] + (cond (< n 0x0) (throw (IllegalArgumentException. + (str "Number " n " can't be negative"))) + (<= n 0x0) 1 + (<= n 0xff) 2 + (<= n 0xffff) 3 + (<= n 0xffffff) 4 + (<= n 0xffffffff) 5 + true (throw (IllegalArgumentException. + (str "Number " n " is too large"))))) + + (write! [n buf] + (let [int-size (dec (byte-size n))] + (.put buf (unchecked-byte int-size)) ; Write size byte + (condp = int-size + 0 nil + 1 (.put buf (unchecked-byte n)) + 2 (.putShort buf (unchecked-short n)) + 3 (throw (IllegalArgumentException. "Todo: bit munging")) + 4 (.putInt buf (unchecked-int n)))) + buf) + + ByteBuffer + (byte-size [b] + (let [r (.remaining b)] + (+ (byte-size r) r))) + + (write! [inbuf outbuf] + (write! (.remaining inbuf) outbuf) + (.put outbuf inbuf) + outbuf) + + ; To write sequential collections, just write each thing recursively + clojure.lang.Sequential + (byte-size [coll] + (reduce + 0 (map byte-size coll))) + + (write! [coll buf] + (doseq [x coll] + (write! x buf)) + buf)) + +(defn buffer-for + "Constructs a ByteBuffer big enough to write the given collection of objects + to. Extra is an integer number of extra bytes to allocate." + [x] + (ByteBuffer/allocate (byte-size x))) + +(defn write + "Creates a new buffer, writes the given data structure to it, and returns + that ByteBuffer, flipped." + [x] + (->> (buffer-for x) + (write! x) + .flip)) diff --git a/test/jepsen/jepsen/src/jepsen/tendermint/util.clj b/test/jepsen/jepsen/src/jepsen/tendermint/util.clj new file mode 100644 index 000000000..390a205ef --- /dev/null +++ b/test/jepsen/jepsen/src/jepsen/tendermint/util.clj @@ -0,0 +1,4 @@ +(ns jepsen.tendermint.util + "Kitchen sink.") + +(def base-dir "/opt/tendermint") diff --git a/test/jepsen/jepsen/src/jepsen/tendermint/validator.clj b/test/jepsen/jepsen/src/jepsen/tendermint/validator.clj new file mode 100644 index 000000000..2d1c959a3 --- /dev/null +++ b/test/jepsen/jepsen/src/jepsen/tendermint/validator.clj @@ -0,0 +1,946 @@ +(ns jepsen.tendermint.validator + {:lang :core.typed + :doc "Supports validator set configuration and changes."} + (:require [clojure.set :as set] + [clojure.tools.logging :refer [info warn]] + [clojure.pprint :refer [pprint]] + [clojure.core.typed :as t] + [cheshire.core :as json] + [dom-top.core :as dt] + [jepsen.tendermint [client :as tc] + [util :refer [base-dir]]] + [jepsen [util :as util :refer [map-vals]] + [control :as c] + [client :as client] + [nemesis :as nemesis] + [generator :as gen]]) + (:import (clojure.tools.logging.impl LoggerFactory Logger) + (clojure.lang Namespace + Symbol))) + +; Type support +(defmacro tk + "Typechecked keyword function. Returns the given keyword, but tells + core.typed it's a function of [m -> v]." + [kw m v] + `(t/ann-form ~kw [~m ~'-> ~v])) + +(defmacro tmfn + "Typed map fn. Core.typed doesn't know (Map a b) is also the fn [a -> (Option + b)], so we have to tell it." + [m K V] + `(t/fn [k# :- ~K] :- (t/Option ~V) + (get ~m k#))) + + +; Domain types + +(t/defalias Node + "Jepsen nodes are strings." + String) + +(t/defalias Test + "Jepsen tests have nodes and a current validator atom." + (HMap :mandatory {:nodes (t/NonEmptyVec Node) + :validator-config (t/Atom1 Config)} + :optional {:dup-validators Boolean + :max-byzantine-vote-fraction Number + :super-byzantine-validators Boolean})) + +(t/defalias Version + "Tendermint cluster version numbers." + Long) + +(t/defalias ShortKey + "In some places, Tendermint represents keys only by their raw data." + String) + +(t/defalias Key + "A key is a map with :type and :data. Tendermint uses this to represent + public and private keys in validators" + (HMap :mandatory {:type String + :data ShortKey} + :complete? true)) + +(t/defalias GenValidator + "The structure of a validator as generated by tendermint, and stored in + priv_validator.json. Does not include votes." + (HMap :mandatory {:address String + :pub_key Key + :priv_key Key})) + +(t/defalias Validator + "A Validator's complete structure, including both votes and information + necessary to construct priv_validator.json & genesis.json." + (HMap :mandatory {:address String + :pub_key Key + :priv_key Key + :votes Long})) + +(t/defalias Config + "A configuration represents a definite state of the cluster: the validators + which are a part of the cluster, what nodes are running what validators, the + version number of the config in tendermint, the nodes that are in the test, + etc. + + :prospective-validators is used to track validators we *try* to add to the + cluster, but which haven't *actually* been added yet." + (HMap :mandatory {:version Version + :node-set (t/Set Node) + :nodes (t/Map Node Key) + :validators (t/Map Key Validator) + :prospective-validators (t/Map Key Validator) + :max-byzantine-vote-fraction Number + :super-byzantine-validators Boolean})) + +(t/defalias TendermintValidator + "The cluster's representation of a validator." + (t/HMap :mandatory {:pub_key ShortKey + :power Long})) + +(t/defalias TendermintValidatorSet + "The cluster's representation of a validator set." + (t/HMap :mandatory {:validators (t/Coll TendermintValidator) + :version Version})) + +(t/defalias CreateTransition + "Create an instance of a validator on a node" + (t/HMap :mandatory {:type (t/Val :create) + :node Node + :validator Validator} + :complete? true)) + +(t/defalias DestroyTransition + "Destroy an instance of a validator" + (t/HMap :mandatory {:type (t/Val :destroy) + :node Node} + :complete? true)) + +(t/defalias AddTransition + "Add a new validator to the config" + (t/HMap :mandatory {:type (t/Val :add) + :version Version + :validator Validator} + :complete? true)) + +(t/defalias RemoveTransition + "Remove a validator from the config" + (t/HMap :mandatory {:type (t/Val :remove) + :version Version + :pub_key Key} + :complete? true)) + +(t/defalias AlterVotesTransition + "Change the votes allocated to a validator" + (t/HMap :mandatory {:type (t/Val :alter-votes) + :version Version + :pub_key Key + :votes Long} + :complete? true)) + +(t/defalias Transition (t/U CreateTransition + DestroyTransition + AddTransition + RemoveTransition + AlterVotesTransition)) + +; External types + +(t/ann jepsen.control/*dir* String) +(t/ann jepsen.tendermint.util/base-dir String) + +(t/ann ^:no-check clojure.core/update + (t/All [m k v v' arg ...] + (t/IFn + [m k [v arg ... arg -> v'] arg ... arg -> (t/Assoc m k v')]))) + +(t/ann ^:no-check clojure.tools.logging/*logger-factory* LoggerFactory) + +(t/ann ^:no-check clojure.tools.logging.impl/get-logger + [LoggerFactory (t/U clojure.lang.Symbol Namespace) + -> clojure.tools.logging.impl.Logger]) + +(t/ann ^:no-check clojure.tools.logging.impl/enabled? + [Logger t/Keyword -> Boolean]) + +(t/ann ^:no-check clojure.tools.logging/log* + [Logger t/Keyword (t/U Throwable nil) String -> nil]) + +(t/ann ^:no-check jepsen.util/map-vals + (t/All [k v1 v2] + [[v1 -> v2] (t/Map k v1) -> (t/Map k v2)])) + +(t/ann ^:no-check jepsen.control/on-nodes + (t/All [res] + (t/IFn [Test [Test Node -> res] + -> (t/Map Node res)] + [Test (t/NonEmptyColl Node) [Test Node -> res] + -> (t/I (t/Map Node res) + (t/NonEmptySeqable + (clojure.lang.AMapEntry Node res)))]))) + +(t/ann ^:no-check jepsen.control/expand-path [String -> String]) + +(t/ann ^:no-check jepsen.control/exec [t/Any * -> String]) + +(t/ann ^:no-check cheshire.core/parse-string [String true -> + (t/Map t/Keyword t/Any)]) + +(t/ann jepsen.tendermint.client/validator-set + [Node -> TendermintValidatorSet]) + +; A regression in core.typed breaks occurrence typing for locals (!?), so we +; can only convince the type system of filters using function args. +(t/ann conform-map [t/Any -> (t/Map t/Any t/Any)]) +(defn conform-map + [x] + (assert (map? x)) + x) + +(t/ann conform-string [t/Any -> String]) +(defn conform-string + [x] + (assert (string? x)) + x) + +(t/ann conform-long [t/Any -> Long]) +(defn conform-long + [x] + (assert (instance? Long x)) + x) + +(t/ann conform-key [t/Any -> Key]) +(defn conform-key + [x] + (let [m (conform-map x)] + {:type (conform-string (:type m)) + :data (conform-string (:data m))})) + +(t/ann conform-gen-validator [t/Any -> GenValidator]) +(defn conform-gen-validator + [x] + (let [m (conform-map x)] + {:address (conform-string (:address x)) + :pub_key (conform-key (:pub_key x)) + :priv_key (conform-key (:priv_key x))})) + +; OK, let's begin + +(t/ann nodes-running-validators [Config -> (t/Map Key (t/Coll Node))]) +(defn nodes-running-validators + "Takes a config, yielding a map of validator keys to groups of nodes that run + that validator." + [config] + (->> (:nodes config) + (reduce (t/fn [m :- (t/Map Key (t/Vec Node)) + [node pub-key] :- '[Node Key]] + (assoc m pub-key (conj (get m pub-key []) node))) + {}))) + +(t/ann ^:no-check byzantine-validators [Config -> (t/Coll Validator)]) +(defn byzantine-validators + "A collection of all validators in the validator set which are running on + more than one node." + [config] + (->> (nodes-running-validators config) + (filter (t/fn [[key nodes] :- '[Key (t/Coll Node)]] + (< 1 (count nodes)))) + (map key) + (keep (tmfn (:validators config) Key Validator)))) + + +(t/ann initial-validator-votes [Config -> (t/Map Key Long)]) +(defn initial-validator-votes + "Takes a config. Computes a map of validator public keys to votes. When there + are byzantine validators and the config has :super-byzantine-validators + enabled, allocates just shy of 2/3 votes to the byzantine validator. + Otherwise, allocates just shy of 1/3 votes to the byzantine validator." + [config] + (if-let [bs (seq (byzantine-validators config))] + (do (assert (= 1 (count bs)) + "Only know how to deal with 1 or 0 byzantine validators") + (let [b (:pub_key (first bs)) + n (count (:validators config))] + ; For super dup validators, we want the dup validator key to have + ; just shy of 2/3 voting power. That means the sum of the normal + ; nodes weights should be just over 1/3, so that the remaining node + ; can make up just under 2/3rds of the votes by itself. Let a normal + ; node's weight be 2. Then 2(n-1) is the combined voting power of the + ; normal bloc. We can then choose 4(n-1) - 1 as the weight for the + ; dup validator. The total votes are + ; + ; 2(n-1) + 4(n-1) - 1 + ; = 6(n-1) - 1 + ; + ; which implies a single dup node has fraction... + ; + ; (4(n-1) - 1) / (6(n-1) - 1) + ; + ; which approaches 2/3 from 0 for n = 1 -> infinity, and if a single + ; regular node is added to a duplicate node, a 2/3+ majority is + ; available for all n >= 1. + ; + ; For regular dup validators, let an individual node have weight 2. + ; The total number of individual votes is 2(n-1), which should be + ; just larger than twice the number of dup votes, e.g: + ; + ; 2(n-1) = 2d + e + ; + ; where e is some small positive integer, and d is the number of dup + ; votes. Solving for d: + ; + ; (2(n-1) - e) / 2 = d + ; n - 1 - e/2 = d ; Choose e = 2 + ; n - 2 = d + ; + ; The total number of votes is therefore: + ; + ; 2(n-1) + n - 2 + ; = 3n - 4 + ; + ; So a dup validator alone has vote fraction: + ; + ; (n - 2) / (3n - 4) + ; + ; which is always under 1/3. And with a single validator, it has vote + ; fraction: + ; + ; (n - 2) + 2 / (3n - 4) + ; = n / (3n - 4) + ; + ; which is always over 1/3. + (let [base-votes (zipmap (remove #{b} (keys (:validators config))) + (repeat 2)) + byz-votes {b (conform-long + (if (:super-byzantine-validators config) + (dec (* 4 (dec n))) + (- n 2)))}] + (t/ann-form base-votes (t/Map Key Long)) + (merge base-votes byz-votes)))) + + ; Default case: no byzantine validator, everyone has 2 votes. + (zipmap (keys (:validators config)) (repeat 2)))) + +(t/ann with-initial-validator-votes [Config -> Config]) +(defn with-initial-validator-votes + "Takes a config, computes the correct distribution of initial validator + votes, and assigns those votes to validators, returning the resulting + config." + [config] + (let [votes (initial-validator-votes config) + validators (reduce (t/fn [m :- (t/Map Key Validator) + [k votes] :- '[Key Long]] + (let [v (get m k)] + (assert v) + (assoc m k (assoc v :votes votes)))) + (:validators config) + (initial-validator-votes config))] + (assoc config :validators validators))) + +(t/ann gen-validator [-> GenValidator]) +(defn gen-validator + "Generate a new validator structure, and return the validator's data as a + map." + [] + (conform-gen-validator + (c/cd base-dir + (-> (c/exec "./tendermint" :--home base-dir :gen_validator) + (json/parse-string true))))) + +(t/ann augment-gen-validator [GenValidator -> Validator]) +(defn augment-gen-validator + "Takes a GenValidator, as generated by tendermint, and adds :votes to make it + a complete representation of a Validator." + [v] + (assoc v :votes 2)) + +(t/ann config [(HMap :optional {:version Version + :node-set (t/Set Node) + :nodes (t/Map Node Key) + :validators (t/Map Key Validator) + :super-byzantine-validators Boolean + :max-byzantine-vote-fraction Number}) + -> Config]) +(defn config + "There are two pieces of state we need to handle. The first is the validator + set, as known to the cluster, which maps public keys to maps like: + + {:address + :pub_key {:type ... + :data ...} + :priv_key {:type ... + :data ...} + :votes an-int} + + And the second is a map of nodes to the validator key they're running: + + {\"n1\" \"ABCD...\" + ...} + + Additionally, we need a bound :max-byzantine-vote-fraction on the fraction of + the vote any byzantine validator is allowed to control, a :version, denoting + the version of the validator set that the cluster knows, and a :node-set, the + set of nodes that exist." + [opts] + (merge {:validators {} + :nodes {} + :node-set #{} + :version -1 + :max-byzantine-vote-fraction 1/3 + :super-byzantine-validators false} + opts + {:prospective-validators {}})) + +(t/ann initial-config [Test -> Config]) +(defn initial-config + "Constructs an initial configuration for a test with a list of :nodes + provided." + [test] + (let [; Generate a validator for every node + validators (c/with-test-nodes test + (augment-gen-validator (gen-validator))) + ; Map of nodes to validators + nodes (map-vals (tk :pub_key Validator Key) validators) + ; Map of validator keys to validators + validators (reduce (t/fn [m :- (t/Map Key Validator) + [node v] :- '[Node Validator]] + (assoc m (:pub_key v) v)) + {} + validators) + + ; If we're working with dup validators, run the second validator on 2 + ; nodes and drop the first. + [n1 n2] (:nodes test) + validators (if (:dup-validators test) + (let [v1 (get nodes n1)] + (assert v1) + (dissoc validators v1)) + validators) + nodes (if (:dup-validators test) + (let [v2 (get validators (get nodes n2))] + (assert v2) + (assoc nodes n1 (:pub_key v2))) + nodes)] + (t/ann-form validators (t/Map Key Validator)) + (-> {:validators validators + :nodes nodes + :node-set (set (:nodes test)) + :super-byzantine-validators (:super-byzantine-validators test false) + :max-byzantine-vote-fraction (:max-byzantine-vote-fraction test 1/3)} + config + with-initial-validator-votes))) + +(t/ann genesis [Config -> t/Any]) +(defn genesis + "Computes a genesis.json structure for the given config." + [config] + {:app_hash "" + :chain_id "jepsen" + :genesis_time "0001-01-01T00:00:00.000Z" + :validators (->> (:validators config) + vals + (map (t/fn [validator :- Validator] + (let [pub-key (:pub_key validator) + name (->> (:nodes config) + (filter + (t/fn [[_ v] :- '[t/Any Key]] + (= v pub-key))) + first) + _ (assert name) + name (key name)] + {:amount (:votes validator) + :name name + :pub_key pub-key}))))}) + +(t/ann pub-key-on-node [Config Node -> (t/Option Key)]) +(defn pub-key-on-node + "What pubkey is running on a given node?" + [config node] + (-> config :nodes (get node))) + +(t/ann total-votes [Config -> Number]) +(defn total-votes + "How many votes are in the validator set total?" + [config] + (->> (:validators config) + vals + (map (tk :votes Validator Number)) + (reduce + 0))) + +(t/ann compact-key [Key -> ShortKey]) +(defn compact-key + "A compact, lossy, human-friendly representation of a validator key." + [k] + (subs (:data k) 0 5)) + +(t/ann compact-config [Config -> (HMap)]) +(defn compact-config + "Just the essentials, please. Compacts a config into a human-readable, + limited representation for debugging." + [c] + {:version (:version c) + :total-votes (total-votes c) + :prospective-validators (->> (:prospective-validators c) + (map (t/fn [[k v] :- '[Key Validator]] + (compact-key k))) + sort) + :validators (->> (:validators c) + (map (t/fn [pair :- (clojure.lang.AMapEntry Key Validator)] + (let [k (key pair) + v (val pair)] + [(compact-key k) + {:votes (:votes v)}]))) + (into (sorted-map))) + :nodes (map-vals compact-key (:nodes c)) + :max-byzantine-vote-fraction (:max-byzantine-vote-fraction c)}) + + + +(t/ann vote-fractions [Config -> (t/Map Key Number)]) +(defn vote-fractions + "A map of validator public keys to the fraction of the vote they control." + [config] + (let [total (total-votes config)] + (->> (:validators config) + (map-vals (t/fn [v :- Validator] + (/ (:votes v) total)))))) + +(t/ann running-validators [Config -> (t/Option (t/Coll Validator))]) +(defn running-validators + "A collection of validators running on at least one node." + [config] + (->> (set (vals (:nodes config))) + (keep (tmfn (:validators config) Key Validator)))) + +(t/ann ghost-validators [Config -> (t/Coll Validator)]) +(defn ghost-validators + "A collection of validators not running on any node." + [config] + (set/difference (set (vals (:validators config))) + (set (running-validators config)))) + +(t/ann byzantine-validator-keys [Config -> (t/Coll Key)]) +(defn byzantine-validator-keys + "A collection of all validator keys in the validator set which are running on + more than one node." + [config] + (map (tk :pub_key Validator Key) (byzantine-validators config))) + +(t/ann dup-groups [Config -> (HMap :mandatory {:groups (t/Coll (t/Coll Node)) + :singles (t/Coll (t/Coll Node)) + :dups (t/Coll (t/Coll Node))} + :complete? true)]) +(defn dup-groups + "Takes a config. Computes a map of: + + {:groups A collection of groups of nodes, each running the same validator + :singles Groups with only one nodes + :dups Groups with multiple nodes}" + [config] + (let [groups (-> config nodes-running-validators vals)] + {:groups groups + :singles (filter (t/fn [g :- (t/Coll t/Any)] (= 1 (count g))) groups) + :dups (filter (t/fn [g :- (t/Coll t/Any)] (< 1 (count g))) groups)})) + +(t/ann at-least-one-running-validator? [Config -> Boolean]) +(defn at-least-one-running-validator? + "Does the given config have at least one validator which is running on some + node?" + [config] + (boolean (seq (running-validators config)))) + +(t/ann omnipotent-byzantines? [Config -> Boolean]) +(defn omnipotent-byzantines? + "Does this config contain any byzantine validator which controls more than + max-byzantine-vote-fraction of the vote?" + [config] + (let [vfs (vote-fractions config) + threshold (:max-byzantine-vote-fraction config)] + (boolean (some (t/fn [k :- Key] + (let [vf (get vfs k)] + (assert vf (str "No vote fraction for " k)) + (<= threshold vf))) + (byzantine-validator-keys config))))) + +(t/ann ghost-limit Long) +(def ghost-limit + "Ghosts are souls without bodies. How many validators can exist without + actually running on any node?" + 2) + +(t/ann too-many-ghosts? [Config -> Boolean]) +(defn too-many-ghosts? + "Does this config have too many validators which aren't running on any + nodes?" + [config] + (< ghost-limit + (count + (set/difference (set (keys (:validators config))) + (set (vals (:nodes config))))))) + +(t/ann zombie-limit Long) +(def zombie-limit + "Zombies are bodies without souls. How many nodes can run a validator that's + not actually a part of the cluster?" + 2) + +(t/ann too-many-zombies? [Config -> Boolean]) +(defn too-many-zombies? + "Does this config have too many nodes which are running validators that + aren't a part of the cluster?" + [config] + (< zombie-limit + (count + (remove (set (keys (:validators config))) + (vals (:nodes config)))))) + +(t/ann quorum Number) +(def quorum + "What fraction of the configuration's voting power should be + online and non-byzantine in order to perform operations?" + 2/3) + +(t/ann quorum? [Config -> Boolean]) +(defn quorum? + "Does the given configuration provide a quorum of running votes?" + [config] + (< quorum (/ (reduce + 0 (map (tk :votes Validator Number) + (running-validators config))) + (total-votes config)))) + +(t/ann fault-limit Number) +(def fault-limit + "What fraction of votes can be either byzantine or ghosts?" + 1/3) + +(t/ann faulty? [Config -> Boolean]) +(defn faulty? + "Are too many nodes byzantine or down?" + [config] + (<= fault-limit + (/ (reduce + 0 (map (tk :votes Validator Number) + (set/union (set (byzantine-validators config)) + (set (ghost-validators config))))) + (total-votes config)))) + + +(t/ann assert-valid [Config -> Config]) +(defn assert-valid + "Ensures that the given config is valid, and returns it. Throws + AssertError if not." + [config] + (assert (at-least-one-running-validator? config)) + (assert (not (omnipotent-byzantines? config))) + (assert (not (too-many-ghosts? config))) + (assert (not (too-many-zombies? config))) + (assert (quorum? config)) + (assert (not (faulty? config))) + (assert (every? (:node-set config) (keys (:nodes config)))) + (assert (every? pos? (map (tk :votes Validator Number) + (vals (:validators config))))) + config) + +; Possible state transitions: +; - Create an instance of a validator on a node +; - Destroy a validator instance on some node + +; - Add a validator to the validator set +; - Remove a validator from the config set + +; - Adjust the weight of a validator + +(t/ann pre-step [Config Transition -> Config]) +(defn pre-step + "Where `step` defines the consequences of an atomic transition, we don't + actually get to perform all transitions atomically. In particular, when we + create or delete a validator, we *request* that the system create or delete + it, but we don't actually *know* whether it will happen until the transaction + completes. This function transitions a configuration to that in-between + state." + [config transition] + (assert-valid + (case (:type transition) + :create config + :destroy config + :add (let [v (:validator transition)] + (assert (not (get-in config [:validators (:pub_key v)]))) + (assoc config :prospective-validators + (assoc (:prospective-validators config) + (:pub_key v) v))) + :remove config + :alter-votes config))) + +(t/ann post-step [Config Transition -> Config]) +(defn post-step + "Complete a transition once we know it's been executed." + [config transition] + (assert-valid + (case (:type transition) + ; Create a new validator on a node + :create (let [n (:node transition) + v (:validator transition)] + (assert (not (get-in config [:nodes n]))) + (assoc config :nodes (assoc (:nodes config) n (:pub_key v)))) + + ; Destroy a validator on a node + :destroy (assoc config :nodes (dissoc (:nodes config) + (:node transition))) + + + ; Add a validator to the validator set + :add (let [v (:validator transition)] + (assert (not (get-in config [:validators (:pub_key v)]))) + (-> config + (assoc :prospective-validators + (dissoc (:prospective-validators config) + (:pub_key v))) + (assoc :validators + (assoc (:validators config) (:pub_key v) v)))) + + ; Remove a validator from the validator set + :remove (assoc config :validators + (dissoc (:validators config) (:pub_key transition))) + + ; Change the votes allocated to a validator + :alter-votes (let [k (:pub_key transition) + v (:votes transition) + validators (:validators config) + validator (get validators k) + _ (assert validator) + validator' (assoc validator :votes v) + validators' (assoc validators k validator')] + (assoc config :validators validators'))))) + +(t/ann step [Config Transition -> Config]) +(defn step + "Apply a low-level state transition to a config, returning a new config. + Throws if the requested transition is illegal." + [config transition] + (-> config + (pre-step transition) + (post-step transition))) + +(t/ann rand-validator [Config -> Validator]) +(defn rand-validator + "Selects a random validator from the config." + [config] + (rand-nth (vals (:validators config)))) + +(t/ann rand-free-node [Config -> (t/Option Node)]) +(defn rand-free-node + "Selects a random node which isn't running anything." + [config] + (when-let [candidates (seq (set/difference (:node-set config) + (set (keys (:nodes config)))))] + (rand-nth candidates))) + +(t/ann rand-taken-node [Config -> (t/Option Node)]) +(defn rand-taken-node + "Selects a random node that's running a validator." + [config] + (rand-nth (keys (:nodes config)))) + +(t/ann rand-transition [Test Config -> Transition]) +(defn rand-transition + "Generates a random transition on the given config." + [test config] + (or (condp <= (rand) + ; Create a new instance of a validator on a node. + 4/5 (let [v (rand-validator config) + n (rand-free-node config)] + (when (and v n) + {:type :create + :node n + :validator v})) + + ;; Nuke a node + 3/5 (when-let [node (rand-taken-node config)] + {:type :destroy + :node node}) + + ;; Create a new validator + 2/5 (let [v (-> (c/on-nodes test + [(rand-nth (:nodes test))] + (t/fn [test :- Test, node :- Node] + (gen-validator))) + first + val + (assoc :votes 2))] + {:type :add + :version (:version config) + :validator v}) + + ;; Remove a validator + 1/5 (let [v (rand-validator config)] + {:type :remove + :version (:version config) + :pub_key (:pub_key v)}) + + ; Adjust a node's weight + 0/5 (let [v (rand-validator config)] + {:type :alter-votes + :version (:version config) + :pub_key (:pub_key v) + ; Long forces core.typed to admit it's not AnyInteger; + ; strictly speaking we might go out of bounds + :votes (long (max 1 (+ (:votes v) (- (rand-int 11) 5))))})) + + ; We rolled an impossible transition; try again + (recur test config))) + +(t/ann ^:no-check rand-legal-transition [Test Config -> Transition]) +(defn rand-legal-transition + "Generates a random transition on the given config which results in a legal + state." + [test config] + (dt/with-retry [i 0] + (if (<= 100 i) + (throw (RuntimeException. (str "Unable to generate state transition from " + (pr-str config) + " in less than 100 tries; aborting." + " Last failure was:"))) + (let [t (rand-transition test config)] + (step config t) + t)) + (catch AssertionError e + (retry (inc i))))) + +(t/ann prospective-validator-by-short-key + [Config ShortKey -> (t/Option Validator)]) +(defn prospective-validator-by-short-key + "Looks up a prospective validator by key data alone; e.g. instead of by + {:type ... :data ...}." + [config pub-key-data] + (t/loop [validators :- (t/Option (t/NonEmptyASeq Validator)) + (seq (vals (:prospective-validators config)))] + (when validators + (if (= pub-key-data (:data (:pub_key (first validators)))) + (first validators) + (recur (next validators)))))) + +(t/ann validator-by-short-key [Config ShortKey -> (t/Option Validator)]) +(defn validator-by-short-key + "Looks up a validator by key data alone; e.g. instead of by {:type ... :data + ...}." + [config pub-key-data] + (t/loop [validators :- (t/Option (t/NonEmptyASeq Validator)) + (seq (vals (:validators config)))] + (when validators + (if (= pub-key-data (:data (:pub_key (first validators)))) + (first validators) + (recur (next validators)))))) + +(t/ann tendermint-validator-set->vote-map + [Config TendermintValidatorSet -> (t/Map Key Long)]) +(defn tendermint-validator-set->vote-map + "Converts a map of tendermint short keys to tendermint validators into a + map of full public keys to votes." + [config validator-set] + (->> validator-set + :validators + (map (t/fn [v :- TendermintValidator] + (let [short-key (:pub_key v) + k (or (validator-by-short-key config short-key) + (prospective-validator-by-short-key config short-key) + (throw (IllegalStateException. + (str "Don't recognize cluster validator " + (pr-str v) + "; where did it come from?"))))] + [(:pub_key k) (:power v)]))) + (into {}))) + +(t/ann clear-removed-nodes [Config (t/Map Key Long) -> Config]) +(defn clear-removed-nodes + "Takes a config and a map of validator keys to votes, and clears out config + validators which no longer exist in votes map." + [config votes] + (->> (:validators config) + (filter (t/fn [[k v] :- '[Key Validator]] + (contains? votes k))) + (into {}) + (assoc config :validators))) + +(t/ann update-known-nodes [Config (t/Map Key Long) -> Config]) +(defn update-known-nodes + "Takes a config and a map of validator keys to votes, and where a key is + present in the vote map but is not yet a validator in the config, promotes + that validator from :prospective-validators to :validators in the config. + Also updates all votes in the config." + [config votes] + (reduce (t/fn [config :- Config, [k v] :- '[Key Long]] + (let [validators (:validators config)] + (if-let [validator (get validators k)] + ; We know this validator already + (let [validator (assoc validator :votes v) + validators (assoc validators k validator)] + (assoc config :validators validators)) + + ; Promote from prospective-validators + (let [prospective (:prospective-validators config) + validator (get prospective k) + _ (assert validator + (str "Don't recognize validator " + k "; where did it come from?")) + validator (assoc validator :votes v) + validators (assoc validators k validator) + prospective (dissoc prospective k)] + (assoc config + :validators validators + :prospective-validators prospective))))) + config + votes)) + +(t/ann current-config [Test Node -> Config]) +(defn current-config + "Combines our internal test view of which nodes are running what validators + with a transactional read of the current state of validator votes, producing + a config that can be used to generate cluster transitions." + [test node] + ; TODO: improve node selection + (let [local-config @(:validator-config test) + cluster-config (tc/validator-set node) + votes (tendermint-validator-set->vote-map + local-config cluster-config)] + (-> local-config + (clear-removed-nodes votes) + (update-known-nodes votes) + (assoc :version (:version cluster-config))))) + +(t/tc-ignore + +(defn refresh-config! + "Attempts to update the test's config with new information from the cluster. + Returns our estimate of the current config. Not threadsafe." + [test] + ; TODO: make this threadsafe + (or (reduce (fn [_ node] + (try + (when-let [c (current-config test node)] + (reset! (:validator-config test) c) + (reduced c)) + (catch java.io.IOException e + ; (info e "unable to fetch current validator set config") + nil))) + nil + (shuffle (:nodes test))) + @(:validator-config test))) + +(defn generator + "A generator of legal state transitions on the current validator state." + [] + (reify gen/Generator + (op [this test process] + (try + (info "refreshing config") + (let [config (refresh-config! test)] + (info :config-refreshed) + (info (with-out-str (pprint config))) + (info (with-out-str (pprint (compact-config config)))) + {:type :info + :f :transition + :value (rand-legal-transition test config)}) + (catch Exception e + (warn e "error generating transition") + (throw e)))))) + +) diff --git a/test/jepsen/jepsen/test/jepsen/tendermint_test.clj b/test/jepsen/jepsen/test/jepsen/tendermint_test.clj new file mode 100644 index 000000000..102ab15d9 --- /dev/null +++ b/test/jepsen/jepsen/test/jepsen/tendermint_test.clj @@ -0,0 +1,7 @@ +(ns jepsen.tendermint-test + (:require [clojure.test :refer :all] + [jepsen.tendermint :refer :all])) + +(deftest a-test + (testing "FIXME, I fail." + (is (= 0 1))))