add back persistence tests

.github/workflows/tests.yml

@@ -73,3 +73,9 @@ jobs:
         with:
           file: ./coverage.txt
         if: env.GIT_DIFF
+  
+  test_persistence:
+    runs-on: ubuntu-latest
+    steps:
+      - name: run persistence tests
+        run: sh ./test/persist/test_failure_indices.sh

test/persist/test_failure_indices.sh

@@ -0,0 +1,124 @@
+#! /bin/bash
+
+export PATH="$GOBIN:$PATH"
+export TMHOME=$HOME/.tendermint_persist
+
+rm -rf "$TMHOME"
+tendermint init
+
+# use a unix socket so we can remove it
+RPC_ADDR="$(pwd)/rpc.sock"
+
+TM_CMD="tendermint node --log_level=debug --rpc.laddr=unix://$RPC_ADDR" # &> tendermint_${name}.log"
+DUMMY_CMD="abci-cli kvstore --persist $TMHOME/kvstore" # &> kvstore_${name}.log"
+
+
+function start_procs(){
+    name=$1
+    indexToFail=$2
+    echo "Starting persistent kvstore and tendermint"
+    if [[ "$CIRCLECI" == true ]]; then
+        $DUMMY_CMD &
+    else
+        $DUMMY_CMD &> "kvstore_${name}.log" &
+    fi
+    PID_DUMMY=$!
+
+    # before starting tendermint, remove the rpc socket
+    rm -f $RPC_ADDR
+    if [[ "$indexToFail" == "" ]]; then
+        # run in background, dont fail
+		if [[ "$CIRCLECI" == true ]]; then
+			$TM_CMD &
+		else
+            $TM_CMD &> "tendermint_${name}.log" &
+		fi
+        PID_TENDERMINT=$!
+    else
+        # run in foreground, fail
+		if [[ "$CIRCLECI" == true ]]; then
+			FAIL_TEST_INDEX=$indexToFail $TM_CMD
+		else
+            FAIL_TEST_INDEX=$indexToFail $TM_CMD &> "tendermint_${name}.log"
+		fi
+        PID_TENDERMINT=$!
+    fi
+}
+
+function kill_procs(){
+    kill -9 "$PID_DUMMY" "$PID_TENDERMINT"
+    wait "$PID_DUMMY"
+    wait "$PID_TENDERMINT"
+}
+
+# wait for port to be available
+function wait_for_port() {
+    port=$1
+    # this will succeed while port is bound
+    nc -z 127.0.0.1 $port
+    ERR=$?
+    i=0
+    while [ "$ERR" == 0 ]; do
+        echo "... port $port is still bound. waiting ..."
+        sleep 1
+        nc -z 127.0.0.1 $port
+        ERR=$?
+        i=$((i + 1))
+        if [[ $i == 10 ]]; then
+            echo "Timed out waiting for port to be released"
+            exit 1
+        fi
+    done
+    echo "... port $port is free!"
+}
+
+
+failsStart=0
+fails=$(grep -r "fail.Fail" --include \*.go . | wc -l)
+failsEnd=$((fails-1))
+
+for failIndex in $(seq $failsStart $failsEnd); do
+    echo ""
+    echo "* Test FailIndex $failIndex"
+    # test failure at failIndex
+
+    bash $(dirname $0)/txs.sh "localhost:26657" &
+    start_procs 1 "$failIndex"
+
+    # tendermint should already have exited when it hits the fail index
+    # but kill -9 for good measure
+    kill_procs
+
+    start_procs 2
+
+    # wait for node to handshake and make a new block
+    # NOTE: --unix-socket is only available in curl v7.40+
+    curl -s --unix-socket "$RPC_ADDR" http://localhost/status > /dev/null
+    ERR=$?
+    i=0
+    while [ "$ERR" != 0 ]; do
+        sleep 1
+        curl -s --unix-socket "$RPC_ADDR" http://localhost/status > /dev/null
+        ERR=$?
+        i=$((i + 1))
+        if [[ $i == 20 ]]; then
+            echo "Timed out waiting for tendermint to start"
+            exit 1
+        fi
+    done
+
+    # wait for a new block
+    h1=$(curl -s --unix-socket "$RPC_ADDR" http://localhost/status | jq .result.sync_info.latest_block_height)
+    h2=$h1
+    while [ "$h2" == "$h1" ]; do
+        sleep 1
+        h2=$(curl -s --unix-socket "$RPC_ADDR" http://localhost/status | jq .result.sync_info.latest_block_height)
+    done
+
+    kill_procs
+
+    echo "* Passed Test for FailIndex $failIndex"
+    echo ""
+done
+
+echo "Passed Test: Persistence"

test/persist/test_simple.sh

@@ -0,0 +1,70 @@
+#! /bin/bash
+
+
+export TMHOME=$HOME/.tendermint_persist
+
+rm -rf $TMHOME
+tendermint init
+
+function start_procs(){
+    name=$1
+    echo "Starting persistent kvstore and tendermint"
+    abci-cli kvstore --persist $TMHOME/kvstore &> "kvstore_${name}.log" &
+    PID_DUMMY=$!
+    tendermint node &> tendermint_${name}.log &
+    PID_TENDERMINT=$!
+    sleep 5
+}
+
+function kill_procs(){
+    kill -9 $PID_DUMMY $PID_TENDERMINT
+}
+
+
+function send_txs(){
+    # send a bunch of txs over a few blocks
+    echo "Sending txs"
+    for i in `seq 1 5`; do
+        for j in `seq 1 100`; do
+            tx=`head -c 8 /dev/urandom | hexdump -ve '1/1 "%.2X"'`
+            curl -s 127.0.0.1:26657/broadcast_tx_async?tx=0x$tx &> /dev/null
+        done
+        sleep 1
+    done
+}
+
+
+start_procs 1
+send_txs
+kill_procs
+
+start_procs 2
+
+# wait for node to handshake and make a new block
+addr="localhost:26657"
+curl -s $addr/status > /dev/null
+ERR=$?
+i=0
+while [ "$ERR" != 0 ]; do
+    sleep 1
+    curl -s $addr/status > /dev/null
+    ERR=$?
+    i=$(($i + 1))
+    if [[ $i == 10 ]]; then
+        echo "Timed out waiting for tendermint to start"
+        exit 1
+    fi
+done
+
+# wait for a new block
+h1=`curl -s $addr/status | jq .result.sync_info.latest_block_height`
+h2=$h1
+while [ "$h2" == "$h1" ]; do
+    sleep 1
+    h2=`curl -s $addr/status | jq .result.sync_info.latest_block_height`
+done
+
+kill_procs
+sleep 2
+
+echo "Passed Test: Persistence"

test/persist/txs.sh

@@ -0,0 +1,23 @@
+#! /bin/bash
+set -u
+
+# wait till node is up, send txs
+ADDR=$1 #="127.0.0.1:26657"
+curl -s $ADDR/status > /dev/null
+ERR=$?
+while [ "$ERR" != 0 ]; do
+	sleep 1	
+	curl -s $ADDR/status > /dev/null
+	ERR=$?
+done
+
+# send a bunch of txs over a few blocks
+echo "Node is up, sending txs"
+for i in $(seq 1 5); do
+	for _ in $(seq 1 100); do
+		tx=$(head -c 8 /dev/urandom | hexdump -ve '1/1 "%.2X"')
+		curl -s "$ADDR/broadcast_tx_async?tx=0x$tx" &> /dev/null
+	done
+	echo "sent 100"
+	sleep 1
+done