use run-multiple

.github/workflows/e2e-regressions.yml

@@ -0,0 +1,61 @@
+name: e2e-regression
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 2 * * *'
+
+jobs:
+  e2e-nightly-regression:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '1.16'
+      - uses: actions/checkout@v2.3.4
+      - uses: technote-space/get-diff-action@v5
+        with:
+          PATTERNS: |
+            **/**.go
+            go.mod
+            go.sum
+
+      - name: Build
+        working-directory: test/e2e
+        run: make -j2 docker runner tests
+
+      - name: Run reproducer test cases
+        working-directory: test/e2e
+        run: ./run-multiple.sh networks/regression/*.toml
+
+  e2e-nightly-fail:
+    needs: e2e-nightly-test
+    if: ${{ failure() }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Notify Slack on failure
+        uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+          SLACK_CHANNEL: tendermint-internal
+          SLACK_USERNAME: Nightly E2E Tests
+          SLACK_ICON_EMOI: ':skull:'
+          SLACK_COLOR: danger
+          SLACK_MESSAGE: Nightly E2E regression tests failed on master
+          SLACK_FOOTER: ''
+
+  e2e-nightly-success: 
+    needs: e2e-nightly-test
+    if: ${{ success() }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Notify Slack on success
+        uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+          SLACK_CHANNEL: tendermint-internal
+          SLACK_USERNAME: Nightly E2E Tests
+          SLACK_ICON_EMOI: ':white_check_mark:'
+          SLACK_COLOR: good
+          SLACK_MESSAGE: Nightly E2E regression tests passed on master
+          SLACK_FOOTER: ''

.github/workflows/e2e.yml

@@ -39,4 +39,4 @@ jobs:
       - name: Emit logs on failure
         if: ${{ failure() }}
         working-directory: test/e2e
-        run: ./build/runner -f networks/ci.toml logs
+        run: ./run-multiple.sh networks/regression/*.toml

.gitignore

@@ -38,7 +38,9 @@ terraform.tfstate.backup
 terraform.tfstate.d
 test/app/grpc_client
 test/e2e/build
-test/e2e/networks/*/
+test/e2e/networks/*
+test/e2e/networks/regression/*/
+!test/e2e/networks/regression/*
 test/logs
 test/p2p/data/
 vendor

test/e2e/networks/regression/apphash-fail.toml

@@ -0,0 +1,68 @@
+# This file represents an attempt at reproducing a commonly observed failure in 
+# the generated Tendermint e2e test suite. The failure manifests as an node whose
+# AppHash as reported by /status does not agree with the AppHash as reported by the
+# block of the same height in the chain.
+
+ipv6 = true
+initial_height = 1000
+key_type = "secp256k1"
+evidence = 1
+queue_type = "priority"
+TxSize = 20480
+
+[initial_state]
+  initial01 = "a"
+  initial02 = "b"
+  initial03 = "c"
+
+[validators]
+
+[validator_update]
+  [validator_update.0]
+    validator01 = 47
+    validator02 = 93
+    validator03 = 79
+  [validator_update.1010]
+    validator04 = 63
+
+[node]
+  [node.validator01]
+    mode = "validator"
+    database = "badgerdb"
+    abci_protocol = "builtin"
+    privval_protocol = "file"
+    block_sync = "v0"
+    mempool_version = "v0"
+    persist_interval = 5
+    snapshot_interval = 3
+  [node.validator02]
+    mode = "validator"
+    database = "badgerdb"
+    abci_protocol = "builtin"
+    privval_protocol = "file"
+    block_sync = "v0"
+    mempool_version = "v1"
+    persist_interval = 5
+    snapshot_interval = 3
+  [node.validator03]
+    mode = "validator"
+    database = "badgerdb"
+    abci_protocol = "builtin"
+    privval_protocol = "file"
+    block_sync = "v0"
+    mempool_version = "v0"
+    snapshot_interval = 3
+    perturb = ["pause"]
+  [node.validator04]
+    mode = "validator"
+    database = "badgerdb"
+    abci_protocol = "builtin"
+    privval_protocol = "file"
+    start_at = 1005
+    block_sync = "v0"
+    mempool_version = "v0"
+    state_sync = "p2p"
+    persist_interval = 5
+    retain_blocks = 14
+    perturb = ["kill", "kill"]
+    use_legacy_p2p = true