state sync: reverse sync implementation (#6463)

test/e2e/networks/simple.toml

@@ -1,5 +1,4 @@
 [node.validator01]
 [node.validator02]
 [node.validator03]
-[node.validator04]
-
+[node.validator04]

test/e2e/pkg/testnet.go

@@ -49,7 +49,7 @@ const (
 	PerturbationRestart    Perturbation = "restart"
 
 	EvidenceAgeHeight int64         = 5
-	EvidenceAgeTime   time.Duration = 10 * time.Second
+	EvidenceAgeTime   time.Duration = 500 * time.Millisecond
 )
 
 // Testnet represents a single testnet.

test/e2e/runner/load.go

@@ -81,7 +81,7 @@ func loadGenerate(ctx context.Context, chTx chan<- types.Tx, multiplier int) {
 
 		select {
 		case chTx <- tx:
-			time.Sleep(time.Duration(100/multiplier) * time.Millisecond)
+			time.Sleep(time.Second / time.Duration(multiplier))
 
 		case <-ctx.Done():
 			close(chTx)

test/e2e/tests/block_test.go

@@ -32,6 +32,11 @@ func TestBlock_Header(t *testing.T) {
 			if block.Header.Height < first {
 				continue
 			}
+			// the first blocks after state sync come from the backfill process
+			// and are therefore not complete
+			if node.StateSync && block.Header.Height <= first+e2e.EvidenceAgeHeight+1 {
+				continue
+			}
 			if block.Header.Height > last {
 				break
 			}
@@ -63,10 +68,10 @@ func TestBlock_Range(t *testing.T) {
 		last := status.SyncInfo.LatestBlockHeight
 
 		switch {
+		// if the node state synced we ignore any assertions because it's hard to know how far back
+		// the node ran reverse sync for
 		case node.StateSync:
-			assert.Greater(t, first, node.Testnet.InitialHeight,
-				"state synced nodes should not contain network's initial height")
-
+			break
 		case node.RetainBlocks > 0 && int64(node.RetainBlocks) < (last-node.Testnet.InitialHeight+1):
 			// Delta handles race conditions in reading first/last heights.
 			assert.InDelta(t, node.RetainBlocks, last-first+1, 1,
@@ -78,12 +83,16 @@ func TestBlock_Range(t *testing.T) {
 		}
 
 		for h := first; h <= last; h++ {
+			if node.StateSync && h <= first+e2e.EvidenceAgeHeight+1 {
+				continue
+			}
 			resp, err := client.Block(ctx, &(h))
 			if err != nil && node.RetainBlocks > 0 && h == first {
 				// Ignore errors in first block if node is pruning blocks due to race conditions.
 				continue
 			}
 			require.NoError(t, err)
+			require.NotNil(t, resp.Block)
 			assert.Equal(t, h, resp.Block.Height)
 		}