diff --git a/test/e2e/tests/validator_test.go b/test/e2e/tests/validator_test.go
index 6e836ff78..8292e86ee 100644
--- a/test/e2e/tests/validator_test.go
+++ b/test/e2e/tests/validator_test.go
@@ -20,6 +20,14 @@ func TestValidator_Sets(t *testing.T) {
 		require.NoError(t, err)
 
 		first := status.SyncInfo.EarliestBlockHeight
+
+		// for nodes that have to catch up, we should only
+		// check the validator sets for nodes after this
+		// point, to avoid inconsistencies with backfill.
+		if node.StartAt > first {
+			first = node.StartAt
+		}
+
 		last := status.SyncInfo.LatestBlockHeight
 
 		// skip first block if node is pruning blocks, to avoid race conditions