cs: don't panic when block is not found in store (#4163)

Fixes #4069

CHANGELOG_PENDING.md

@@ -3,7 +3,7 @@
 \*\*
 
 Special thanks to external contributors on this release:
-@erikgrinaker, @PSalant726, @gchaincl
+@erikgrinaker, @PSalant726, @gchaincl, @gregzaitsev
 
 Friendly reminder, we have a [bug bounty
 program](https://hackerone.com/tendermint).
@@ -73,3 +73,4 @@ program](https://hackerone.com/tendermint).
 - [rpc][\#4141](https://github.com/tendermint/tendermint/pull/4141) JSONRPCClient: validate that Response.ID matches Request.ID
 - [rpc][\#4141](https://github.com/tendermint/tendermint/pull/4141) WSClient: check for unsolicited responses
 - [types][\4164](https://github.com/tendermint/tendermint/pull/4164) Prevent temporary power overflows on validator updates
+- [cs] \#4069 Don't panic when block meta is not found in store (@gregzaitsev)

consensus/reactor.go

@@ -501,10 +501,12 @@ OUTER_LOOP:
 			if prs.ProposalBlockParts == nil {
 				blockMeta := conR.conS.blockStore.LoadBlockMeta(prs.Height)
 				if blockMeta == nil {
-					panic(fmt.Sprintf("Failed to load block %d when blockStore is at %d",
-						prs.Height, conR.conS.blockStore.Height()))
+					heightLogger.Error("Failed to load block meta",
+						"blockstoreHeight", conR.conS.blockStore.Height())
+					time.Sleep(conR.conS.config.PeerGossipSleepDuration)
+				} else {
+					ps.InitProposalBlockParts(blockMeta.BlockID.PartsHeader)
 				}
-				ps.InitProposalBlockParts(blockMeta.BlockID.PartsHeader)
 				// continue the loop since prs is a copy and not effected by this initialization
 				continue OUTER_LOOP
 			}