mempool: return error when mempool is full and inbound tx is rejected (#8942) (#8952)

Addresses: https://github.com/tendermint/tendermint/issues/8928

(cherry picked from commit 27c523dccb)

internal/mempool/mempool.go

@@ -540,7 +540,7 @@ func (txmp *TxMempool) initTxCallback(wtx *WrappedTx, res *abci.ResponseCheckTx,
 		)
 		if len(evictTxs) == 0 {
 			// No room for the new incoming transaction so we just remove it from
-			// the cache.
+			// the cache and return an error to the user.
 			txmp.cache.Remove(wtx.tx)
 			txmp.logger.Error(
 				"rejected incoming good transaction; mempool full",
@@ -548,7 +548,7 @@ func (txmp *TxMempool) initTxCallback(wtx *WrappedTx, res *abci.ResponseCheckTx,
 				"err", err.Error(),
 			)
 			txmp.metrics.RejectedTxs.Add(1)
-			return nil
+			return err
 		}
 
 		// evict an existing transaction(s)