add ABCI interface for state sync snapshots (#4704)

Adds the ABCI interface for [state sync](https://github.com/tendermint/tendermint/issues/828) as outlined in [ADR-053](https://github.com/tendermint/tendermint/blob/master/docs/architecture/adr-053-state-sync-prototype.md), and bumps ABCIVersion to `0.17.0`.

The interface adds a new ABCI connection which Tendermint can use to query and load snapshots from the app (for serving snapshots to other nodes), and to offer and apply snapshots to the app (for state syncing a local node from peers).

Split out from the original PR in #4645, state sync reactor will be submitted as a separate PR. The interface is implemented by the Cosmos SDK in https://github.com/cosmos/cosmos-sdk/pull/5803.

CHANGELOG_PENDING.md

@@ -14,6 +14,8 @@ Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermi
 
 - Apps
 
+  - [abci] [\#4704](https://github.com/tendermint/tendermint/pull/4704) Add ABCI methods `ListSnapshots`, `LoadSnapshotChunk`, `OfferSnapshot`, and `ApplySnapshotChunk` for state sync snapshots. `ABCIVersion` bumped to 0.17.0.
+
 - P2P Protocol
 
 - Go API

abci/client/client.go

@@ -35,6 +35,10 @@ type Client interface {
 	InitChainAsync(types.RequestInitChain) *ReqRes
 	BeginBlockAsync(types.RequestBeginBlock) *ReqRes
 	EndBlockAsync(types.RequestEndBlock) *ReqRes
+	ListSnapshotsAsync(types.RequestListSnapshots) *ReqRes
+	OfferSnapshotAsync(types.RequestOfferSnapshot) *ReqRes
+	LoadSnapshotChunkAsync(types.RequestLoadSnapshotChunk) *ReqRes
+	ApplySnapshotChunkAsync(types.RequestApplySnapshotChunk) *ReqRes
 
 	FlushSync() error
 	EchoSync(msg string) (*types.ResponseEcho, error)
@@ -47,6 +51,10 @@ type Client interface {
 	InitChainSync(types.RequestInitChain) (*types.ResponseInitChain, error)
 	BeginBlockSync(types.RequestBeginBlock) (*types.ResponseBeginBlock, error)
 	EndBlockSync(types.RequestEndBlock) (*types.ResponseEndBlock, error)
+	ListSnapshotsSync(types.RequestListSnapshots) (*types.ResponseListSnapshots, error)
+	OfferSnapshotSync(types.RequestOfferSnapshot) (*types.ResponseOfferSnapshot, error)
+	LoadSnapshotChunkSync(types.RequestLoadSnapshotChunk) (*types.ResponseLoadSnapshotChunk, error)
+	ApplySnapshotChunkSync(types.RequestApplySnapshotChunk) (*types.ResponseApplySnapshotChunk, error)
 }
 
 //----------------------------------------

abci/client/grpc_client.go

@@ -223,6 +223,42 @@ func (cli *grpcClient) EndBlockAsync(params types.RequestEndBlock) *ReqRes {
 	return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_EndBlock{EndBlock: res}})
 }
 
+func (cli *grpcClient) ListSnapshotsAsync(params types.RequestListSnapshots) *ReqRes {
+	req := types.ToRequestListSnapshots(params)
+	res, err := cli.client.ListSnapshots(context.Background(), req.GetListSnapshots(), grpc.WaitForReady(true))
+	if err != nil {
+		cli.StopForError(err)
+	}
+	return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_ListSnapshots{ListSnapshots: res}})
+}
+
+func (cli *grpcClient) OfferSnapshotAsync(params types.RequestOfferSnapshot) *ReqRes {
+	req := types.ToRequestOfferSnapshot(params)
+	res, err := cli.client.OfferSnapshot(context.Background(), req.GetOfferSnapshot(), grpc.WaitForReady(true))
+	if err != nil {
+		cli.StopForError(err)
+	}
+	return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_OfferSnapshot{OfferSnapshot: res}})
+}
+
+func (cli *grpcClient) LoadSnapshotChunkAsync(params types.RequestLoadSnapshotChunk) *ReqRes {
+	req := types.ToRequestLoadSnapshotChunk(params)
+	res, err := cli.client.LoadSnapshotChunk(context.Background(), req.GetLoadSnapshotChunk(), grpc.WaitForReady(true))
+	if err != nil {
+		cli.StopForError(err)
+	}
+	return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_LoadSnapshotChunk{LoadSnapshotChunk: res}})
+}
+
+func (cli *grpcClient) ApplySnapshotChunkAsync(params types.RequestApplySnapshotChunk) *ReqRes {
+	req := types.ToRequestApplySnapshotChunk(params)
+	res, err := cli.client.ApplySnapshotChunk(context.Background(), req.GetApplySnapshotChunk(), grpc.WaitForReady(true))
+	if err != nil {
+		cli.StopForError(err)
+	}
+	return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_ApplySnapshotChunk{ApplySnapshotChunk: res}})
+}
+
 func (cli *grpcClient) finishAsyncCall(req *types.Request, res *types.Response) *ReqRes {
 	reqres := NewReqRes(req)
 	reqres.Response = res // Set response
@@ -304,3 +340,25 @@ func (cli *grpcClient) EndBlockSync(params types.RequestEndBlock) (*types.Respon
 	reqres := cli.EndBlockAsync(params)
 	return reqres.Response.GetEndBlock(), cli.Error()
 }
+
+func (cli *grpcClient) ListSnapshotsSync(params types.RequestListSnapshots) (*types.ResponseListSnapshots, error) {
+	reqres := cli.ListSnapshotsAsync(params)
+	return reqres.Response.GetListSnapshots(), cli.Error()
+}
+
+func (cli *grpcClient) OfferSnapshotSync(params types.RequestOfferSnapshot) (*types.ResponseOfferSnapshot, error) {
+	reqres := cli.OfferSnapshotAsync(params)
+	return reqres.Response.GetOfferSnapshot(), cli.Error()
+}
+
+func (cli *grpcClient) LoadSnapshotChunkSync(
+	params types.RequestLoadSnapshotChunk) (*types.ResponseLoadSnapshotChunk, error) {
+	reqres := cli.LoadSnapshotChunkAsync(params)
+	return reqres.Response.GetLoadSnapshotChunk(), cli.Error()
+}
+
+func (cli *grpcClient) ApplySnapshotChunkSync(
+	params types.RequestApplySnapshotChunk) (*types.ResponseApplySnapshotChunk, error) {
+	reqres := cli.ApplySnapshotChunkAsync(params)
+	return reqres.Response.GetApplySnapshotChunk(), cli.Error()
+}

abci/client/local_client.go

@@ -158,6 +158,50 @@ func (app *localClient) EndBlockAsync(req types.RequestEndBlock) *ReqRes {
 	)
 }
 
+func (app *localClient) ListSnapshotsAsync(req types.RequestListSnapshots) *ReqRes {
+	app.mtx.Lock()
+	defer app.mtx.Unlock()
+
+	res := app.Application.ListSnapshots(req)
+	return app.callback(
+		types.ToRequestListSnapshots(req),
+		types.ToResponseListSnapshots(res),
+	)
+}
+
+func (app *localClient) OfferSnapshotAsync(req types.RequestOfferSnapshot) *ReqRes {
+	app.mtx.Lock()
+	defer app.mtx.Unlock()
+
+	res := app.Application.OfferSnapshot(req)
+	return app.callback(
+		types.ToRequestOfferSnapshot(req),
+		types.ToResponseOfferSnapshot(res),
+	)
+}
+
+func (app *localClient) LoadSnapshotChunkAsync(req types.RequestLoadSnapshotChunk) *ReqRes {
+	app.mtx.Lock()
+	defer app.mtx.Unlock()
+
+	res := app.Application.LoadSnapshotChunk(req)
+	return app.callback(
+		types.ToRequestLoadSnapshotChunk(req),
+		types.ToResponseLoadSnapshotChunk(res),
+	)
+}
+
+func (app *localClient) ApplySnapshotChunkAsync(req types.RequestApplySnapshotChunk) *ReqRes {
+	app.mtx.Lock()
+	defer app.mtx.Unlock()
+
+	res := app.Application.ApplySnapshotChunk(req)
+	return app.callback(
+		types.ToRequestApplySnapshotChunk(req),
+		types.ToResponseApplySnapshotChunk(res),
+	)
+}
+
 //-------------------------------------------------------
 
 func (app *localClient) FlushSync() error {
@@ -240,6 +284,40 @@ func (app *localClient) EndBlockSync(req types.RequestEndBlock) (*types.Response
 	return &res, nil
 }
 
+func (app *localClient) ListSnapshotsSync(req types.RequestListSnapshots) (*types.ResponseListSnapshots, error) {
+	app.mtx.Lock()
+	defer app.mtx.Unlock()
+
+	res := app.Application.ListSnapshots(req)
+	return &res, nil
+}
+
+func (app *localClient) OfferSnapshotSync(req types.RequestOfferSnapshot) (*types.ResponseOfferSnapshot, error) {
+	app.mtx.Lock()
+	defer app.mtx.Unlock()
+
+	res := app.Application.OfferSnapshot(req)
+	return &res, nil
+}
+
+func (app *localClient) LoadSnapshotChunkSync(
+	req types.RequestLoadSnapshotChunk) (*types.ResponseLoadSnapshotChunk, error) {
+	app.mtx.Lock()
+	defer app.mtx.Unlock()
+
+	res := app.Application.LoadSnapshotChunk(req)
+	return &res, nil
+}
+
+func (app *localClient) ApplySnapshotChunkSync(
+	req types.RequestApplySnapshotChunk) (*types.ResponseApplySnapshotChunk, error) {
+	app.mtx.Lock()
+	defer app.mtx.Unlock()
+
+	res := app.Application.ApplySnapshotChunk(req)
+	return &res, nil
+}
+
 //-------------------------------------------------------
 
 func (app *localClient) callback(req *types.Request, res *types.Response) *ReqRes {

abci/client/socket_client.go

@@ -266,6 +266,22 @@ func (cli *socketClient) EndBlockAsync(req types.RequestEndBlock) *ReqRes {
 	return cli.queueRequest(types.ToRequestEndBlock(req))
 }
 
+func (cli *socketClient) ListSnapshotsAsync(req types.RequestListSnapshots) *ReqRes {
+	return cli.queueRequest(types.ToRequestListSnapshots(req))
+}
+
+func (cli *socketClient) OfferSnapshotAsync(req types.RequestOfferSnapshot) *ReqRes {
+	return cli.queueRequest(types.ToRequestOfferSnapshot(req))
+}
+
+func (cli *socketClient) LoadSnapshotChunkAsync(req types.RequestLoadSnapshotChunk) *ReqRes {
+	return cli.queueRequest(types.ToRequestLoadSnapshotChunk(req))
+}
+
+func (cli *socketClient) ApplySnapshotChunkAsync(req types.RequestApplySnapshotChunk) *ReqRes {
+	return cli.queueRequest(types.ToRequestApplySnapshotChunk(req))
+}
+
 //----------------------------------------
 
 func (cli *socketClient) FlushSync() error {
@@ -337,6 +353,32 @@ func (cli *socketClient) EndBlockSync(req types.RequestEndBlock) (*types.Respons
 	return reqres.Response.GetEndBlock(), cli.Error()
 }
 
+func (cli *socketClient) ListSnapshotsSync(req types.RequestListSnapshots) (*types.ResponseListSnapshots, error) {
+	reqres := cli.queueRequest(types.ToRequestListSnapshots(req))
+	cli.FlushSync()
+	return reqres.Response.GetListSnapshots(), cli.Error()
+}
+
+func (cli *socketClient) OfferSnapshotSync(req types.RequestOfferSnapshot) (*types.ResponseOfferSnapshot, error) {
+	reqres := cli.queueRequest(types.ToRequestOfferSnapshot(req))
+	cli.FlushSync()
+	return reqres.Response.GetOfferSnapshot(), cli.Error()
+}
+
+func (cli *socketClient) LoadSnapshotChunkSync(
+	req types.RequestLoadSnapshotChunk) (*types.ResponseLoadSnapshotChunk, error) {
+	reqres := cli.queueRequest(types.ToRequestLoadSnapshotChunk(req))
+	cli.FlushSync()
+	return reqres.Response.GetLoadSnapshotChunk(), cli.Error()
+}
+
+func (cli *socketClient) ApplySnapshotChunkSync(
+	req types.RequestApplySnapshotChunk) (*types.ResponseApplySnapshotChunk, error) {
+	reqres := cli.queueRequest(types.ToRequestApplySnapshotChunk(req))
+	cli.FlushSync()
+	return reqres.Response.GetApplySnapshotChunk(), cli.Error()
+}
+
 //----------------------------------------
 
 func (cli *socketClient) queueRequest(req *types.Request) *ReqRes {

abci/example/kvstore/persistent_kvstore.go

@@ -144,6 +144,26 @@ func (app *PersistentKVStoreApplication) EndBlock(req types.RequestEndBlock) typ
 	return types.ResponseEndBlock{ValidatorUpdates: app.ValUpdates}
 }
 
+func (app *PersistentKVStoreApplication) ListSnapshots(
+	req types.RequestListSnapshots) types.ResponseListSnapshots {
+	return types.ResponseListSnapshots{}
+}
+
+func (app *PersistentKVStoreApplication) LoadSnapshotChunk(
+	req types.RequestLoadSnapshotChunk) types.ResponseLoadSnapshotChunk {
+	return types.ResponseLoadSnapshotChunk{}
+}
+
+func (app *PersistentKVStoreApplication) OfferSnapshot(
+	req types.RequestOfferSnapshot) types.ResponseOfferSnapshot {
+	return types.ResponseOfferSnapshot{Result: types.ResponseOfferSnapshot_abort}
+}
+
+func (app *PersistentKVStoreApplication) ApplySnapshotChunk(
+	req types.RequestApplySnapshotChunk) types.ResponseApplySnapshotChunk {
+	return types.ResponseApplySnapshotChunk{Result: types.ResponseApplySnapshotChunk_abort}
+}
+
 //---------------------------------------------
 // update validators
 

abci/server/socket_server.go

@@ -224,6 +224,18 @@ func (s *SocketServer) handleRequest(req *types.Request, responses chan<- *types
 	case *types.Request_EndBlock:
 		res := s.app.EndBlock(*r.EndBlock)
 		responses <- types.ToResponseEndBlock(res)
+	case *types.Request_ListSnapshots:
+		res := s.app.ListSnapshots(*r.ListSnapshots)
+		responses <- types.ToResponseListSnapshots(res)
+	case *types.Request_OfferSnapshot:
+		res := s.app.OfferSnapshot(*r.OfferSnapshot)
+		responses <- types.ToResponseOfferSnapshot(res)
+	case *types.Request_LoadSnapshotChunk:
+		res := s.app.LoadSnapshotChunk(*r.LoadSnapshotChunk)
+		responses <- types.ToResponseLoadSnapshotChunk(res)
+	case *types.Request_ApplySnapshotChunk:
+		res := s.app.ApplySnapshotChunk(*r.ApplySnapshotChunk)
+		responses <- types.ToResponseApplySnapshotChunk(res)
 	default:
 		responses <- types.ToResponseException("Unknown request")
 	}

abci/types/application.go

@@ -23,6 +23,12 @@ type Application interface {
 	DeliverTx(RequestDeliverTx) ResponseDeliverTx    // Deliver a tx for full processing
 	EndBlock(RequestEndBlock) ResponseEndBlock       // Signals the end of a block, returns changes to the validator set
 	Commit() ResponseCommit                          // Commit the state and return the application Merkle root hash
+
+	// State Sync Connection
+	ListSnapshots(RequestListSnapshots) ResponseListSnapshots                // List available snapshots
+	OfferSnapshot(RequestOfferSnapshot) ResponseOfferSnapshot                // Offer a snapshot to the application
+	LoadSnapshotChunk(RequestLoadSnapshotChunk) ResponseLoadSnapshotChunk    // Load a snapshot chunk
+	ApplySnapshotChunk(RequestApplySnapshotChunk) ResponseApplySnapshotChunk // Apply a shapshot chunk
 }
 
 //-------------------------------------------------------
@@ -73,6 +79,22 @@ func (BaseApplication) EndBlock(req RequestEndBlock) ResponseEndBlock {
 	return ResponseEndBlock{}
 }
 
+func (BaseApplication) ListSnapshots(req RequestListSnapshots) ResponseListSnapshots {
+	return ResponseListSnapshots{}
+}
+
+func (BaseApplication) OfferSnapshot(req RequestOfferSnapshot) ResponseOfferSnapshot {
+	return ResponseOfferSnapshot{}
+}
+
+func (BaseApplication) LoadSnapshotChunk(req RequestLoadSnapshotChunk) ResponseLoadSnapshotChunk {
+	return ResponseLoadSnapshotChunk{}
+}
+
+func (BaseApplication) ApplySnapshotChunk(req RequestApplySnapshotChunk) ResponseApplySnapshotChunk {
+	return ResponseApplySnapshotChunk{}
+}
+
 //-------------------------------------------------------
 
 // GRPCApplication is a GRPC wrapper for Application
@@ -136,3 +158,27 @@ func (app *GRPCApplication) EndBlock(ctx context.Context, req *RequestEndBlock)
 	res := app.app.EndBlock(*req)
 	return &res, nil
 }
+
+func (app *GRPCApplication) ListSnapshots(
+	ctx context.Context, req *RequestListSnapshots) (*ResponseListSnapshots, error) {
+	res := app.app.ListSnapshots(*req)
+	return &res, nil
+}
+
+func (app *GRPCApplication) OfferSnapshot(
+	ctx context.Context, req *RequestOfferSnapshot) (*ResponseOfferSnapshot, error) {
+	res := app.app.OfferSnapshot(*req)
+	return &res, nil
+}
+
+func (app *GRPCApplication) LoadSnapshotChunk(
+	ctx context.Context, req *RequestLoadSnapshotChunk) (*ResponseLoadSnapshotChunk, error) {
+	res := app.app.LoadSnapshotChunk(*req)
+	return &res, nil
+}
+
+func (app *GRPCApplication) ApplySnapshotChunk(
+	ctx context.Context, req *RequestApplySnapshotChunk) (*ResponseApplySnapshotChunk, error) {
+	res := app.app.ApplySnapshotChunk(*req)
+	return &res, nil
+}

abci/types/messages.go

@@ -135,6 +135,30 @@ func ToRequestEndBlock(req RequestEndBlock) *Request {
 	}
 }
 
+func ToRequestListSnapshots(req RequestListSnapshots) *Request {
+	return &Request{
+		Value: &Request_ListSnapshots{&req},
+	}
+}
+
+func ToRequestOfferSnapshot(req RequestOfferSnapshot) *Request {
+	return &Request{
+		Value: &Request_OfferSnapshot{&req},
+	}
+}
+
+func ToRequestLoadSnapshotChunk(req RequestLoadSnapshotChunk) *Request {
+	return &Request{
+		Value: &Request_LoadSnapshotChunk{&req},
+	}
+}
+
+func ToRequestApplySnapshotChunk(req RequestApplySnapshotChunk) *Request {
+	return &Request{
+		Value: &Request_ApplySnapshotChunk{&req},
+	}
+}
+
 //----------------------------------------
 
 func ToResponseException(errStr string) *Response {
@@ -208,3 +232,27 @@ func ToResponseEndBlock(res ResponseEndBlock) *Response {
 		Value: &Response_EndBlock{&res},
 	}
 }
+
+func ToResponseListSnapshots(res ResponseListSnapshots) *Response {
+	return &Response{
+		Value: &Response_ListSnapshots{&res},
+	}
+}
+
+func ToResponseOfferSnapshot(res ResponseOfferSnapshot) *Response {
+	return &Response{
+		Value: &Response_OfferSnapshot{&res},
+	}
+}
+
+func ToResponseLoadSnapshotChunk(res ResponseLoadSnapshotChunk) *Response {
+	return &Response{
+		Value: &Response_LoadSnapshotChunk{&res},
+	}
+}
+
+func ToResponseApplySnapshotChunk(res ResponseApplySnapshotChunk) *Response {
+	return &Response{
+		Value: &Response_ApplySnapshotChunk{&res},
+	}
+}

abci/types/types.proto

@@ -27,17 +27,21 @@ option (gogoproto.testgen_all)  = true;
 
 message Request {
   oneof value {
-    RequestEcho       echo        = 2;
-    RequestFlush      flush       = 3;
-    RequestInfo       info        = 4;
-    RequestSetOption  set_option  = 5;
-    RequestInitChain  init_chain  = 6;
-    RequestQuery      query       = 7;
-    RequestBeginBlock begin_block = 8;
-    RequestCheckTx    check_tx    = 9;
-    RequestDeliverTx  deliver_tx  = 19;
-    RequestEndBlock   end_block   = 11;
-    RequestCommit     commit      = 12;
+    RequestEcho               echo                 = 2;
+    RequestFlush              flush                = 3;
+    RequestInfo               info                 = 4;
+    RequestSetOption          set_option           = 5;
+    RequestInitChain          init_chain           = 6;
+    RequestQuery              query                = 7;
+    RequestBeginBlock         begin_block          = 8;
+    RequestCheckTx            check_tx             = 9;
+    RequestDeliverTx          deliver_tx           = 19;
+    RequestEndBlock           end_block            = 11;
+    RequestCommit             commit               = 12;
+    RequestListSnapshots      list_snapshots       = 13;
+    RequestOfferSnapshot      offer_snapshot       = 14;
+    RequestLoadSnapshotChunk  load_snapshot_chunk  = 15;
+    RequestApplySnapshotChunk apply_snapshot_chunk = 16;
   }
 }
 
@@ -101,23 +105,51 @@ message RequestEndBlock {
 
 message RequestCommit {}
 
+// lists available snapshots
+message RequestListSnapshots {
+}
+
+// offers a snapshot to the application
+message RequestOfferSnapshot {
+  Snapshot snapshot = 1;  // snapshot offered by peers
+  bytes    app_hash = 2;  // light client-verified app hash for snapshot height
+}
+
+// loads a snapshot chunk
+message RequestLoadSnapshotChunk {
+  uint64 height = 1;
+  uint32 format = 2;
+  uint32 chunk  = 3;
+}
+
+// Applies a snapshot chunk
+message RequestApplySnapshotChunk {
+  uint32 index  = 1;
+  bytes  chunk  = 2;
+  string sender = 3;
+}
+
 //----------------------------------------
 // Response types
 
 message Response {
   oneof value {
-    ResponseException  exception   = 1;
-    ResponseEcho       echo        = 2;
-    ResponseFlush      flush       = 3;
-    ResponseInfo       info        = 4;
-    ResponseSetOption  set_option  = 5;
-    ResponseInitChain  init_chain  = 6;
-    ResponseQuery      query       = 7;
-    ResponseBeginBlock begin_block = 8;
-    ResponseCheckTx    check_tx    = 9;
-    ResponseDeliverTx  deliver_tx  = 10;
-    ResponseEndBlock   end_block   = 11;
-    ResponseCommit     commit      = 12;
+    ResponseException          exception            = 1;
+    ResponseEcho               echo                 = 2;
+    ResponseFlush              flush                = 3;
+    ResponseInfo               info                 = 4;
+    ResponseSetOption          set_option           = 5;
+    ResponseInitChain          init_chain           = 6;
+    ResponseQuery              query                = 7;
+    ResponseBeginBlock         begin_block          = 8;
+    ResponseCheckTx            check_tx             = 9;
+    ResponseDeliverTx          deliver_tx           = 10;
+    ResponseEndBlock           end_block            = 11;
+    ResponseCommit             commit               = 12;
+    ResponseListSnapshots      list_snapshots       = 13;
+    ResponseOfferSnapshot      offer_snapshot       = 14;
+    ResponseLoadSnapshotChunk  load_snapshot_chunk  = 15;
+    ResponseApplySnapshotChunk apply_snapshot_chunk = 16;
   }
 }
 
@@ -210,6 +242,40 @@ message ResponseCommit {
   int64 retain_height = 3;
 }
 
+message ResponseListSnapshots {
+  repeated Snapshot snapshots = 1;
+}
+
+message ResponseOfferSnapshot {
+  Result result = 1;
+
+  enum Result {
+    accept        = 0;  // Snapshot accepted, apply chunks
+    abort         = 1;  // Abort all snapshot restoration
+    reject        = 2;  // Reject this specific snapshot, and try a different one
+    reject_format = 3;  // Reject all snapshots of this format, and try a different one
+    reject_sender = 4;  // Reject all snapshots from the sender(s), and try a different one
+  }
+}
+
+message ResponseLoadSnapshotChunk {
+  bytes chunk = 1;
+}
+
+message ResponseApplySnapshotChunk {
+  Result          result         = 1;
+  repeated uint32 refetch_chunks = 2;  // Chunks to refetch and reapply (regardless of result)
+  repeated string reject_senders = 3;  // Chunk senders to reject and ban (regardless of result)
+
+  enum Result {
+    accept          = 0;  // Chunk successfully accepted
+    abort           = 1;  // Abort all snapshot restoration
+    retry           = 2;  // Retry chunk, combine with refetch and reject as appropriate
+    retry_snapshot  = 3;  // Retry snapshot, combine with refetch and reject as appropriate
+    reject_snapshot = 4;  // Reject this snapshot, try a different one but keep sender rejections
+  }
+}
+
 //----------------------------------------
 // Misc.
 
@@ -336,6 +402,17 @@ message Evidence {
   int64                     total_voting_power = 5;
 }
 
+//----------------------------------------
+// State Sync Types
+
+message Snapshot {
+  uint64 height   = 1;  // The height at which the snapshot was taken
+  uint32 format   = 2;  // The application-specific snapshot format
+  uint32 chunks   = 3;  // Number of chunks in the snapshot
+  bytes  hash     = 4;  // Arbitrary snapshot hash - should be equal only for identical snapshots
+  bytes  metadata = 5;  // Arbitrary application metadata
+}
+
 //----------------------------------------
 // Service Definition
 
@@ -351,4 +428,8 @@ service ABCIApplication {
   rpc InitChain(RequestInitChain) returns (ResponseInitChain);
   rpc BeginBlock(RequestBeginBlock) returns (ResponseBeginBlock);
   rpc EndBlock(RequestEndBlock) returns (ResponseEndBlock);
+  rpc ListSnapshots(RequestListSnapshots) returns (ResponseListSnapshots);
+  rpc OfferSnapshot(RequestOfferSnapshot) returns (ResponseOfferSnapshot);
+  rpc LoadSnapshotChunk(RequestLoadSnapshotChunk) returns (ResponseLoadSnapshotChunk);
+  rpc ApplySnapshotChunk(RequestApplySnapshotChunk) returns (ResponseApplySnapshotChunk);
 }

proxy/app_conn.go

@@ -5,6 +5,8 @@ import (
 	"github.com/tendermint/tendermint/abci/types"
 )
 
+//go:generate mockery -case underscore -name AppConnConsensus|AppConnMempool|AppConnQuery|AppConnSnapshot
+
 //----------------------------------------------------------------------------------------
 // Enforce which abci msgs can be sent on a connection at the type level
 
@@ -40,6 +42,15 @@ type AppConnQuery interface {
 	//	SetOptionSync(key string, value string) (res types.Result)
 }
 
+type AppConnSnapshot interface {
+	Error() error
+
+	ListSnapshotsSync(types.RequestListSnapshots) (*types.ResponseListSnapshots, error)
+	OfferSnapshotSync(types.RequestOfferSnapshot) (*types.ResponseOfferSnapshot, error)
+	LoadSnapshotChunkSync(types.RequestLoadSnapshotChunk) (*types.ResponseLoadSnapshotChunk, error)
+	ApplySnapshotChunkSync(types.RequestApplySnapshotChunk) (*types.ResponseApplySnapshotChunk, error)
+}
+
 //-----------------------------------------------------------------------------------------
 // Implements AppConnConsensus (subset of abcicli.Client)
 
@@ -142,3 +153,38 @@ func (app *appConnQuery) InfoSync(req types.RequestInfo) (*types.ResponseInfo, e
 func (app *appConnQuery) QuerySync(reqQuery types.RequestQuery) (*types.ResponseQuery, error) {
 	return app.appConn.QuerySync(reqQuery)
 }
+
+//------------------------------------------------
+// Implements AppConnSnapshot (subset of abcicli.Client)
+
+type appConnSnapshot struct {
+	appConn abcicli.Client
+}
+
+func NewAppConnSnapshot(appConn abcicli.Client) AppConnSnapshot {
+	return &appConnSnapshot{
+		appConn: appConn,
+	}
+}
+
+func (app *appConnSnapshot) Error() error {
+	return app.appConn.Error()
+}
+
+func (app *appConnSnapshot) ListSnapshotsSync(req types.RequestListSnapshots) (*types.ResponseListSnapshots, error) {
+	return app.appConn.ListSnapshotsSync(req)
+}
+
+func (app *appConnSnapshot) OfferSnapshotSync(req types.RequestOfferSnapshot) (*types.ResponseOfferSnapshot, error) {
+	return app.appConn.OfferSnapshotSync(req)
+}
+
+func (app *appConnSnapshot) LoadSnapshotChunkSync(
+	req types.RequestLoadSnapshotChunk) (*types.ResponseLoadSnapshotChunk, error) {
+	return app.appConn.LoadSnapshotChunkSync(req)
+}
+
+func (app *appConnSnapshot) ApplySnapshotChunkSync(
+	req types.RequestApplySnapshotChunk) (*types.ResponseApplySnapshotChunk, error) {
+	return app.appConn.ApplySnapshotChunkSync(req)
+}

proxy/mocks/app_conn_consensus.go

@@ -0,0 +1,142 @@
+// Code generated by mockery v1.0.0. DO NOT EDIT.
+
+package mocks
+
+import (
+	mock "github.com/stretchr/testify/mock"
+	abcicli "github.com/tendermint/tendermint/abci/client"
+
+	types "github.com/tendermint/tendermint/abci/types"
+)
+
+// AppConnConsensus is an autogenerated mock type for the AppConnConsensus type
+type AppConnConsensus struct {
+	mock.Mock
+}
+
+// BeginBlockSync provides a mock function with given fields: _a0
+func (_m *AppConnConsensus) BeginBlockSync(_a0 types.RequestBeginBlock) (*types.ResponseBeginBlock, error) {
+	ret := _m.Called(_a0)
+
+	var r0 *types.ResponseBeginBlock
+	if rf, ok := ret.Get(0).(func(types.RequestBeginBlock) *types.ResponseBeginBlock); ok {
+		r0 = rf(_a0)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*types.ResponseBeginBlock)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(types.RequestBeginBlock) error); ok {
+		r1 = rf(_a0)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// CommitSync provides a mock function with given fields:
+func (_m *AppConnConsensus) CommitSync() (*types.ResponseCommit, error) {
+	ret := _m.Called()
+
+	var r0 *types.ResponseCommit
+	if rf, ok := ret.Get(0).(func() *types.ResponseCommit); ok {
+		r0 = rf()
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*types.ResponseCommit)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func() error); ok {
+		r1 = rf()
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// DeliverTxAsync provides a mock function with given fields: _a0
+func (_m *AppConnConsensus) DeliverTxAsync(_a0 types.RequestDeliverTx) *abcicli.ReqRes {
+	ret := _m.Called(_a0)
+
+	var r0 *abcicli.ReqRes
+	if rf, ok := ret.Get(0).(func(types.RequestDeliverTx) *abcicli.ReqRes); ok {
+		r0 = rf(_a0)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*abcicli.ReqRes)
+		}
+	}
+
+	return r0
+}
+
+// EndBlockSync provides a mock function with given fields: _a0
+func (_m *AppConnConsensus) EndBlockSync(_a0 types.RequestEndBlock) (*types.ResponseEndBlock, error) {
+	ret := _m.Called(_a0)
+
+	var r0 *types.ResponseEndBlock
+	if rf, ok := ret.Get(0).(func(types.RequestEndBlock) *types.ResponseEndBlock); ok {
+		r0 = rf(_a0)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*types.ResponseEndBlock)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(types.RequestEndBlock) error); ok {
+		r1 = rf(_a0)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Error provides a mock function with given fields:
+func (_m *AppConnConsensus) Error() error {
+	ret := _m.Called()
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func() error); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// InitChainSync provides a mock function with given fields: _a0
+func (_m *AppConnConsensus) InitChainSync(_a0 types.RequestInitChain) (*types.ResponseInitChain, error) {
+	ret := _m.Called(_a0)
+
+	var r0 *types.ResponseInitChain
+	if rf, ok := ret.Get(0).(func(types.RequestInitChain) *types.ResponseInitChain); ok {
+		r0 = rf(_a0)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*types.ResponseInitChain)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(types.RequestInitChain) error); ok {
+		r1 = rf(_a0)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// SetResponseCallback provides a mock function with given fields: _a0
+func (_m *AppConnConsensus) SetResponseCallback(_a0 abcicli.Callback) {
+	_m.Called(_a0)
+}

proxy/mocks/app_conn_mempool.go

@@ -0,0 +1,80 @@
+// Code generated by mockery v1.0.0. DO NOT EDIT.
+
+package mocks
+
+import (
+	mock "github.com/stretchr/testify/mock"
+	abcicli "github.com/tendermint/tendermint/abci/client"
+
+	types "github.com/tendermint/tendermint/abci/types"
+)
+
+// AppConnMempool is an autogenerated mock type for the AppConnMempool type
+type AppConnMempool struct {
+	mock.Mock
+}
+
+// CheckTxAsync provides a mock function with given fields: _a0
+func (_m *AppConnMempool) CheckTxAsync(_a0 types.RequestCheckTx) *abcicli.ReqRes {
+	ret := _m.Called(_a0)
+
+	var r0 *abcicli.ReqRes
+	if rf, ok := ret.Get(0).(func(types.RequestCheckTx) *abcicli.ReqRes); ok {
+		r0 = rf(_a0)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*abcicli.ReqRes)
+		}
+	}
+
+	return r0
+}
+
+// Error provides a mock function with given fields:
+func (_m *AppConnMempool) Error() error {
+	ret := _m.Called()
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func() error); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// FlushAsync provides a mock function with given fields:
+func (_m *AppConnMempool) FlushAsync() *abcicli.ReqRes {
+	ret := _m.Called()
+
+	var r0 *abcicli.ReqRes
+	if rf, ok := ret.Get(0).(func() *abcicli.ReqRes); ok {
+		r0 = rf()
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*abcicli.ReqRes)
+		}
+	}
+
+	return r0
+}
+
+// FlushSync provides a mock function with given fields:
+func (_m *AppConnMempool) FlushSync() error {
+	ret := _m.Called()
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func() error); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// SetResponseCallback provides a mock function with given fields: _a0
+func (_m *AppConnMempool) SetResponseCallback(_a0 abcicli.Callback) {
+	_m.Called(_a0)
+}

proxy/mocks/app_conn_query.go

@@ -0,0 +1,97 @@
+// Code generated by mockery v1.0.0. DO NOT EDIT.
+
+package mocks
+
+import (
+	mock "github.com/stretchr/testify/mock"
+
+	types "github.com/tendermint/tendermint/abci/types"
+)
+
+// AppConnQuery is an autogenerated mock type for the AppConnQuery type
+type AppConnQuery struct {
+	mock.Mock
+}
+
+// EchoSync provides a mock function with given fields: _a0
+func (_m *AppConnQuery) EchoSync(_a0 string) (*types.ResponseEcho, error) {
+	ret := _m.Called(_a0)
+
+	var r0 *types.ResponseEcho
+	if rf, ok := ret.Get(0).(func(string) *types.ResponseEcho); ok {
+		r0 = rf(_a0)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*types.ResponseEcho)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(string) error); ok {
+		r1 = rf(_a0)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Error provides a mock function with given fields:
+func (_m *AppConnQuery) Error() error {
+	ret := _m.Called()
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func() error); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// InfoSync provides a mock function with given fields: _a0
+func (_m *AppConnQuery) InfoSync(_a0 types.RequestInfo) (*types.ResponseInfo, error) {
+	ret := _m.Called(_a0)
+
+	var r0 *types.ResponseInfo
+	if rf, ok := ret.Get(0).(func(types.RequestInfo) *types.ResponseInfo); ok {
+		r0 = rf(_a0)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*types.ResponseInfo)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(types.RequestInfo) error); ok {
+		r1 = rf(_a0)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// QuerySync provides a mock function with given fields: _a0
+func (_m *AppConnQuery) QuerySync(_a0 types.RequestQuery) (*types.ResponseQuery, error) {
+	ret := _m.Called(_a0)
+
+	var r0 *types.ResponseQuery
+	if rf, ok := ret.Get(0).(func(types.RequestQuery) *types.ResponseQuery); ok {
+		r0 = rf(_a0)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*types.ResponseQuery)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(types.RequestQuery) error); ok {
+		r1 = rf(_a0)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}

proxy/mocks/app_conn_snapshot.go

@@ -0,0 +1,120 @@
+// Code generated by mockery v1.0.0. DO NOT EDIT.
+
+package mocks
+
+import (
+	mock "github.com/stretchr/testify/mock"
+
+	types "github.com/tendermint/tendermint/abci/types"
+)
+
+// AppConnSnapshot is an autogenerated mock type for the AppConnSnapshot type
+type AppConnSnapshot struct {
+	mock.Mock
+}
+
+// ApplySnapshotChunkSync provides a mock function with given fields: _a0
+func (_m *AppConnSnapshot) ApplySnapshotChunkSync(_a0 types.RequestApplySnapshotChunk) (*types.ResponseApplySnapshotChunk, error) {
+	ret := _m.Called(_a0)
+
+	var r0 *types.ResponseApplySnapshotChunk
+	if rf, ok := ret.Get(0).(func(types.RequestApplySnapshotChunk) *types.ResponseApplySnapshotChunk); ok {
+		r0 = rf(_a0)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*types.ResponseApplySnapshotChunk)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(types.RequestApplySnapshotChunk) error); ok {
+		r1 = rf(_a0)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Error provides a mock function with given fields:
+func (_m *AppConnSnapshot) Error() error {
+	ret := _m.Called()
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func() error); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// ListSnapshotsSync provides a mock function with given fields: _a0
+func (_m *AppConnSnapshot) ListSnapshotsSync(_a0 types.RequestListSnapshots) (*types.ResponseListSnapshots, error) {
+	ret := _m.Called(_a0)
+
+	var r0 *types.ResponseListSnapshots
+	if rf, ok := ret.Get(0).(func(types.RequestListSnapshots) *types.ResponseListSnapshots); ok {
+		r0 = rf(_a0)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*types.ResponseListSnapshots)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(types.RequestListSnapshots) error); ok {
+		r1 = rf(_a0)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// LoadSnapshotChunkSync provides a mock function with given fields: _a0
+func (_m *AppConnSnapshot) LoadSnapshotChunkSync(_a0 types.RequestLoadSnapshotChunk) (*types.ResponseLoadSnapshotChunk, error) {
+	ret := _m.Called(_a0)
+
+	var r0 *types.ResponseLoadSnapshotChunk
+	if rf, ok := ret.Get(0).(func(types.RequestLoadSnapshotChunk) *types.ResponseLoadSnapshotChunk); ok {
+		r0 = rf(_a0)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*types.ResponseLoadSnapshotChunk)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(types.RequestLoadSnapshotChunk) error); ok {
+		r1 = rf(_a0)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// OfferSnapshotSync provides a mock function with given fields: _a0
+func (_m *AppConnSnapshot) OfferSnapshotSync(_a0 types.RequestOfferSnapshot) (*types.ResponseOfferSnapshot, error) {
+	ret := _m.Called(_a0)
+
+	var r0 *types.ResponseOfferSnapshot
+	if rf, ok := ret.Get(0).(func(types.RequestOfferSnapshot) *types.ResponseOfferSnapshot); ok {
+		r0 = rf(_a0)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*types.ResponseOfferSnapshot)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(types.RequestOfferSnapshot) error); ok {
+		r1 = rf(_a0)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}

proxy/multi_app_conn.go

@@ -1,7 +1,7 @@
 package proxy
 
 import (
-	"github.com/pkg/errors"
+	"fmt"
 
 	"github.com/tendermint/tendermint/libs/service"
 )
@@ -15,6 +15,7 @@ type AppConns interface {
 	Mempool() AppConnMempool
 	Consensus() AppConnConsensus
 	Query() AppConnQuery
+	Snapshot() AppConnSnapshot
 }
 
 func NewAppConns(clientCreator ClientCreator) AppConns {
@@ -33,6 +34,7 @@ type multiAppConn struct {
 	mempoolConn   AppConnMempool
 	consensusConn AppConnConsensus
 	queryConn     AppConnQuery
+	snapshotConn  AppConnSnapshot
 
 	clientCreator ClientCreator
 }
@@ -61,37 +63,53 @@ func (app *multiAppConn) Query() AppConnQuery {
 	return app.queryConn
 }
 
+// Returns the snapshot Connection
+func (app *multiAppConn) Snapshot() AppConnSnapshot {
+	return app.snapshotConn
+}
+
 func (app *multiAppConn) OnStart() error {
 	// query connection
 	querycli, err := app.clientCreator.NewABCIClient()
 	if err != nil {
-		return errors.Wrap(err, "Error creating ABCI client (query connection)")
+		return fmt.Errorf("error creating ABCI client (query connection): %w", err)
 	}
 	querycli.SetLogger(app.Logger.With("module", "abci-client", "connection", "query"))
 	if err := querycli.Start(); err != nil {
-		return errors.Wrap(err, "Error starting ABCI client (query connection)")
+		return fmt.Errorf("error starting ABCI client (query connection): %w", err)
 	}
 	app.queryConn = NewAppConnQuery(querycli)
 
+	// snapshot connection
+	snapshotcli, err := app.clientCreator.NewABCIClient()
+	if err != nil {
+		return fmt.Errorf("error creating ABCI client (snapshot connection): %w", err)
+	}
+	snapshotcli.SetLogger(app.Logger.With("module", "abci-client", "connection", "snapshot"))
+	if err := snapshotcli.Start(); err != nil {
+		return fmt.Errorf("error starting ABCI client (snapshot connection): %w", err)
+	}
+	app.snapshotConn = NewAppConnSnapshot(snapshotcli)
+
 	// mempool connection
 	memcli, err := app.clientCreator.NewABCIClient()
 	if err != nil {
-		return errors.Wrap(err, "Error creating ABCI client (mempool connection)")
+		return fmt.Errorf("error creating ABCI client (mempool connection): %w", err)
 	}
 	memcli.SetLogger(app.Logger.With("module", "abci-client", "connection", "mempool"))
 	if err := memcli.Start(); err != nil {
-		return errors.Wrap(err, "Error starting ABCI client (mempool connection)")
+		return fmt.Errorf("error starting ABCI client (mempool connection): %w", err)
 	}
 	app.mempoolConn = NewAppConnMempool(memcli)
 
 	// consensus connection
 	concli, err := app.clientCreator.NewABCIClient()
 	if err != nil {
-		return errors.Wrap(err, "Error creating ABCI client (consensus connection)")
+		return fmt.Errorf("error creating ABCI client (consensus connection): %w", err)
 	}
 	concli.SetLogger(app.Logger.With("module", "abci-client", "connection", "consensus"))
 	if err := concli.Start(); err != nil {
-		return errors.Wrap(err, "Error starting ABCI client (consensus connection)")
+		return fmt.Errorf("error starting ABCI client (consensus connection): %w", err)
 	}
 	app.consensusConn = NewAppConnConsensus(concli)
 

version/version.go

@@ -23,7 +23,7 @@ const (
 	TMCoreSemVer = "0.33.4"
 
 	// ABCISemVer is the semantic version of the ABCI library
-	ABCISemVer  = "0.16.2"
+	ABCISemVer  = "0.17.0"
 	ABCIVersion = ABCISemVer
 )