send plugin error stack traces over gRPC and log error locations

Signed-off-by: Steve Kriss <krisss@vmware.com>

pkg/plugin/framework/backup_item_action_client.go

@@ -19,6 +19,7 @@ package framework
 import (
 	"encoding/json"
 
+	"github.com/pkg/errors"
 	"golang.org/x/net/context"
 	"google.golang.org/grpc"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -52,9 +53,13 @@ func newBackupItemActionGRPCClient(base *clientBase, clientConn *grpc.ClientConn
 }
 
 func (c *BackupItemActionGRPCClient) AppliesTo() (velero.ResourceSelector, error) {
-	res, err := c.grpcClient.AppliesTo(context.Background(), &proto.AppliesToRequest{Plugin: c.plugin})
+	req := &proto.AppliesToRequest{
+		Plugin: c.plugin,
+	}
+
+	res, err := c.grpcClient.AppliesTo(context.Background(), req)
 	if err != nil {
-		return velero.ResourceSelector{}, err
+		return velero.ResourceSelector{}, fromGRPCError(err)
 	}
 
 	return velero.ResourceSelector{
@@ -69,12 +74,12 @@ func (c *BackupItemActionGRPCClient) AppliesTo() (velero.ResourceSelector, error
 func (c *BackupItemActionGRPCClient) Execute(item runtime.Unstructured, backup *api.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, error) {
 	itemJSON, err := json.Marshal(item.UnstructuredContent())
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, errors.WithStack(err)
 	}
 
 	backupJSON, err := json.Marshal(backup)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, errors.WithStack(err)
 	}
 
 	req := &proto.ExecuteRequest{
@@ -85,12 +90,12 @@ func (c *BackupItemActionGRPCClient) Execute(item runtime.Unstructured, backup *
 
 	res, err := c.grpcClient.Execute(context.Background(), req)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, fromGRPCError(err)
 	}
 
 	var updatedItem unstructured.Unstructured
 	if err := json.Unmarshal(res.Item, &updatedItem); err != nil {
-		return nil, nil, err
+		return nil, nil, errors.WithStack(err)
 	}
 
 	var additionalItems []velero.ResourceIdentifier

pkg/plugin/framework/backup_item_action_server.go

@@ -57,12 +57,12 @@ func (s *BackupItemActionGRPCServer) AppliesTo(ctx context.Context, req *proto.A
 
 	impl, err := s.getImpl(req.Plugin)
 	if err != nil {
-		return nil, err
+		return nil, newGRPCError(err)
 	}
 
 	resourceSelector, err := impl.AppliesTo()
 	if err != nil {
-		return nil, err
+		return nil, newGRPCError(err)
 	}
 
 	return &proto.AppliesToResponse{
@@ -83,22 +83,22 @@ func (s *BackupItemActionGRPCServer) Execute(ctx context.Context, req *proto.Exe
 
 	impl, err := s.getImpl(req.Plugin)
 	if err != nil {
-		return nil, err
+		return nil, newGRPCError(err)
 	}
 
 	var item unstructured.Unstructured
 	var backup api.Backup
 
 	if err := json.Unmarshal(req.Item, &item); err != nil {
-		return nil, err
+		return nil, newGRPCError(errors.WithStack(err))
 	}
 	if err := json.Unmarshal(req.Backup, &backup); err != nil {
-		return nil, err
+		return nil, newGRPCError(errors.WithStack(err))
 	}
 
 	updatedItem, additionalItems, err := impl.Execute(&item, &backup)
 	if err != nil {
-		return nil, err
+		return nil, newGRPCError(err)
 	}
 
 	// If the plugin implementation returned a nil updatedItem (meaning no modifications), reset updatedItem to the
@@ -109,7 +109,7 @@ func (s *BackupItemActionGRPCServer) Execute(ctx context.Context, req *proto.Exe
 	} else {
 		updatedItemJSON, err = json.Marshal(updatedItem.UnstructuredContent())
 		if err != nil {
-			return nil, err
+			return nil, newGRPCError(errors.WithStack(err))
 		}
 	}
 

pkg/plugin/framework/block_store_client.go

@@ -19,6 +19,7 @@ package framework
 import (
 	"encoding/json"
 
+	"github.com/pkg/errors"
 	"golang.org/x/net/context"
 	"google.golang.org/grpc"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -52,9 +53,16 @@ func newBlockStoreGRPCClient(base *clientBase, clientConn *grpc.ClientConn) inte
 // configuration key-value pairs. It returns an error if the BlockStore
 // cannot be initialized from the provided config.
 func (c *BlockStoreGRPCClient) Init(config map[string]string) error {
-	_, err := c.grpcClient.Init(context.Background(), &proto.InitRequest{Plugin: c.plugin, Config: config})
+	req := &proto.InitRequest{
+		Plugin: c.plugin,
+		Config: config,
+	}
 
-	return err
+	if _, err := c.grpcClient.Init(context.Background(), req); err != nil {
+		return fromGRPCError(err)
+	}
+
+	return nil
 }
 
 // CreateVolumeFromSnapshot creates a new block volume, initialized from the provided snapshot,
@@ -75,7 +83,7 @@ func (c *BlockStoreGRPCClient) CreateVolumeFromSnapshot(snapshotID, volumeType,
 
 	res, err := c.grpcClient.CreateVolumeFromSnapshot(context.Background(), req)
 	if err != nil {
-		return "", err
+		return "", fromGRPCError(err)
 	}
 
 	return res.VolumeID, nil
@@ -84,9 +92,15 @@ func (c *BlockStoreGRPCClient) CreateVolumeFromSnapshot(snapshotID, volumeType,
 // GetVolumeInfo returns the type and IOPS (if using provisioned IOPS) for a specified block
 // volume.
 func (c *BlockStoreGRPCClient) GetVolumeInfo(volumeID, volumeAZ string) (string, *int64, error) {
-	res, err := c.grpcClient.GetVolumeInfo(context.Background(), &proto.GetVolumeInfoRequest{Plugin: c.plugin, VolumeID: volumeID, VolumeAZ: volumeAZ})
+	req := &proto.GetVolumeInfoRequest{
+		Plugin:   c.plugin,
+		VolumeID: volumeID,
+		VolumeAZ: volumeAZ,
+	}
+
+	res, err := c.grpcClient.GetVolumeInfo(context.Background(), req)
 	if err != nil {
-		return "", nil, err
+		return "", nil, fromGRPCError(err)
 	}
 
 	var iops *int64
@@ -109,7 +123,7 @@ func (c *BlockStoreGRPCClient) CreateSnapshot(volumeID, volumeAZ string, tags ma
 
 	res, err := c.grpcClient.CreateSnapshot(context.Background(), req)
 	if err != nil {
-		return "", err
+		return "", fromGRPCError(err)
 	}
 
 	return res.SnapshotID, nil
@@ -117,15 +131,22 @@ func (c *BlockStoreGRPCClient) CreateSnapshot(volumeID, volumeAZ string, tags ma
 
 // DeleteSnapshot deletes the specified volume snapshot.
 func (c *BlockStoreGRPCClient) DeleteSnapshot(snapshotID string) error {
-	_, err := c.grpcClient.DeleteSnapshot(context.Background(), &proto.DeleteSnapshotRequest{Plugin: c.plugin, SnapshotID: snapshotID})
+	req := &proto.DeleteSnapshotRequest{
+		Plugin:     c.plugin,
+		SnapshotID: snapshotID,
+	}
 
-	return err
+	if _, err := c.grpcClient.DeleteSnapshot(context.Background(), req); err != nil {
+		return fromGRPCError(err)
+	}
+
+	return nil
 }
 
 func (c *BlockStoreGRPCClient) GetVolumeID(pv runtime.Unstructured) (string, error) {
 	encodedPV, err := json.Marshal(pv.UnstructuredContent())
 	if err != nil {
-		return "", err
+		return "", errors.WithStack(err)
 	}
 
 	req := &proto.GetVolumeIDRequest{
@@ -135,7 +156,7 @@ func (c *BlockStoreGRPCClient) GetVolumeID(pv runtime.Unstructured) (string, err
 
 	resp, err := c.grpcClient.GetVolumeID(context.Background(), req)
 	if err != nil {
-		return "", err
+		return "", fromGRPCError(err)
 	}
 
 	return resp.VolumeID, nil
@@ -144,7 +165,7 @@ func (c *BlockStoreGRPCClient) GetVolumeID(pv runtime.Unstructured) (string, err
 func (c *BlockStoreGRPCClient) SetVolumeID(pv runtime.Unstructured, volumeID string) (runtime.Unstructured, error) {
 	encodedPV, err := json.Marshal(pv.UnstructuredContent())
 	if err != nil {
-		return nil, err
+		return nil, errors.WithStack(err)
 	}
 
 	req := &proto.SetVolumeIDRequest{
@@ -155,13 +176,12 @@ func (c *BlockStoreGRPCClient) SetVolumeID(pv runtime.Unstructured, volumeID str
 
 	resp, err := c.grpcClient.SetVolumeID(context.Background(), req)
 	if err != nil {
-		return nil, err
+		return nil, fromGRPCError(err)
 	}
 
 	var updatedPV unstructured.Unstructured
 	if err := json.Unmarshal(resp.PersistentVolume, &updatedPV); err != nil {
-		return nil, err
-
+		return nil, errors.WithStack(err)
 	}
 
 	return &updatedPV, nil

pkg/plugin/framework/block_store_server.go

@@ -59,11 +59,11 @@ func (s *BlockStoreGRPCServer) Init(ctx context.Context, req *proto.InitRequest)
 
 	impl, err := s.getImpl(req.Plugin)
 	if err != nil {
-		return nil, err
+		return nil, newGRPCError(err)
 	}
 
 	if err := impl.Init(req.Config); err != nil {
-		return nil, err
+		return nil, newGRPCError(err)
 	}
 
 	return &proto.Empty{}, nil
@@ -80,7 +80,7 @@ func (s *BlockStoreGRPCServer) CreateVolumeFromSnapshot(ctx context.Context, req
 
 	impl, err := s.getImpl(req.Plugin)
 	if err != nil {
-		return nil, err
+		return nil, newGRPCError(err)
 	}
 
 	snapshotID := req.SnapshotID
@@ -94,7 +94,7 @@ func (s *BlockStoreGRPCServer) CreateVolumeFromSnapshot(ctx context.Context, req
 
 	volumeID, err := impl.CreateVolumeFromSnapshot(snapshotID, volumeType, volumeAZ, iops)
 	if err != nil {
-		return nil, err
+		return nil, newGRPCError(err)
 	}
 
 	return &proto.CreateVolumeResponse{VolumeID: volumeID}, nil
@@ -111,12 +111,12 @@ func (s *BlockStoreGRPCServer) GetVolumeInfo(ctx context.Context, req *proto.Get
 
 	impl, err := s.getImpl(req.Plugin)
 	if err != nil {
-		return nil, err
+		return nil, newGRPCError(err)
 	}
 
 	volumeType, iops, err := impl.GetVolumeInfo(req.VolumeID, req.VolumeAZ)
 	if err != nil {
-		return nil, err
+		return nil, newGRPCError(err)
 	}
 
 	res := &proto.GetVolumeInfoResponse{
@@ -141,12 +141,12 @@ func (s *BlockStoreGRPCServer) CreateSnapshot(ctx context.Context, req *proto.Cr
 
 	impl, err := s.getImpl(req.Plugin)
 	if err != nil {
-		return nil, err
+		return nil, newGRPCError(err)
 	}
 
 	snapshotID, err := impl.CreateSnapshot(req.VolumeID, req.VolumeAZ, req.Tags)
 	if err != nil {
-		return nil, err
+		return nil, newGRPCError(err)
 	}
 
 	return &proto.CreateSnapshotResponse{SnapshotID: snapshotID}, nil
@@ -162,11 +162,11 @@ func (s *BlockStoreGRPCServer) DeleteSnapshot(ctx context.Context, req *proto.De
 
 	impl, err := s.getImpl(req.Plugin)
 	if err != nil {
-		return nil, err
+		return nil, newGRPCError(err)
 	}
 
 	if err := impl.DeleteSnapshot(req.SnapshotID); err != nil {
-		return nil, err
+		return nil, newGRPCError(err)
 	}
 
 	return &proto.Empty{}, nil
@@ -181,18 +181,18 @@ func (s *BlockStoreGRPCServer) GetVolumeID(ctx context.Context, req *proto.GetVo
 
 	impl, err := s.getImpl(req.Plugin)
 	if err != nil {
-		return nil, err
+		return nil, newGRPCError(err)
 	}
 
 	var pv unstructured.Unstructured
 
 	if err := json.Unmarshal(req.PersistentVolume, &pv); err != nil {
-		return nil, err
+		return nil, newGRPCError(errors.WithStack(err))
 	}
 
 	volumeID, err := impl.GetVolumeID(&pv)
 	if err != nil {
-		return nil, err
+		return nil, newGRPCError(err)
 	}
 
 	return &proto.GetVolumeIDResponse{VolumeID: volumeID}, nil
@@ -207,23 +207,22 @@ func (s *BlockStoreGRPCServer) SetVolumeID(ctx context.Context, req *proto.SetVo
 
 	impl, err := s.getImpl(req.Plugin)
 	if err != nil {
-		return nil, err
+		return nil, newGRPCError(err)
 	}
 
 	var pv unstructured.Unstructured
-
 	if err := json.Unmarshal(req.PersistentVolume, &pv); err != nil {
-		return nil, err
+		return nil, newGRPCError(errors.WithStack(err))
 	}
 
 	updatedPV, err := impl.SetVolumeID(&pv, req.VolumeID)
 	if err != nil {
-		return nil, err
+		return nil, newGRPCError(err)
 	}
 
 	updatedPVBytes, err := json.Marshal(updatedPV.UnstructuredContent())
 	if err != nil {
-		return nil, err
+		return nil, newGRPCError(err)
 	}
 
 	return &proto.SetVolumeIDResponse{PersistentVolume: updatedPVBytes}, nil

pkg/plugin/framework/client_errors.go

@@ -0,0 +1,79 @@
+/*
+Copyright 2019 the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"google.golang.org/grpc/status"
+
+	proto "github.com/heptio/velero/pkg/plugin/generated"
+)
+
+// fromGRPCError takes a gRPC status error, extracts a stack trace
+// from the details if it exists, and returns an error that can
+// provide information about where it was created.
+//
+// This function should be used in the internal plugin client code to convert
+// all errors returned from the plugin server before they're passed back to
+// the rest of the Velero codebase. This will enable them to display location
+// information when they're logged.
+func fromGRPCError(err error) error {
+	statusErr, ok := status.FromError(err)
+	if !ok {
+		return statusErr.Err()
+	}
+
+	for _, detail := range statusErr.Details() {
+		switch t := detail.(type) {
+		case *proto.Stack:
+			return &protoStackError{
+				error: err,
+				stack: t,
+			}
+		}
+	}
+
+	return err
+}
+
+type protoStackError struct {
+	error
+	stack *proto.Stack
+}
+
+func (e *protoStackError) File() string {
+	if e.stack == nil || len(e.stack.Frames) < 1 {
+		return ""
+	}
+
+	return e.stack.Frames[0].File
+}
+
+func (e *protoStackError) Line() int32 {
+	if e.stack == nil || len(e.stack.Frames) < 1 {
+		return 0
+	}
+
+	return e.stack.Frames[0].Line
+}
+
+func (e *protoStackError) Function() string {
+	if e.stack == nil || len(e.stack.Frames) < 1 {
+		return ""
+	}
+
+	return e.stack.Frames[0].Function
+}

pkg/plugin/framework/handle_panic.go

@@ -17,8 +17,8 @@ limitations under the License.
 package framework
 
 import (
+	"github.com/pkg/errors"
 	"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/status"
 )
 
 // handlePanic is a panic handler for the server half of velero plugins.
@@ -27,5 +27,20 @@ func handlePanic(p interface{}) error {
 		return nil
 	}
 
-	return status.Errorf(codes.Aborted, "plugin panicked: %v", p)
+	// If p is an error with a stack trace, we want to retain
+	// it to preserve the stack trace. Otherwise, create a new
+	// error here.
+	var err error
+
+	if panicErr, ok := p.(error); !ok {
+		err = errors.Errorf("plugin panicked: %v", p)
+	} else {
+		if _, ok := panicErr.(stackTracer); ok {
+			err = panicErr
+		} else {
+			err = errors.Wrap(panicErr, "plugin panicked")
+		}
+	}
+
+	return newGRPCErrorWithCode(err, codes.Aborted)
 }

pkg/plugin/framework/object_store_client.go

@@ -20,6 +20,7 @@ import (
 	"io"
 	"time"
 
+	"github.com/pkg/errors"
 	"golang.org/x/net/context"
 	"google.golang.org/grpc"
 
@@ -53,9 +54,16 @@ func newObjectStoreGRPCClient(base *clientBase, clientConn *grpc.ClientConn) int
 // configuration key-value pairs. It returns an error if the ObjectStore
 // cannot be initialized from the provided config.
 func (c *ObjectStoreGRPCClient) Init(config map[string]string) error {
-	_, err := c.grpcClient.Init(context.Background(), &proto.InitRequest{Plugin: c.plugin, Config: config})
+	req := &proto.InitRequest{
+		Plugin: c.plugin,
+		Config: config,
+	}
 
-	return err
+	if _, err := c.grpcClient.Init(context.Background(), req); err != nil {
+		return fromGRPCError(err)
+	}
+
+	return nil
 }
 
 // PutObject creates a new object using the data in body within the specified
@@ -63,7 +71,7 @@ func (c *ObjectStoreGRPCClient) Init(config map[string]string) error {
 func (c *ObjectStoreGRPCClient) PutObject(bucket, key string, body io.Reader) error {
 	stream, err := c.grpcClient.PutObject(context.Background())
 	if err != nil {
-		return err
+		return fromGRPCError(err)
 	}
 
 	// read from the provider io.Reader into chunks, and send each one over
@@ -72,16 +80,18 @@ func (c *ObjectStoreGRPCClient) PutObject(bucket, key string, body io.Reader) er
 	for {
 		n, err := body.Read(chunk)
 		if err == io.EOF {
-			_, resErr := stream.CloseAndRecv()
-			return resErr
+			if _, resErr := stream.CloseAndRecv(); resErr != nil {
+				return fromGRPCError(resErr)
+			}
+			return nil
 		}
 		if err != nil {
 			stream.CloseSend()
-			return err
+			return errors.WithStack(err)
 		}
 
 		if err := stream.Send(&proto.PutObjectRequest{Plugin: c.plugin, Bucket: bucket, Key: key, Body: chunk[0:n]}); err != nil {
-			return err
+			return fromGRPCError(err)
 		}
 	}
 }
@@ -89,22 +99,31 @@ func (c *ObjectStoreGRPCClient) PutObject(bucket, key string, body io.Reader) er
 // GetObject retrieves the object with the given key from the specified
 // bucket in object storage.
 func (c *ObjectStoreGRPCClient) GetObject(bucket, key string) (io.ReadCloser, error) {
-	stream, err := c.grpcClient.GetObject(context.Background(), &proto.GetObjectRequest{Plugin: c.plugin, Bucket: bucket, Key: key})
+	req := &proto.GetObjectRequest{
+		Plugin: c.plugin,
+		Bucket: bucket,
+		Key:    key,
+	}
+
+	stream, err := c.grpcClient.GetObject(context.Background(), req)
 	if err != nil {
-		return nil, err
+		return nil, fromGRPCError(err)
 	}
 
 	receive := func() ([]byte, error) {
 		data, err := stream.Recv()
 		if err != nil {
-			return nil, err
+			return nil, fromGRPCError(err)
 		}
 
 		return data.Data, nil
 	}
 
 	close := func() error {
-		return stream.CloseSend()
+		if err := stream.CloseSend(); err != nil {
+			return fromGRPCError(err)
+		}
+		return nil
 	}
 
 	return &StreamReadCloser{receive: receive, close: close}, nil
@@ -123,7 +142,7 @@ func (c *ObjectStoreGRPCClient) ListCommonPrefixes(bucket, prefix, delimiter str
 
 	res, err := c.grpcClient.ListCommonPrefixes(context.Background(), req)
 	if err != nil {
-		return nil, err
+		return nil, fromGRPCError(err)
 	}
 
 	return res.Prefixes, nil
@@ -131,9 +150,15 @@ func (c *ObjectStoreGRPCClient) ListCommonPrefixes(bucket, prefix, delimiter str
 
 // ListObjects gets a list of all objects in bucket that have the same prefix.
 func (c *ObjectStoreGRPCClient) ListObjects(bucket, prefix string) ([]string, error) {
-	res, err := c.grpcClient.ListObjects(context.Background(), &proto.ListObjectsRequest{Plugin: c.plugin, Bucket: bucket, Prefix: prefix})
+	req := &proto.ListObjectsRequest{
+		Plugin: c.plugin,
+		Bucket: bucket,
+		Prefix: prefix,
+	}
+
+	res, err := c.grpcClient.ListObjects(context.Background(), req)
 	if err != nil {
-		return nil, err
+		return nil, fromGRPCError(err)
 	}
 
 	return res.Keys, nil
@@ -142,21 +167,31 @@ func (c *ObjectStoreGRPCClient) ListObjects(bucket, prefix string) ([]string, er
 // DeleteObject removes object with the specified key from the given
 // bucket.
 func (c *ObjectStoreGRPCClient) DeleteObject(bucket, key string) error {
-	_, err := c.grpcClient.DeleteObject(context.Background(), &proto.DeleteObjectRequest{Plugin: c.plugin, Bucket: bucket, Key: key})
+	req := &proto.DeleteObjectRequest{
+		Plugin: c.plugin,
+		Bucket: bucket,
+		Key:    key,
+	}
 
-	return err
+	if _, err := c.grpcClient.DeleteObject(context.Background(), req); err != nil {
+		return fromGRPCError(err)
+	}
+
+	return nil
 }
 
 // CreateSignedURL creates a pre-signed URL for the given bucket and key that expires after ttl.
 func (c *ObjectStoreGRPCClient) CreateSignedURL(bucket, key string, ttl time.Duration) (string, error) {
-	res, err := c.grpcClient.CreateSignedURL(context.Background(), &proto.CreateSignedURLRequest{
+	req := &proto.CreateSignedURLRequest{
 		Plugin: c.plugin,
 		Bucket: bucket,
 		Key:    key,
 		Ttl:    int64(ttl),
-	})
+	}
+
+	res, err := c.grpcClient.CreateSignedURL(context.Background(), req)
 	if err != nil {
-		return "", nil
+		return "", fromGRPCError(err)
 	}
 
 	return res.Url, nil

pkg/plugin/framework/object_store_server.go

@@ -59,11 +59,11 @@ func (s *ObjectStoreGRPCServer) Init(ctx context.Context, req *proto.InitRequest
 
 	impl, err := s.getImpl(req.Plugin)
 	if err != nil {
-		return nil, err
+		return nil, newGRPCError(err)
 	}
 
 	if err := impl.Init(req.Config); err != nil {
-		return nil, err
+		return nil, newGRPCError(err)
 	}
 
 	return &proto.Empty{}, nil
@@ -82,12 +82,12 @@ func (s *ObjectStoreGRPCServer) PutObject(stream proto.ObjectStore_PutObjectServ
 	// in our receive method, we'll use `first` on the first call
 	firstChunk, err := stream.Recv()
 	if err != nil {
-		return err
+		return newGRPCError(errors.WithStack(err))
 	}
 
 	impl, err := s.getImpl(firstChunk.Plugin)
 	if err != nil {
-		return err
+		return newGRPCError(err)
 	}
 
 	bucket := firstChunk.Bucket
@@ -102,7 +102,7 @@ func (s *ObjectStoreGRPCServer) PutObject(stream proto.ObjectStore_PutObjectServ
 
 		data, err := stream.Recv()
 		if err != nil {
-			return nil, err
+			return nil, errors.WithStack(err)
 		}
 		return data.Body, nil
 	}
@@ -112,10 +112,14 @@ func (s *ObjectStoreGRPCServer) PutObject(stream proto.ObjectStore_PutObjectServ
 	}
 
 	if err := impl.PutObject(bucket, key, &StreamReadCloser{receive: receive, close: close}); err != nil {
-		return err
+		return newGRPCError(err)
 	}
 
-	return stream.SendAndClose(&proto.Empty{})
+	if err := stream.SendAndClose(&proto.Empty{}); err != nil {
+		return newGRPCError(errors.WithStack(err))
+	}
+
+	return nil
 }
 
 // GetObject retrieves the object with the given key from the specified
@@ -129,12 +133,12 @@ func (s *ObjectStoreGRPCServer) GetObject(req *proto.GetObjectRequest, stream pr
 
 	impl, err := s.getImpl(req.Plugin)
 	if err != nil {
-		return err
+		return newGRPCError(err)
 	}
 
 	rdr, err := impl.GetObject(req.Bucket, req.Key)
 	if err != nil {
-		return err
+		return newGRPCError(err)
 	}
 	defer rdr.Close()
 
@@ -142,14 +146,14 @@ func (s *ObjectStoreGRPCServer) GetObject(req *proto.GetObjectRequest, stream pr
 	for {
 		n, err := rdr.Read(chunk)
 		if err != nil && err != io.EOF {
-			return err
+			return newGRPCError(errors.WithStack(err))
 		}
 		if n == 0 {
 			return nil
 		}
 
 		if err := stream.Send(&proto.Bytes{Data: chunk[0:n]}); err != nil {
-			return err
+			return newGRPCError(errors.WithStack(err))
 		}
 	}
 }
@@ -166,12 +170,12 @@ func (s *ObjectStoreGRPCServer) ListCommonPrefixes(ctx context.Context, req *pro
 
 	impl, err := s.getImpl(req.Plugin)
 	if err != nil {
-		return nil, err
+		return nil, newGRPCError(err)
 	}
 
 	prefixes, err := impl.ListCommonPrefixes(req.Bucket, req.Prefix, req.Delimiter)
 	if err != nil {
-		return nil, err
+		return nil, newGRPCError(err)
 	}
 
 	return &proto.ListCommonPrefixesResponse{Prefixes: prefixes}, nil
@@ -187,12 +191,12 @@ func (s *ObjectStoreGRPCServer) ListObjects(ctx context.Context, req *proto.List
 
 	impl, err := s.getImpl(req.Plugin)
 	if err != nil {
-		return nil, err
+		return nil, newGRPCError(err)
 	}
 
 	keys, err := impl.ListObjects(req.Bucket, req.Prefix)
 	if err != nil {
-		return nil, err
+		return nil, newGRPCError(err)
 	}
 
 	return &proto.ListObjectsResponse{Keys: keys}, nil
@@ -209,11 +213,11 @@ func (s *ObjectStoreGRPCServer) DeleteObject(ctx context.Context, req *proto.Del
 
 	impl, err := s.getImpl(req.Plugin)
 	if err != nil {
-		return nil, err
+		return nil, newGRPCError(err)
 	}
 
 	if err := impl.DeleteObject(req.Bucket, req.Key); err != nil {
-		return nil, err
+		return nil, newGRPCError(err)
 	}
 
 	return &proto.Empty{}, nil
@@ -229,12 +233,12 @@ func (s *ObjectStoreGRPCServer) CreateSignedURL(ctx context.Context, req *proto.
 
 	impl, err := s.getImpl(req.Plugin)
 	if err != nil {
-		return nil, err
+		return nil, newGRPCError(err)
 	}
 
 	url, err := impl.CreateSignedURL(req.Bucket, req.Key, time.Duration(req.Ttl))
 	if err != nil {
-		return nil, err
+		return nil, newGRPCError(err)
 	}
 
 	return &proto.CreateSignedURLResponse{Url: url}, nil

pkg/plugin/framework/restore_item_action_client.go

@@ -54,7 +54,7 @@ func newRestoreItemActionGRPCClient(base *clientBase, clientConn *grpc.ClientCon
 func (c *RestoreItemActionGRPCClient) AppliesTo() (velero.ResourceSelector, error) {
 	res, err := c.grpcClient.AppliesTo(context.Background(), &proto.AppliesToRequest{Plugin: c.plugin})
 	if err != nil {
-		return velero.ResourceSelector{}, err
+		return velero.ResourceSelector{}, fromGRPCError(err)
 	}
 
 	return velero.ResourceSelector{
@@ -69,17 +69,17 @@ func (c *RestoreItemActionGRPCClient) AppliesTo() (velero.ResourceSelector, erro
 func (c *RestoreItemActionGRPCClient) Execute(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) {
 	itemJSON, err := json.Marshal(input.Item.UnstructuredContent())
 	if err != nil {
-		return nil, err
+		return nil, errors.WithStack(err)
 	}
 
 	itemFromBackupJSON, err := json.Marshal(input.ItemFromBackup.UnstructuredContent())
 	if err != nil {
-		return nil, err
+		return nil, errors.WithStack(err)
 	}
 
 	restoreJSON, err := json.Marshal(input.Restore)
 	if err != nil {
-		return nil, err
+		return nil, errors.WithStack(err)
 	}
 
 	req := &proto.RestoreExecuteRequest{
@@ -91,12 +91,12 @@ func (c *RestoreItemActionGRPCClient) Execute(input *velero.RestoreItemActionExe
 
 	res, err := c.grpcClient.Execute(context.Background(), req)
 	if err != nil {
-		return nil, err
+		return nil, fromGRPCError(err)
 	}
 
 	var updatedItem unstructured.Unstructured
 	if err := json.Unmarshal(res.Item, &updatedItem); err != nil {
-		return nil, err
+		return nil, errors.WithStack(err)
 	}
 
 	var warning error

pkg/plugin/framework/restore_item_action_server.go

@@ -57,12 +57,12 @@ func (s *RestoreItemActionGRPCServer) AppliesTo(ctx context.Context, req *proto.
 
 	impl, err := s.getImpl(req.Plugin)
 	if err != nil {
-		return nil, err
+		return nil, newGRPCError(err)
 	}
 
 	appliesTo, err := impl.AppliesTo()
 	if err != nil {
-		return nil, err
+		return nil, newGRPCError(err)
 	}
 
 	return &proto.AppliesToResponse{
@@ -83,7 +83,7 @@ func (s *RestoreItemActionGRPCServer) Execute(ctx context.Context, req *proto.Re
 
 	impl, err := s.getImpl(req.Plugin)
 	if err != nil {
-		return nil, err
+		return nil, newGRPCError(err)
 	}
 
 	var (
@@ -93,15 +93,15 @@ func (s *RestoreItemActionGRPCServer) Execute(ctx context.Context, req *proto.Re
 	)
 
 	if err := json.Unmarshal(req.Item, &item); err != nil {
-		return nil, err
+		return nil, newGRPCError(errors.WithStack(err))
 	}
 
 	if err := json.Unmarshal(req.ItemFromBackup, &itemFromBackup); err != nil {
-		return nil, err
+		return nil, newGRPCError(errors.WithStack(err))
 	}
 
 	if err := json.Unmarshal(req.Restore, &restoreObj); err != nil {
-		return nil, err
+		return nil, newGRPCError(errors.WithStack(err))
 	}
 
 	executeOutput, err := impl.Execute(&velero.RestoreItemActionExecuteInput{
@@ -110,12 +110,12 @@ func (s *RestoreItemActionGRPCServer) Execute(ctx context.Context, req *proto.Re
 		Restore:        &restoreObj,
 	})
 	if err != nil {
-		return nil, err
+		return nil, newGRPCError(err)
 	}
 
 	updatedItem, err := json.Marshal(executeOutput.UpdatedItem)
 	if err != nil {
-		return nil, err
+		return nil, newGRPCError(errors.WithStack(err))
 	}
 
 	var warnMessage string

pkg/plugin/framework/server_errors.go

@@ -0,0 +1,85 @@
+/*
+Copyright 2019 the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	goproto "github.com/golang/protobuf/proto"
+	"github.com/pkg/errors"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+
+	proto "github.com/heptio/velero/pkg/plugin/generated"
+	"github.com/heptio/velero/pkg/util/logging"
+)
+
+// newGRPCErrorWithCode wraps err in a gRPC status error with the error's stack trace
+// included in the details if it exists. This provides an easy way to send
+// stack traces from plugin servers across the wire to the plugin client.
+//
+// This function should be used in the internal plugin server code to wrap
+// all errors before they're returned.
+func newGRPCErrorWithCode(err error, code codes.Code, details ...goproto.Message) error {
+	// if it's already a gRPC status error, use it; otherwise, create a new one
+	statusErr, ok := status.FromError(err)
+	if !ok {
+		statusErr = status.New(code, err.Error())
+	}
+
+	// get a Stack for the error and add it to details
+	if stack := errorStack(err); stack != nil {
+		details = append(details, stack)
+	}
+
+	statusErr, err = statusErr.WithDetails(details...)
+	if err != nil {
+		return status.Errorf(codes.Unknown, "error adding details to the gRPC error: %v", err)
+	}
+
+	return statusErr.Err()
+}
+
+// newGRPCError is a convenience functino for creating a new gRPC error
+// with code = codes.Unknown
+func newGRPCError(err error, details ...goproto.Message) error {
+	return newGRPCErrorWithCode(err, codes.Unknown, details...)
+}
+
+// errorStack gets a stack trace, if it exists, from the provided error, and
+// returns it as a *proto.Stack.
+func errorStack(err error) *proto.Stack {
+	stackTracer, ok := err.(stackTracer)
+	if !ok {
+		return nil
+	}
+
+	stackTrace := new(proto.Stack)
+	for _, frame := range stackTracer.StackTrace() {
+		location := logging.GetFrameLocationInfo(frame)
+
+		stackTrace.Frames = append(stackTrace.Frames, &proto.StackFrame{
+			File:     location.File,
+			Line:     int32(location.Line),
+			Function: location.Function,
+		})
+	}
+
+	return stackTrace
+}
+
+type stackTracer interface {
+	StackTrace() errors.StackTrace
+}