Initial commit

Signed-off-by: Andy Goldstein <andy.goldstein@gmail.com>

pkg/cloudprovider/aws/block_storage_adapter.go

@@ -0,0 +1,164 @@
+/*
+Copyright 2017 Heptio Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aws
+
+import (
+	"fmt"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+
+	"github.com/heptio/ark/pkg/cloudprovider"
+)
+
+var _ cloudprovider.BlockStorageAdapter = &blockStorageAdapter{}
+
+type blockStorageAdapter struct {
+	ec2 *ec2.EC2
+	az  string
+}
+
+func (op *blockStorageAdapter) CreateVolumeFromSnapshot(snapshotID, volumeType string, iops *int) (volumeID string, err error) {
+	req := &ec2.CreateVolumeInput{
+		SnapshotId:       &snapshotID,
+		AvailabilityZone: &op.az,
+		VolumeType:       &volumeType,
+	}
+
+	if iops != nil {
+		req.SetIops(int64(*iops))
+	}
+
+	res, err := op.ec2.CreateVolume(req)
+	if err != nil {
+		return "", err
+	}
+
+	return *res.VolumeId, nil
+}
+
+func (op *blockStorageAdapter) GetVolumeInfo(volumeID string) (string, *int, error) {
+	req := &ec2.DescribeVolumesInput{
+		VolumeIds: []*string{&volumeID},
+	}
+
+	res, err := op.ec2.DescribeVolumes(req)
+	if err != nil {
+		return "", nil, err
+	}
+
+	if len(res.Volumes) != 1 {
+		return "", nil, fmt.Errorf("Expected one volume from DescribeVolumes for volume ID %v, got %v", volumeID, len(res.Volumes))
+	}
+
+	vol := res.Volumes[0]
+
+	var (
+		volumeType string
+		iops       int
+	)
+
+	if vol.VolumeType != nil {
+		volumeType = *vol.VolumeType
+	}
+
+	if vol.Iops != nil {
+		iops = int(*vol.Iops)
+	}
+
+	return volumeType, &iops, nil
+}
+
+func (op *blockStorageAdapter) IsVolumeReady(volumeID string) (ready bool, err error) {
+	req := &ec2.DescribeVolumesInput{
+		VolumeIds: []*string{&volumeID},
+	}
+
+	res, err := op.ec2.DescribeVolumes(req)
+	if err != nil {
+		return false, err
+	}
+	if len(res.Volumes) != 1 {
+		return false, fmt.Errorf("Expected one volume from DescribeVolumes for volume ID %v, got %v", volumeID, len(res.Volumes))
+	}
+
+	return *res.Volumes[0].State == ec2.VolumeStateAvailable, nil
+}
+
+func (op *blockStorageAdapter) ListSnapshots(tagFilters map[string]string) ([]string, error) {
+	req := &ec2.DescribeSnapshotsInput{}
+
+	for k, v := range tagFilters {
+		filter := &ec2.Filter{}
+		filter.SetName(k)
+		filter.SetValues([]*string{&v})
+
+		req.Filters = append(req.Filters, filter)
+	}
+
+	res, err := op.ec2.DescribeSnapshots(req)
+	if err != nil {
+		return nil, err
+	}
+
+	var ret []string
+
+	for _, snapshot := range res.Snapshots {
+		ret = append(ret, *snapshot.SnapshotId)
+	}
+
+	return ret, nil
+}
+
+func (op *blockStorageAdapter) CreateSnapshot(volumeID string, tags map[string]string) (string, error) {
+	req := &ec2.CreateSnapshotInput{
+		VolumeId: &volumeID,
+	}
+
+	res, err := op.ec2.CreateSnapshot(req)
+	if err != nil {
+		return "", err
+	}
+
+	tagsReq := &ec2.CreateTagsInput{}
+	tagsReq.SetResources([]*string{res.SnapshotId})
+
+	ec2Tags := make([]*ec2.Tag, 0, len(tags))
+
+	for k, v := range tags {
+		key := k
+		val := v
+
+		tag := &ec2.Tag{Key: &key, Value: &val}
+		ec2Tags = append(ec2Tags, tag)
+	}
+
+	tagsReq.SetTags(ec2Tags)
+
+	_, err = op.ec2.CreateTags(tagsReq)
+
+	return *res.SnapshotId, err
+}
+
+func (op *blockStorageAdapter) DeleteSnapshot(snapshotID string) error {
+	req := &ec2.DeleteSnapshotInput{
+		SnapshotId: &snapshotID,
+	}
+
+	_, err := op.ec2.DeleteSnapshot(req)
+
+	return err
+}

pkg/cloudprovider/aws/object_storage_adapter.go

@@ -0,0 +1,88 @@
+/*
+Copyright 2017 Heptio Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aws
+
+import (
+	"io"
+
+	"github.com/aws/aws-sdk-go/service/s3"
+
+	"github.com/heptio/ark/pkg/cloudprovider"
+)
+
+var _ cloudprovider.ObjectStorageAdapter = &objectStorageAdapter{}
+
+type objectStorageAdapter struct {
+	s3 *s3.S3
+}
+
+func (op *objectStorageAdapter) PutObject(bucket string, key string, body io.ReadSeeker) error {
+	req := &s3.PutObjectInput{
+		Bucket: &bucket,
+		Key:    &key,
+		Body:   body,
+	}
+
+	_, err := op.s3.PutObject(req)
+
+	return err
+}
+
+func (op *objectStorageAdapter) GetObject(bucket string, key string) (io.ReadCloser, error) {
+	req := &s3.GetObjectInput{
+		Bucket: &bucket,
+		Key:    &key,
+	}
+
+	res, err := op.s3.GetObject(req)
+	if err != nil {
+		return nil, err
+	}
+
+	return res.Body, nil
+}
+
+func (op *objectStorageAdapter) ListCommonPrefixes(bucket string, delimiter string) ([]string, error) {
+	req := &s3.ListObjectsV2Input{
+		Bucket:    &bucket,
+		Delimiter: &delimiter,
+	}
+
+	res, err := op.s3.ListObjectsV2(req)
+	if err != nil {
+		return nil, err
+	}
+
+	ret := make([]string, 0, len(res.CommonPrefixes))
+
+	for _, prefix := range res.CommonPrefixes {
+		ret = append(ret, *prefix.Prefix)
+	}
+
+	return ret, nil
+}
+
+func (op *objectStorageAdapter) DeleteObject(bucket string, key string) error {
+	req := &s3.DeleteObjectInput{
+		Bucket: &bucket,
+		Key:    &key,
+	}
+
+	_, err := op.s3.DeleteObject(req)
+
+	return err
+}

pkg/cloudprovider/aws/storage_adapter.go

@@ -0,0 +1,62 @@
+/*
+Copyright 2017 Heptio Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aws
+
+import (
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/s3"
+
+	"github.com/heptio/ark/pkg/cloudprovider"
+)
+
+type storageAdapter struct {
+	blockStorage  *blockStorageAdapter
+	objectStorage *objectStorageAdapter
+}
+
+var _ cloudprovider.StorageAdapter = &storageAdapter{}
+
+func NewStorageAdapter(config *aws.Config, availabilityZone string) (cloudprovider.StorageAdapter, error) {
+	sess, err := session.NewSession(config)
+	if err != nil {
+		return nil, err
+	}
+
+	if _, err := sess.Config.Credentials.Get(); err != nil {
+		return nil, err
+	}
+
+	return &storageAdapter{
+		blockStorage: &blockStorageAdapter{
+			ec2: ec2.New(sess),
+			az:  availabilityZone,
+		},
+		objectStorage: &objectStorageAdapter{
+			s3: s3.New(sess),
+		},
+	}, nil
+}
+
+func (op *storageAdapter) ObjectStorage() cloudprovider.ObjectStorageAdapter {
+	return op.objectStorage
+}
+
+func (op *storageAdapter) BlockStorage() cloudprovider.BlockStorageAdapter {
+	return op.blockStorage
+}

pkg/cloudprovider/azure/block_storage_adapter.go

@@ -0,0 +1,187 @@
+/*
+Copyright 2017 Heptio Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azure
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	azure "github.com/Azure/azure-sdk-for-go/arm/disk"
+	"github.com/satori/uuid"
+
+	"github.com/heptio/ark/pkg/cloudprovider"
+)
+
+type blockStorageAdapter struct {
+	disks         *azure.DisksClient
+	snaps         *azure.SnapshotsClient
+	subscription  string
+	resourceGroup string
+	location      string
+	apiTimeout    time.Duration
+}
+
+var _ cloudprovider.BlockStorageAdapter = &blockStorageAdapter{}
+
+func (op *blockStorageAdapter) CreateVolumeFromSnapshot(snapshotID, volumeType string, iops *int) (string, error) {
+	fullSnapshotName := getFullSnapshotName(op.subscription, op.resourceGroup, snapshotID)
+	diskName := "restore-" + uuid.NewV4().String()
+
+	disk := azure.Model{
+		Name:     &diskName,
+		Location: &op.location,
+		Properties: &azure.Properties{
+			CreationData: &azure.CreationData{
+				CreateOption:     azure.Copy,
+				SourceResourceID: &fullSnapshotName,
+			},
+			AccountType: azure.StorageAccountTypes(volumeType),
+		},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), op.apiTimeout)
+	defer cancel()
+
+	_, errChan := op.disks.CreateOrUpdate(op.resourceGroup, *disk.Name, disk, ctx.Done())
+
+	err := <-errChan
+
+	if err != nil {
+		return "", err
+	}
+	return diskName, nil
+}
+
+func (op *blockStorageAdapter) GetVolumeInfo(volumeID string) (string, *int, error) {
+	res, err := op.disks.Get(op.resourceGroup, volumeID)
+	if err != nil {
+		return "", nil, err
+	}
+
+	return string(res.AccountType), nil, nil
+}
+
+func (op *blockStorageAdapter) IsVolumeReady(volumeID string) (ready bool, err error) {
+	res, err := op.disks.Get(op.resourceGroup, volumeID)
+	if err != nil {
+		return false, err
+	}
+
+	if res.ProvisioningState == nil {
+		return false, errors.New("nil ProvisioningState returned from Get call")
+	}
+
+	return *res.ProvisioningState == "Succeeded", nil
+}
+
+func (op *blockStorageAdapter) ListSnapshots(tagFilters map[string]string) ([]string, error) {
+	res, err := op.snaps.ListByResourceGroup(op.resourceGroup)
+	if err != nil {
+		return nil, err
+	}
+
+	if res.Value == nil {
+		return nil, errors.New("nil Value returned from ListByResourceGroup call")
+	}
+
+	ret := make([]string, 0, len(*res.Value))
+Snapshot:
+	for _, snap := range *res.Value {
+		if snap.Tags == nil && len(tagFilters) > 0 {
+			continue
+		}
+		if snap.ID == nil {
+			continue
+		}
+
+		// Azure doesn't offer tag-filtering through the API so we have to manually
+		// filter results. Require all filter keys to be present, with matching vals.
+		for filterKey, filterVal := range tagFilters {
+			if val, ok := (*snap.Tags)[filterKey]; !ok || val == nil || *val != filterVal {
+				continue Snapshot
+			}
+		}
+
+		ret = append(ret, *snap.Name)
+	}
+
+	return ret, nil
+}
+
+func (op *blockStorageAdapter) CreateSnapshot(volumeID string, tags map[string]string) (string, error) {
+	fullDiskName := getFullDiskName(op.subscription, op.resourceGroup, volumeID)
+	// snapshot names must be <= 80 characters long
+	var snapshotName string
+	suffix := "-" + uuid.NewV4().String()
+
+	if len(volumeID) <= (80 - len(suffix)) {
+		snapshotName = volumeID + suffix
+	} else {
+		snapshotName = volumeID[0:80-len(suffix)] + suffix
+	}
+
+	snap := azure.Snapshot{
+		Name: &snapshotName,
+		Properties: &azure.Properties{
+			CreationData: &azure.CreationData{
+				CreateOption:     azure.Copy,
+				SourceResourceID: &fullDiskName,
+			},
+		},
+		Tags:     &map[string]*string{},
+		Location: &op.location,
+	}
+
+	for k, v := range tags {
+		val := v
+		(*snap.Tags)[k] = &val
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), op.apiTimeout)
+	defer cancel()
+
+	_, errChan := op.snaps.CreateOrUpdate(op.resourceGroup, *snap.Name, snap, ctx.Done())
+
+	err := <-errChan
+
+	if err != nil {
+		return "", err
+	}
+
+	return snapshotName, nil
+}
+
+func (op *blockStorageAdapter) DeleteSnapshot(snapshotID string) error {
+	ctx, cancel := context.WithTimeout(context.Background(), op.apiTimeout)
+	defer cancel()
+
+	_, errChan := op.snaps.Delete(op.resourceGroup, snapshotID, ctx.Done())
+
+	err := <-errChan
+
+	return err
+}
+
+func getFullDiskName(subscription string, resourceGroup string, diskName string) string {
+	return fmt.Sprintf("/subscriptions/%v/resourceGroups/%v/providers/Microsoft.Compute/disks/%v", subscription, resourceGroup, diskName)
+}
+
+func getFullSnapshotName(subscription string, resourceGroup string, snapshotName string) string {
+	return fmt.Sprintf("/subscriptions/%v/resourceGroups/%v/providers/Microsoft.Compute/snapshots/%v", subscription, resourceGroup, snapshotName)
+}

pkg/cloudprovider/azure/object_storage_adapter.go

@@ -0,0 +1,138 @@
+/*
+Copyright 2017 Heptio Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azure
+
+import (
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/Azure/azure-sdk-for-go/storage"
+
+	"github.com/heptio/ark/pkg/cloudprovider"
+)
+
+// ref. https://github.com/Azure-Samples/storage-blob-go-getting-started/blob/master/storageExample.go
+
+type objectStorageAdapter struct {
+	blobClient *storage.BlobStorageClient
+}
+
+var _ cloudprovider.ObjectStorageAdapter = &objectStorageAdapter{}
+
+func (op *objectStorageAdapter) PutObject(bucket string, key string, body io.ReadSeeker) error {
+	container, err := getContainerReference(op.blobClient, bucket)
+	if err != nil {
+		return err
+	}
+
+	blob, err := getBlobReference(container, key)
+	if err != nil {
+		return err
+	}
+
+	// TODO having to seek to end/back to beginning to get
+	// length here is ugly. refactor to make this better.
+	len, err := body.Seek(0, io.SeekEnd)
+	if err != nil {
+		return err
+	}
+
+	blob.Properties.ContentLength = len
+
+	if _, err := body.Seek(0, 0); err != nil {
+		return err
+	}
+
+	return blob.CreateBlockBlobFromReader(body, nil)
+}
+
+func (op *objectStorageAdapter) GetObject(bucket string, key string) (io.ReadCloser, error) {
+	container, err := getContainerReference(op.blobClient, bucket)
+	if err != nil {
+		return nil, err
+	}
+
+	blob, err := getBlobReference(container, key)
+	if err != nil {
+		return nil, err
+	}
+
+	res, err := blob.Get(nil)
+	if err != nil {
+		return nil, err
+	}
+
+	return res, nil
+}
+
+func (op *objectStorageAdapter) ListCommonPrefixes(bucket string, delimiter string) ([]string, error) {
+	container, err := getContainerReference(op.blobClient, bucket)
+	if err != nil {
+		return nil, err
+	}
+
+	params := storage.ListBlobsParameters{
+		Delimiter: delimiter,
+	}
+
+	res, err := container.ListBlobs(params)
+	if err != nil {
+		return nil, err
+	}
+
+	// Azure returns prefixes inclusive of the last delimiter. We need to strip
+	// it.
+	ret := make([]string, 0, len(res.BlobPrefixes))
+	for _, prefix := range res.BlobPrefixes {
+		ret = append(ret, prefix[0:strings.LastIndex(prefix, delimiter)])
+	}
+
+	return ret, nil
+}
+
+func (op *objectStorageAdapter) DeleteObject(bucket string, key string) error {
+	container, err := getContainerReference(op.blobClient, bucket)
+	if err != nil {
+		return err
+	}
+
+	blob, err := getBlobReference(container, key)
+	if err != nil {
+		return err
+	}
+
+	return blob.Delete(nil)
+}
+
+func getContainerReference(blobClient *storage.BlobStorageClient, bucket string) (*storage.Container, error) {
+	container := blobClient.GetContainerReference(bucket)
+	if container == nil {
+		return nil, fmt.Errorf("unable to get container reference for bucket %v", bucket)
+	}
+
+	return container, nil
+}
+
+func getBlobReference(container *storage.Container, key string) (*storage.Blob, error) {
+	blob := container.GetBlobReference(key)
+	if blob == nil {
+		return nil, fmt.Errorf("unable to get blob reference for key %v", key)
+	}
+
+	return blob, nil
+}

pkg/cloudprovider/azure/storage_adapter.go

@@ -0,0 +1,103 @@
+/*
+Copyright 2017 Heptio Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azure
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/Azure/azure-sdk-for-go/arm/disk"
+	"github.com/Azure/azure-sdk-for-go/arm/examples/helpers"
+	"github.com/Azure/azure-sdk-for-go/storage"
+	"github.com/Azure/go-autorest/autorest/azure"
+
+	"github.com/heptio/ark/pkg/cloudprovider"
+)
+
+const (
+	azureClientIDKey         string = "AZURE_CLIENT_ID"
+	azureClientSecretKey     string = "AZURE_CLIENT_SECRET"
+	azureSubscriptionIDKey   string = "AZURE_SUBSCRIPTION_ID"
+	azureTenantIDKey         string = "AZURE_TENANT_ID"
+	azureStorageAccountIDKey string = "AZURE_STORAGE_ACCOUNT_ID"
+	azureStorageKeyKey       string = "AZURE_STORAGE_KEY"
+	azureResourceGroupKey    string = "AZURE_RESOURCE_GROUP"
+)
+
+type storageAdapter struct {
+	objectStorage *objectStorageAdapter
+	blockStorage  *blockStorageAdapter
+}
+
+var _ cloudprovider.StorageAdapter = &storageAdapter{}
+
+func NewStorageAdapter(location string, apiTimeout time.Duration) (cloudprovider.StorageAdapter, error) {
+	cfg := map[string]string{
+		azureClientIDKey:         "",
+		azureClientSecretKey:     "",
+		azureSubscriptionIDKey:   "",
+		azureTenantIDKey:         "",
+		azureStorageAccountIDKey: "",
+		azureStorageKeyKey:       "",
+		azureResourceGroupKey:    "",
+	}
+
+	for key := range cfg {
+		cfg[key] = os.Getenv(key)
+	}
+
+	spt, err := helpers.NewServicePrincipalTokenFromCredentials(cfg, azure.PublicCloud.ResourceManagerEndpoint)
+	if err != nil {
+		return nil, fmt.Errorf("error creating new service principal: %v", err)
+	}
+
+	disksClient := disk.NewDisksClient(cfg[azureSubscriptionIDKey])
+	snapsClient := disk.NewSnapshotsClient(cfg[azureSubscriptionIDKey])
+
+	disksClient.Authorizer = spt
+	snapsClient.Authorizer = spt
+
+	storageClient, _ := storage.NewBasicClient(cfg[azureStorageAccountIDKey], cfg[azureStorageKeyKey])
+	blobClient := storageClient.GetBlobService()
+
+	if apiTimeout == 0 {
+		apiTimeout = time.Minute
+	}
+
+	return &storageAdapter{
+		objectStorage: &objectStorageAdapter{
+			blobClient: &blobClient,
+		},
+		blockStorage: &blockStorageAdapter{
+			disks:         &disksClient,
+			snaps:         &snapsClient,
+			subscription:  cfg[azureSubscriptionIDKey],
+			resourceGroup: cfg[azureResourceGroupKey],
+			location:      location,
+			apiTimeout:    apiTimeout,
+		},
+	}, nil
+}
+
+func (op *storageAdapter) ObjectStorage() cloudprovider.ObjectStorageAdapter {
+	return op.objectStorage
+}
+
+func (op *storageAdapter) BlockStorage() cloudprovider.BlockStorageAdapter {
+	return op.blockStorage
+}

pkg/cloudprovider/backup_cache.go

@@ -0,0 +1,92 @@
+/*
+Copyright 2017 Heptio Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cloudprovider
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"github.com/golang/glog"
+
+	"k8s.io/apimachinery/pkg/util/wait"
+
+	"github.com/heptio/ark/pkg/apis/ark/v1"
+)
+
+// backupCacheBucket holds the backups and error from a GetAllBackups call.
+type backupCacheBucket struct {
+	backups []*v1.Backup
+	error   error
+}
+
+// backupCache caches GetAllBackups calls, refreshing them periodically.
+type backupCache struct {
+	delegate BackupGetter
+	lock     sync.RWMutex
+	// This doesn't really need to be a map right now, but if we ever move to supporting multiple
+	// buckets, this will be ready for it.
+	buckets map[string]*backupCacheBucket
+}
+
+var _ BackupGetter = &backupCache{}
+
+// NewBackupCache returns a new backup cache that refreshes from delegate every resyncPeriod.
+func NewBackupCache(ctx context.Context, delegate BackupGetter, resyncPeriod time.Duration) BackupGetter {
+	c := &backupCache{
+		delegate: delegate,
+		buckets:  make(map[string]*backupCacheBucket),
+	}
+
+	// Start the goroutine to refresh all buckets every resyncPeriod. This stops when ctx.Done() is
+	// available.
+	go wait.Until(c.refresh, resyncPeriod, ctx.Done())
+
+	return c
+}
+
+// refresh refreshes all the buckets currently in the cache by doing a live lookup via c.delegate.
+func (c *backupCache) refresh() {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	glog.V(4).Infof("refreshing all cached backup lists from object storage")
+
+	for bucketName, bucket := range c.buckets {
+		glog.V(4).Infof("refreshing bucket %q", bucketName)
+		bucket.backups, bucket.error = c.delegate.GetAllBackups(bucketName)
+	}
+}
+
+func (c *backupCache) GetAllBackups(bucketName string) ([]*v1.Backup, error) {
+	c.lock.RLock()
+	bucket, found := c.buckets[bucketName]
+	c.lock.RUnlock()
+	if found {
+		glog.V(4).Infof("returning cached backup list for bucket %q", bucketName)
+		return bucket.backups, bucket.error
+	}
+
+	glog.V(4).Infof("bucket %q is not in cache - doing a live lookup", bucketName)
+
+	backups, err := c.delegate.GetAllBackups(bucketName)
+	c.lock.Lock()
+	c.buckets[bucketName] = &backupCacheBucket{backups: backups, error: err}
+	c.lock.Unlock()
+
+	return backups, err
+}

pkg/cloudprovider/backup_cache_test.go

@@ -0,0 +1,160 @@
+/*
+Copyright 2017 Heptio Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cloudprovider
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/heptio/ark/pkg/apis/ark/v1"
+	"github.com/heptio/ark/pkg/util/test"
+)
+
+func TestNewBackupCache(t *testing.T) {
+	delegate := &test.FakeBackupService{}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	c := NewBackupCache(ctx, delegate, 100*time.Millisecond)
+
+	// nothing in cache, live lookup
+	bucket1 := []*v1.Backup{
+		test.NewTestBackup().WithName("backup1").Backup,
+		test.NewTestBackup().WithName("backup2").Backup,
+	}
+	delegate.On("GetAllBackups", "bucket1").Return(bucket1, nil).Once()
+
+	// should be updated via refresh
+	updatedBucket1 := []*v1.Backup{
+		test.NewTestBackup().WithName("backup2").Backup,
+	}
+	delegate.On("GetAllBackups", "bucket1").Return(updatedBucket1, nil)
+
+	// nothing in cache, live lookup
+	bucket2 := []*v1.Backup{
+		test.NewTestBackup().WithName("backup5").Backup,
+		test.NewTestBackup().WithName("backup6").Backup,
+	}
+	delegate.On("GetAllBackups", "bucket2").Return(bucket2, nil).Once()
+
+	// should be updated via refresh
+	updatedBucket2 := []*v1.Backup{
+		test.NewTestBackup().WithName("backup7").Backup,
+	}
+	delegate.On("GetAllBackups", "bucket2").Return(updatedBucket2, nil)
+
+	backups, err := c.GetAllBackups("bucket1")
+	assert.Equal(t, bucket1, backups)
+	assert.NoError(t, err)
+
+	backups, err = c.GetAllBackups("bucket2")
+	assert.Equal(t, bucket2, backups)
+	assert.NoError(t, err)
+
+	var done1, done2 bool
+	for {
+		select {
+		case <-ctx.Done():
+			t.Fatal("timed out")
+		default:
+			if done1 && done2 {
+				return
+			}
+		}
+
+		backups, err = c.GetAllBackups("bucket1")
+		if len(backups) == 1 {
+			if assert.Equal(t, updatedBucket1[0], backups[0]) {
+				done1 = true
+			}
+		}
+
+		backups, err = c.GetAllBackups("bucket2")
+		if len(backups) == 1 {
+			if assert.Equal(t, updatedBucket2[0], backups[0]) {
+				done2 = true
+			}
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+}
+
+func TestBackupCacheRefresh(t *testing.T) {
+	delegate := &test.FakeBackupService{}
+
+	c := &backupCache{
+		delegate: delegate,
+		buckets: map[string]*backupCacheBucket{
+			"bucket1": {},
+			"bucket2": {},
+		},
+	}
+
+	bucket1 := []*v1.Backup{
+		test.NewTestBackup().WithName("backup1").Backup,
+		test.NewTestBackup().WithName("backup2").Backup,
+	}
+	delegate.On("GetAllBackups", "bucket1").Return(bucket1, nil)
+
+	delegate.On("GetAllBackups", "bucket2").Return(nil, errors.New("bad"))
+
+	c.refresh()
+
+	assert.Equal(t, bucket1, c.buckets["bucket1"].backups)
+	assert.NoError(t, c.buckets["bucket1"].error)
+
+	assert.Empty(t, c.buckets["bucket2"].backups)
+	assert.EqualError(t, c.buckets["bucket2"].error, "bad")
+}
+
+func TestBackupCacheGetAllBackupsUsesCacheIfPresent(t *testing.T) {
+	delegate := &test.FakeBackupService{}
+
+	bucket1 := []*v1.Backup{
+		test.NewTestBackup().WithName("backup1").Backup,
+		test.NewTestBackup().WithName("backup2").Backup,
+	}
+
+	c := &backupCache{
+		delegate: delegate,
+		buckets: map[string]*backupCacheBucket{
+			"bucket1": {
+				backups: bucket1,
+			},
+		},
+	}
+
+	bucket2 := []*v1.Backup{
+		test.NewTestBackup().WithName("backup3").Backup,
+		test.NewTestBackup().WithName("backup4").Backup,
+	}
+
+	delegate.On("GetAllBackups", "bucket2").Return(bucket2, nil)
+
+	backups, err := c.GetAllBackups("bucket1")
+	assert.Equal(t, bucket1, backups)
+	assert.NoError(t, err)
+
+	backups, err = c.GetAllBackups("bucket2")
+	assert.Equal(t, bucket2, backups)
+	assert.NoError(t, err)
+}

pkg/cloudprovider/backup_service.go

@@ -0,0 +1,184 @@
+/*
+Copyright 2017 Heptio Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cloudprovider
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"time"
+
+	"github.com/golang/glog"
+
+	"k8s.io/apimachinery/pkg/util/errors"
+
+	api "github.com/heptio/ark/pkg/apis/ark/v1"
+	"github.com/heptio/ark/pkg/generated/clientset/scheme"
+)
+
+// BackupService contains methods for working with backups in object storage.
+type BackupService interface {
+	BackupGetter
+	// UploadBackup uploads the specified Ark backup of a set of Kubernetes API objects, whose manifests are
+	// stored in the specified file, into object storage in an Ark bucket, tagged with Ark metadata. Returns
+	// an error if a problem is encountered accessing the file or performing the upload via the cloud API.
+	UploadBackup(bucket, name string, metadata, backup io.ReadSeeker) error
+
+	// DownloadBackup downloads an Ark backup with the specified object key from object storage via the cloud API.
+	// It returns the snapshot metadata and data (separately), or an error if a problem is encountered
+	// downloading or reading the file from the cloud API.
+	DownloadBackup(bucket, name string) (io.ReadCloser, error)
+
+	// DeleteBackup deletes the backup content in object storage for the given api.Backup.
+	DeleteBackup(bucket, backupName string) error
+}
+
+// BackupGetter knows how to list backups in object storage.
+type BackupGetter interface {
+	// GetAllBackups lists all the api.Backups in object storage for the given bucket.
+	GetAllBackups(bucket string) ([]*api.Backup, error)
+}
+
+const (
+	metadataFileFormatString string = "%s/ark-backup.json"
+	backupFileFormatString   string = "%s/%s.tar.gz"
+)
+
+type backupService struct {
+	objectStorage ObjectStorageAdapter
+}
+
+var _ BackupService = &backupService{}
+var _ BackupGetter = &backupService{}
+
+// NewBackupService creates a backup service using the provided object storage adapter
+func NewBackupService(objectStorage ObjectStorageAdapter) BackupService {
+	return &backupService{
+		objectStorage: objectStorage,
+	}
+}
+
+func (br *backupService) UploadBackup(bucket, backupName string, metadata, backup io.ReadSeeker) error {
+	// upload metadata file
+	metadataKey := fmt.Sprintf(metadataFileFormatString, backupName)
+	if err := br.objectStorage.PutObject(bucket, metadataKey, metadata); err != nil {
+		return err
+	}
+
+	// upload tar file
+	if err := br.objectStorage.PutObject(bucket, fmt.Sprintf(backupFileFormatString, backupName, backupName), backup); err != nil {
+		// try to delete the metadata file since the data upload failed
+		deleteErr := br.objectStorage.DeleteObject(bucket, metadataKey)
+
+		return errors.NewAggregate([]error{err, deleteErr})
+	}
+
+	return nil
+}
+
+func (br *backupService) DownloadBackup(bucket, backupName string) (io.ReadCloser, error) {
+	return br.objectStorage.GetObject(bucket, fmt.Sprintf(backupFileFormatString, backupName, backupName))
+}
+
+func (br *backupService) GetAllBackups(bucket string) ([]*api.Backup, error) {
+	prefixes, err := br.objectStorage.ListCommonPrefixes(bucket, "/")
+	if err != nil {
+		return nil, err
+	}
+	if len(prefixes) == 0 {
+		return []*api.Backup{}, nil
+	}
+
+	output := make([]*api.Backup, 0, len(prefixes))
+
+	decoder := scheme.Codecs.UniversalDecoder(api.SchemeGroupVersion)
+
+	for _, backupDir := range prefixes {
+		err := func() error {
+			key := fmt.Sprintf(metadataFileFormatString, backupDir)
+
+			res, err := br.objectStorage.GetObject(bucket, key)
+			if err != nil {
+				return err
+			}
+			defer res.Close()
+
+			data, err := ioutil.ReadAll(res)
+			if err != nil {
+				return err
+			}
+
+			obj, _, err := decoder.Decode(data, nil, nil)
+			if err != nil {
+				return err
+			}
+
+			backup, ok := obj.(*api.Backup)
+			if !ok {
+				return fmt.Errorf("unexpected type for %s/%s: %T", bucket, key, obj)
+			}
+
+			output = append(output, backup)
+
+			return nil
+		}()
+
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return output, nil
+}
+
+func (br *backupService) DeleteBackup(bucket, backupName string) error {
+	var errs []error
+
+	key := fmt.Sprintf(backupFileFormatString, backupName, backupName)
+	glog.V(4).Infof("Trying to delete bucket=%s, key=%s", bucket, key)
+	if err := br.objectStorage.DeleteObject(bucket, key); err != nil {
+		errs = append(errs, err)
+	}
+
+	key = fmt.Sprintf(metadataFileFormatString, backupName)
+	glog.V(4).Infof("Trying to delete bucket=%s, key=%s", bucket, key)
+	if err := br.objectStorage.DeleteObject(bucket, key); err != nil {
+		errs = append(errs, err)
+	}
+
+	return errors.NewAggregate(errs)
+}
+
+// cachedBackupService wraps a real backup service with a cache for getting cloud backups.
+type cachedBackupService struct {
+	BackupService
+	cache BackupGetter
+}
+
+// NewBackupServiceWithCachedBackupGetter returns a BackupService that uses a cache for
+// GetAllBackups().
+func NewBackupServiceWithCachedBackupGetter(ctx context.Context, delegate BackupService, resyncPeriod time.Duration) BackupService {
+	return &cachedBackupService{
+		BackupService: delegate,
+		cache:         NewBackupCache(ctx, delegate, resyncPeriod),
+	}
+}
+
+func (c *cachedBackupService) GetAllBackups(bucketName string) ([]*api.Backup, error) {
+	return c.cache.GetAllBackups(bucketName)
+}

pkg/cloudprovider/backup_service_test.go

@@ -0,0 +1,407 @@
+/*
+Copyright 2017 Heptio Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cloudprovider
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"io"
+	"io/ioutil"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/sets"
+
+	api "github.com/heptio/ark/pkg/apis/ark/v1"
+	"github.com/heptio/ark/pkg/util/encode"
+)
+
+func TestUploadBackup(t *testing.T) {
+	tests := []struct {
+		name            string
+		bucket          string
+		bucketExists    bool
+		backupName      string
+		metadata        io.ReadSeeker
+		backup          io.ReadSeeker
+		objectStoreErrs map[string]map[string]interface{}
+		expectedErr     bool
+		expectedRes     map[string][]byte
+	}{
+		{
+			name:         "normal case",
+			bucket:       "test-bucket",
+			bucketExists: true,
+			backupName:   "test-backup",
+			metadata:     newStringReadSeeker("foo"),
+			backup:       newStringReadSeeker("bar"),
+			expectedErr:  false,
+			expectedRes: map[string][]byte{
+				"test-backup/ark-backup.json":    []byte("foo"),
+				"test-backup/test-backup.tar.gz": []byte("bar"),
+			},
+		},
+		{
+			name:         "no such bucket causes error",
+			bucket:       "test-bucket",
+			bucketExists: false,
+			backupName:   "test-backup",
+			expectedErr:  true,
+		},
+		{
+			name:         "error on metadata upload does not upload data",
+			bucket:       "test-bucket",
+			bucketExists: true,
+			backupName:   "test-backup",
+			metadata:     newStringReadSeeker("foo"),
+			backup:       newStringReadSeeker("bar"),
+			objectStoreErrs: map[string]map[string]interface{}{
+				"putobject": map[string]interface{}{
+					"test-bucket||test-backup/ark-backup.json": true,
+				},
+			},
+			expectedErr: true,
+			expectedRes: make(map[string][]byte),
+		},
+		{
+			name:         "error on data upload deletes metadata",
+			bucket:       "test-bucket",
+			bucketExists: true,
+			backupName:   "test-backup",
+			metadata:     newStringReadSeeker("foo"),
+			backup:       newStringReadSeeker("bar"),
+			objectStoreErrs: map[string]map[string]interface{}{
+				"putobject": map[string]interface{}{
+					"test-bucket||test-backup/test-backup.tar.gz": true,
+				},
+			},
+			expectedErr: true,
+			expectedRes: make(map[string][]byte),
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			objStore := &fakeObjectStorage{
+				returnErrors: test.objectStoreErrs,
+				storage:      make(map[string]map[string][]byte),
+			}
+			if test.bucketExists {
+				objStore.storage[test.bucket] = make(map[string][]byte)
+			}
+
+			backupService := NewBackupService(objStore)
+
+			err := backupService.UploadBackup(test.bucket, test.backupName, test.metadata, test.backup)
+
+			assert.Equal(t, test.expectedErr, err != nil, "got error %v", err)
+
+			assert.Equal(t, test.expectedRes, objStore.storage[test.bucket])
+
+		})
+	}
+}
+
+func TestDownloadBackup(t *testing.T) {
+	tests := []struct {
+		name        string
+		bucket      string
+		backupName  string
+		storage     map[string]map[string][]byte
+		expectedErr bool
+		expectedRes []byte
+	}{
+		{
+			name:       "normal case",
+			bucket:     "test-bucket",
+			backupName: "test-backup",
+			storage: map[string]map[string][]byte{
+				"test-bucket": map[string][]byte{
+					"test-backup/test-backup.tar.gz": []byte("foo"),
+				},
+			},
+			expectedErr: false,
+			expectedRes: []byte("foo"),
+		},
+		{
+			name:        "no such bucket causes error",
+			bucket:      "test-bucket",
+			backupName:  "test-backup",
+			storage:     map[string]map[string][]byte{},
+			expectedErr: true,
+		},
+		{
+			name:       "no such key causes error",
+			bucket:     "test-bucket",
+			backupName: "test-backup",
+			storage: map[string]map[string][]byte{
+				"test-bucket": map[string][]byte{},
+			},
+			expectedErr: true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			objStore := &fakeObjectStorage{storage: test.storage}
+			backupService := NewBackupService(objStore)
+
+			rdr, err := backupService.DownloadBackup(test.bucket, test.backupName)
+
+			assert.Equal(t, test.expectedErr, err != nil, "got error %v", err)
+
+			if err == nil {
+				res, err := ioutil.ReadAll(rdr)
+				assert.Nil(t, err)
+				assert.Equal(t, test.expectedRes, res)
+			}
+		})
+	}
+}
+
+func TestDeleteBackup(t *testing.T) {
+
+	tests := []struct {
+		name        string
+		bucket      string
+		backupName  string
+		storage     map[string]map[string][]byte
+		expectedErr bool
+		expectedRes map[string][]byte
+	}{
+		{
+			name:       "normal case",
+			bucket:     "test-bucket",
+			backupName: "bak",
+			storage: map[string]map[string][]byte{
+				"test-bucket": map[string][]byte{
+					"bak/bak.tar.gz":      nil,
+					"bak/ark-backup.json": nil,
+				},
+			},
+			expectedErr: false,
+			expectedRes: make(map[string][]byte),
+		},
+		{
+			name:       "failed delete of backup doesn't prevent metadata delete but returns error",
+			bucket:     "test-bucket",
+			backupName: "bak",
+			storage: map[string]map[string][]byte{
+				"test-bucket": map[string][]byte{
+					"bak/ark-backup.json": nil,
+				},
+			},
+			expectedErr: true,
+			expectedRes: make(map[string][]byte),
+		},
+		{
+			name:       "failed delete of metadata returns error",
+			bucket:     "test-bucket",
+			backupName: "bak",
+			storage: map[string]map[string][]byte{
+				"test-bucket": map[string][]byte{
+					"bak/bak.tar.gz": nil,
+				},
+			},
+			expectedErr: true,
+			expectedRes: make(map[string][]byte),
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			objStore := &fakeObjectStorage{storage: test.storage}
+			backupService := NewBackupService(objStore)
+
+			res := backupService.DeleteBackup(test.bucket, test.backupName)
+
+			assert.Equal(t, test.expectedErr, res != nil, "got error %v", res)
+
+			assert.Equal(t, test.expectedRes, objStore.storage[test.bucket])
+		})
+	}
+}
+
+func TestGetAllBackups(t *testing.T) {
+	tests := []struct {
+		name        string
+		bucket      string
+		storage     map[string]map[string][]byte
+		expectedRes []*api.Backup
+		expectedErr bool
+	}{
+		{
+			name:   "normal case",
+			bucket: "test-bucket",
+			storage: map[string]map[string][]byte{
+				"test-bucket": map[string][]byte{
+					"backup-1/ark-backup.json": encodeToBytes(&api.Backup{ObjectMeta: metav1.ObjectMeta{Name: "backup-1"}}),
+					"backup-2/ark-backup.json": encodeToBytes(&api.Backup{ObjectMeta: metav1.ObjectMeta{Name: "backup-2"}}),
+				},
+			},
+			expectedErr: false,
+			expectedRes: []*api.Backup{
+				&api.Backup{
+					TypeMeta:   metav1.TypeMeta{Kind: "Backup", APIVersion: "ark.heptio.com/v1"},
+					ObjectMeta: metav1.ObjectMeta{Name: "backup-1"},
+				},
+				&api.Backup{
+					TypeMeta:   metav1.TypeMeta{Kind: "Backup", APIVersion: "ark.heptio.com/v1"},
+					ObjectMeta: metav1.ObjectMeta{Name: "backup-2"},
+				},
+			},
+		},
+		{
+			name:   "decode error returns nil/error",
+			bucket: "test-bucket",
+			storage: map[string]map[string][]byte{
+				"test-bucket": map[string][]byte{
+					"backup-1/ark-backup.json": encodeToBytes(&api.Backup{ObjectMeta: metav1.ObjectMeta{Name: "backup-1"}}),
+					"backup-2/ark-backup.json": []byte("this is not valid backup JSON"),
+				},
+			},
+			expectedErr: true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			objStore := &fakeObjectStorage{storage: test.storage}
+			backupService := NewBackupService(objStore)
+
+			res, err := backupService.GetAllBackups(test.bucket)
+
+			assert.Equal(t, test.expectedErr, err != nil, "got error %v", err)
+
+			assert.Equal(t, test.expectedRes, res)
+		})
+	}
+}
+
+func jsonMarshal(obj interface{}) []byte {
+	res, err := json.Marshal(obj)
+	if err != nil {
+		panic(err)
+	}
+	return res
+}
+
+func encodeToBytes(obj runtime.Object) []byte {
+	res, err := encode.Encode(obj, "json")
+	if err != nil {
+		panic(err)
+	}
+	return res
+}
+
+type stringReadSeeker struct {
+	*strings.Reader
+}
+
+func newStringReadSeeker(s string) *stringReadSeeker {
+	return &stringReadSeeker{
+		Reader: strings.NewReader(s),
+	}
+}
+
+func (srs *stringReadSeeker) Seek(offset int64, whence int) (int64, error) {
+	panic("not implemented")
+}
+
+type fakeObjectStorage struct {
+	storage      map[string]map[string][]byte
+	returnErrors map[string]map[string]interface{}
+}
+
+func (os *fakeObjectStorage) PutObject(bucket string, key string, body io.ReadSeeker) error {
+	if os.returnErrors["putobject"] != nil && os.returnErrors["putobject"][bucket+"||"+key] != nil {
+		return errors.New("error")
+	}
+
+	if os.storage[bucket] == nil {
+		return errors.New("bucket not found")
+	}
+
+	data, err := ioutil.ReadAll(body)
+	if err != nil {
+		return err
+	}
+
+	os.storage[bucket][key] = data
+
+	return nil
+}
+
+func (os *fakeObjectStorage) GetObject(bucket string, key string) (io.ReadCloser, error) {
+	if os.storage == nil {
+		return nil, errors.New("storage not initialized")
+	}
+	if os.storage[bucket] == nil {
+		return nil, errors.New("bucket not found")
+	}
+
+	if os.storage[bucket][key] == nil {
+		return nil, errors.New("key not found")
+	}
+
+	return ioutil.NopCloser(bytes.NewReader(os.storage[bucket][key])), nil
+}
+
+func (os *fakeObjectStorage) ListCommonPrefixes(bucket string, delimiter string) ([]string, error) {
+	if os.storage == nil {
+		return nil, errors.New("storage not initialized")
+	}
+	if os.storage[bucket] == nil {
+		return nil, errors.New("bucket not found")
+	}
+
+	prefixes := sets.NewString()
+
+	for key := range os.storage[bucket] {
+		delimIdx := strings.LastIndex(key, delimiter)
+
+		if delimIdx == -1 {
+			prefixes.Insert(key)
+		}
+
+		prefixes.Insert(key[0:delimIdx])
+	}
+
+	return prefixes.List(), nil
+}
+
+func (os *fakeObjectStorage) DeleteObject(bucket string, key string) error {
+	if os.storage == nil {
+		return errors.New("storage not initialized")
+	}
+	if os.storage[bucket] == nil {
+		return errors.New("bucket not found")
+	}
+
+	if _, exists := os.storage[bucket][key]; !exists {
+		return errors.New("key not found")
+	}
+
+	delete(os.storage[bucket], key)
+
+	return nil
+}

pkg/cloudprovider/gcp/block_storage_adapter.go

@@ -0,0 +1,154 @@
+/*
+Copyright 2017 Heptio Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gcp
+
+import (
+	"strings"
+	"time"
+
+	uuid "github.com/satori/go.uuid"
+	"google.golang.org/api/compute/v0.beta"
+
+	"k8s.io/apimachinery/pkg/util/wait"
+
+	"github.com/heptio/ark/pkg/cloudprovider"
+)
+
+type blockStorageAdapter struct {
+	gce     *compute.Service
+	project string
+	zone    string
+}
+
+var _ cloudprovider.BlockStorageAdapter = &blockStorageAdapter{}
+
+func (op *blockStorageAdapter) CreateVolumeFromSnapshot(snapshotID string, volumeType string, iops *int) (volumeID string, err error) {
+	res, err := op.gce.Snapshots.Get(op.project, snapshotID).Do()
+	if err != nil {
+		return "", err
+	}
+
+	disk := &compute.Disk{
+		Name:           "restore-" + uuid.NewV4().String(),
+		SourceSnapshot: res.SelfLink,
+		Type:           volumeType,
+	}
+
+	if _, err = op.gce.Disks.Insert(op.project, op.zone, disk).Do(); err != nil {
+		return "", err
+	}
+
+	return disk.Name, nil
+}
+
+func (op *blockStorageAdapter) GetVolumeInfo(volumeID string) (string, *int, error) {
+	res, err := op.gce.Disks.Get(op.project, op.zone, volumeID).Do()
+	if err != nil {
+		return "", nil, err
+	}
+
+	return res.Type, nil, nil
+}
+
+func (op *blockStorageAdapter) IsVolumeReady(volumeID string) (ready bool, err error) {
+	disk, err := op.gce.Disks.Get(op.project, op.zone, volumeID).Do()
+	if err != nil {
+		return false, err
+	}
+
+	// TODO can we consider a disk ready while it's in the RESTORING state?
+	return disk.Status == "READY", nil
+}
+
+func (op *blockStorageAdapter) ListSnapshots(tagFilters map[string]string) ([]string, error) {
+	useParentheses := len(tagFilters) > 1
+	subFilters := make([]string, 0, len(tagFilters))
+
+	for k, v := range tagFilters {
+		fs := k + " eq " + v
+		if useParentheses {
+			fs = "(" + fs + ")"
+		}
+		subFilters = append(subFilters, fs)
+	}
+
+	filter := strings.Join(subFilters, " ")
+
+	res, err := op.gce.Snapshots.List(op.project).Filter(filter).Do()
+	if err != nil {
+		return nil, err
+	}
+
+	ret := make([]string, 0, len(res.Items))
+	for _, snap := range res.Items {
+		ret = append(ret, snap.Name)
+	}
+
+	return ret, nil
+}
+
+func (op *blockStorageAdapter) CreateSnapshot(volumeID string, tags map[string]string) (string, error) {
+	// snapshot names must adhere to RFC1035 and be 1-63 characters
+	// long
+	var snapshotName string
+	suffix := "-" + uuid.NewV4().String()
+
+	if len(volumeID) <= (63 - len(suffix)) {
+		snapshotName = volumeID + suffix
+	} else {
+		snapshotName = volumeID[0:63-len(suffix)] + suffix
+	}
+
+	gceSnap := compute.Snapshot{
+		Name: snapshotName,
+	}
+
+	_, err := op.gce.Disks.CreateSnapshot(op.project, op.zone, volumeID, &gceSnap).Do()
+	if err != nil {
+		return "", err
+	}
+
+	// the snapshot is not immediately available after creation for putting labels
+	// on it. poll for a period of time.
+	if pollErr := wait.Poll(1*time.Second, 30*time.Second, func() (bool, error) {
+		if res, err := op.gce.Snapshots.Get(op.project, gceSnap.Name).Do(); err == nil {
+			gceSnap = *res
+			return true, nil
+		}
+		return false, nil
+	}); pollErr != nil {
+		return "", err
+	}
+
+	labels := &compute.GlobalSetLabelsRequest{
+		Labels:           tags,
+		LabelFingerprint: gceSnap.LabelFingerprint,
+	}
+
+	_, err = op.gce.Snapshots.SetLabels(op.project, gceSnap.Name, labels).Do()
+	if err != nil {
+		return "", err
+	}
+
+	return gceSnap.Name, nil
+}
+
+func (op *blockStorageAdapter) DeleteSnapshot(snapshotID string) error {
+	_, err := op.gce.Snapshots.Delete(op.project, snapshotID).Do()
+
+	return err
+}

pkg/cloudprovider/gcp/object_storage_adapter.go

@@ -0,0 +1,73 @@
+/*
+Copyright 2017 Heptio Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gcp
+
+import (
+	"io"
+	"strings"
+
+	storage "google.golang.org/api/storage/v1"
+
+	"github.com/heptio/ark/pkg/cloudprovider"
+)
+
+type objectStorageAdapter struct {
+	project string
+	zone    string
+	gcs     *storage.Service
+}
+
+var _ cloudprovider.ObjectStorageAdapter = &objectStorageAdapter{}
+
+func (op *objectStorageAdapter) PutObject(bucket string, key string, body io.ReadSeeker) error {
+	obj := &storage.Object{
+		Name: key,
+	}
+
+	_, err := op.gcs.Objects.Insert(bucket, obj).Media(body).Do()
+
+	return err
+}
+
+func (op *objectStorageAdapter) GetObject(bucket string, key string) (io.ReadCloser, error) {
+	res, err := op.gcs.Objects.Get(bucket, key).Download()
+	if err != nil {
+		return nil, err
+	}
+
+	return res.Body, nil
+}
+
+func (op *objectStorageAdapter) ListCommonPrefixes(bucket string, delimiter string) ([]string, error) {
+	res, err := op.gcs.Objects.List(bucket).Delimiter(delimiter).Do()
+	if err != nil {
+		return nil, err
+	}
+
+	// GCP returns prefixes inclusive of the last delimiter. We need to strip
+	// it.
+	ret := make([]string, 0, len(res.Prefixes))
+	for _, prefix := range res.Prefixes {
+		ret = append(ret, prefix[0:strings.LastIndex(prefix, delimiter)])
+	}
+
+	return ret, nil
+}
+
+func (op *objectStorageAdapter) DeleteObject(bucket string, key string) error {
+	return op.gcs.Objects.Delete(bucket, key).Do()
+}

pkg/cloudprovider/gcp/storage_adapter.go

@@ -0,0 +1,72 @@
+/*
+Copyright 2017 Heptio Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gcp
+
+import (
+	"golang.org/x/oauth2"
+	"golang.org/x/oauth2/google"
+	"google.golang.org/api/compute/v0.beta"
+	"google.golang.org/api/storage/v1"
+
+	"github.com/heptio/ark/pkg/cloudprovider"
+)
+
+type storageAdapter struct {
+	blockStorage  *blockStorageAdapter
+	objectStorage *objectStorageAdapter
+}
+
+var _ cloudprovider.StorageAdapter = &storageAdapter{}
+
+func NewStorageAdapter(project string, zone string) (cloudprovider.StorageAdapter, error) {
+	client, err := google.DefaultClient(oauth2.NoContext, compute.ComputeScope, storage.DevstorageReadWriteScope)
+
+	if err != nil {
+		return nil, err
+	}
+
+	gce, err := compute.New(client)
+	if err != nil {
+		return nil, err
+	}
+
+	gcs, err := storage.New(client)
+	if err != nil {
+		return nil, err
+	}
+
+	return &storageAdapter{
+		objectStorage: &objectStorageAdapter{
+			gcs:     gcs,
+			project: project,
+			zone:    zone,
+		},
+		blockStorage: &blockStorageAdapter{
+			gce:     gce,
+			project: project,
+			zone:    zone,
+		},
+	}, nil
+}
+
+func (op *storageAdapter) ObjectStorage() cloudprovider.ObjectStorageAdapter {
+	return op.objectStorage
+}
+
+func (op *storageAdapter) BlockStorage() cloudprovider.BlockStorageAdapter {
+	return op.blockStorage
+}

pkg/cloudprovider/snapshot_service.go

@@ -0,0 +1,121 @@
+/*
+Copyright 2017 Heptio Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cloudprovider
+
+import (
+	"fmt"
+	"time"
+)
+
+// SnapshotService exposes Ark-specific operations for snapshotting and restoring block
+// volumes.
+type SnapshotService interface {
+	// GetAllSnapshots returns a slice of all snapshots found in the cloud API that
+	// are tagged with Ark metadata. Returns an error if a problem is encountered accessing
+	// the cloud API.
+	GetAllSnapshots() ([]string, error)
+
+	// CreateSnapshot triggers a snapshot for the specified cloud volume and tags it with metadata.
+	// it returns the cloud snapshot ID, or an error if a problem is encountered triggering the snapshot via
+	// the cloud API.
+	CreateSnapshot(volumeID string) (string, error)
+
+	// CreateVolumeFromSnapshot triggers a restore operation to create a new cloud volume from the specified
+	// snapshot and volume characteristics. Returns the cloud volume ID, or an error if a problem is
+	// encountered triggering the restore via the cloud API.
+	CreateVolumeFromSnapshot(snapshotID, volumeType string, iops *int) (string, error)
+
+	// DeleteSnapshot triggers a deletion of the specified Ark snapshot via the cloud API. It returns an
+	// error if a problem is encountered triggering the deletion via the cloud API.
+	DeleteSnapshot(snapshotID string) error
+
+	// GetVolumeInfo gets the type and IOPS (if applicable) from the cloud API.
+	GetVolumeInfo(volumeID string) (string, *int, error)
+}
+
+const (
+	volumeCreateWaitTimeout  = 30 * time.Second
+	volumeCreatePollInterval = 1 * time.Second
+	snapshotTagKey           = "tag-key"
+	snapshotTagVal           = "ark-snapshot"
+)
+
+type snapshotService struct {
+	blockStorage BlockStorageAdapter
+}
+
+var _ SnapshotService = &snapshotService{}
+
+// NewSnapshotService creates a snapshot service using the provided block storage adapter
+func NewSnapshotService(blockStorage BlockStorageAdapter) SnapshotService {
+	return &snapshotService{
+		blockStorage: blockStorage,
+	}
+}
+
+func (sr *snapshotService) CreateVolumeFromSnapshot(snapshotID string, volumeType string, iops *int) (string, error) {
+	volumeID, err := sr.blockStorage.CreateVolumeFromSnapshot(snapshotID, volumeType, iops)
+	if err != nil {
+		return "", err
+	}
+
+	// wait for volume to be ready (up to a maximum time limit)
+	ticker := time.NewTicker(volumeCreatePollInterval)
+	defer ticker.Stop()
+
+	timeout := time.NewTimer(volumeCreateWaitTimeout)
+
+	for {
+		select {
+		case <-timeout.C:
+			return "", fmt.Errorf("timeout reached waiting for volume %v to be ready", volumeID)
+		case <-ticker.C:
+			if ready, err := sr.blockStorage.IsVolumeReady(volumeID); err == nil && ready {
+				return volumeID, nil
+			}
+		}
+	}
+}
+
+func (sr *snapshotService) GetAllSnapshots() ([]string, error) {
+	tags := map[string]string{
+		snapshotTagKey: snapshotTagVal,
+	}
+
+	res, err := sr.blockStorage.ListSnapshots(tags)
+	if err != nil {
+		return nil, err
+	}
+
+	return res, nil
+}
+
+func (sr *snapshotService) CreateSnapshot(volumeID string) (string, error) {
+	tags := map[string]string{
+		snapshotTagKey: snapshotTagVal,
+	}
+
+	return sr.blockStorage.CreateSnapshot(volumeID, tags)
+}
+
+func (sr *snapshotService) DeleteSnapshot(snapshotID string) error {
+	return sr.blockStorage.DeleteSnapshot(snapshotID)
+}
+
+func (sr *snapshotService) GetVolumeInfo(volumeID string) (string, *int, error) {
+	return sr.blockStorage.GetVolumeInfo(volumeID)
+}

pkg/cloudprovider/storage_interfaces.go

@@ -0,0 +1,72 @@
+/*
+Copyright 2017 Heptio Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cloudprovider
+
+import "io"
+
+// ObjectStorageAdapter exposes basic object-storage operations required
+// by Ark.
+type ObjectStorageAdapter interface {
+	// PutObject creates a new object using the data in body within the specified
+	// object storage bucket with the given key.
+	PutObject(bucket string, key string, body io.ReadSeeker) error
+
+	// GetObject retrieves the object with the given key from the specified
+	// bucket in object storage.
+	GetObject(bucket string, key string) (io.ReadCloser, error)
+
+	// ListCommonPrefixes gets a list of all object key prefixes that come
+	// before the provided delimiter (this is often used to simulate a directory
+	// hierarchy in object storage).
+	ListCommonPrefixes(bucket string, delimiter string) ([]string, error)
+
+	// DeleteObject removes object with the specified key from the given
+	// bucket.
+	DeleteObject(bucket string, key string) error
+}
+
+// BlockStorageAdapter exposes basic block-storage operations required
+// by Ark.
+type BlockStorageAdapter interface {
+	// CreateVolumeFromSnapshot creates a new block volume, initialized from the provided snapshot,
+	// and with the specified type and IOPS (if using provisioned IOPS).
+	CreateVolumeFromSnapshot(snapshotID, volumeType string, iops *int) (volumeID string, err error)
+
+	// GetVolumeInfo returns the type and IOPS (if using provisioned IOPS) for a specified block
+	// volume.
+	GetVolumeInfo(volumeID string) (string, *int, error)
+
+	// IsVolumeReady returns whether the specified volume is ready to be used.
+	IsVolumeReady(volumeID string) (ready bool, err error)
+
+	// ListSnapshots returns a list of all snapshots matching the specified set of tag key/values.
+	ListSnapshots(tagFilters map[string]string) ([]string, error)
+
+	// CreateSnapshot creates a snapshot of the specified block volume, and applies the provided
+	// set of tags to the snapshot.
+	CreateSnapshot(volumeID string, tags map[string]string) (snapshotID string, err error)
+
+	// DeleteSnapshot deletes the specified volume snapshot.
+	DeleteSnapshot(snapshotID string) error
+}
+
+// StorageAdapter exposes object- and block-storage interfaces and associated methods
+// for a given storage provider.
+type StorageAdapter interface {
+	ObjectStorage() ObjectStorageAdapter
+	BlockStorage() BlockStorageAdapter
+}