add a BackupStore to pkg/persistence that supports prefixes

Signed-off-by: Steve Kriss <steve@heptio.com>

pkg/cloudprovider/aws/object_store.go

@@ -116,7 +116,7 @@ func (o *objectStore) Init(config map[string]string) error {
 	return nil
 }
 
-func (o *objectStore) PutObject(bucket string, key string, body io.Reader) error {
+func (o *objectStore) PutObject(bucket, key string, body io.Reader) error {
 	req := &s3manager.UploadInput{
 		Bucket: &bucket,
 		Key:    &key,
@@ -134,7 +134,7 @@ func (o *objectStore) PutObject(bucket string, key string, body io.Reader) error
 	return errors.Wrapf(err, "error putting object %s", key)
 }
 
-func (o *objectStore) GetObject(bucket string, key string) (io.ReadCloser, error) {
+func (o *objectStore) GetObject(bucket, key string) (io.ReadCloser, error) {
 	req := &s3.GetObjectInput{
 		Bucket: &bucket,
 		Key:    &key,
@@ -148,9 +148,10 @@ func (o *objectStore) GetObject(bucket string, key string) (io.ReadCloser, error
 	return res.Body, nil
 }
 
-func (o *objectStore) ListCommonPrefixes(bucket string, delimiter string) ([]string, error) {
+func (o *objectStore) ListCommonPrefixes(bucket, prefix, delimiter string) ([]string, error) {
 	req := &s3.ListObjectsV2Input{
 		Bucket:    &bucket,
+		Prefix:    &prefix,
 		Delimiter: &delimiter,
 	}
 
@@ -161,7 +162,6 @@ func (o *objectStore) ListCommonPrefixes(bucket string, delimiter string) ([]str
 		}
 		return !lastPage
 	})
-
 	if err != nil {
 		return nil, errors.WithStack(err)
 	}
@@ -190,7 +190,7 @@ func (o *objectStore) ListObjects(bucket, prefix string) ([]string, error) {
 	return ret, nil
 }
 
-func (o *objectStore) DeleteObject(bucket string, key string) error {
+func (o *objectStore) DeleteObject(bucket, key string) error {
 	req := &s3.DeleteObjectInput{
 		Bucket: &bucket,
 		Key:    &key,

pkg/cloudprovider/azure/object_store.go

@@ -119,7 +119,7 @@ func (o *objectStore) Init(config map[string]string) error {
 	return nil
 }
 
-func (o *objectStore) PutObject(bucket string, key string, body io.Reader) error {
+func (o *objectStore) PutObject(bucket, key string, body io.Reader) error {
 	container, err := getContainerReference(o.blobClient, bucket)
 	if err != nil {
 		return err
@@ -133,7 +133,7 @@ func (o *objectStore) PutObject(bucket string, key string, body io.Reader) error
 	return errors.WithStack(blob.CreateBlockBlobFromReader(body, nil))
 }
 
-func (o *objectStore) GetObject(bucket string, key string) (io.ReadCloser, error) {
+func (o *objectStore) GetObject(bucket, key string) (io.ReadCloser, error) {
 	container, err := getContainerReference(o.blobClient, bucket)
 	if err != nil {
 		return nil, err
@@ -152,13 +152,14 @@ func (o *objectStore) GetObject(bucket string, key string) (io.ReadCloser, error
 	return res, nil
 }
 
-func (o *objectStore) ListCommonPrefixes(bucket string, delimiter string) ([]string, error) {
+func (o *objectStore) ListCommonPrefixes(bucket, prefix, delimiter string) ([]string, error) {
 	container, err := getContainerReference(o.blobClient, bucket)
 	if err != nil {
 		return nil, err
 	}
 
 	params := storage.ListBlobsParameters{
+		Prefix:    prefix,
 		Delimiter: delimiter,
 	}
 
@@ -167,14 +168,7 @@ func (o *objectStore) ListCommonPrefixes(bucket string, delimiter string) ([]str
 		return nil, errors.WithStack(err)
 	}
 
-	// Azure returns prefixes inclusive of the last delimiter. We need to strip
-	// it.
-	ret := make([]string, 0, len(res.BlobPrefixes))
-	for _, prefix := range res.BlobPrefixes {
-		ret = append(ret, prefix[0:strings.LastIndex(prefix, delimiter)])
-	}
-
-	return ret, nil
+	return res.BlobPrefixes, nil
 }
 
 func (o *objectStore) ListObjects(bucket, prefix string) ([]string, error) {

pkg/cloudprovider/gcp/object_store.go

@@ -21,7 +21,6 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
-	"strings"
 	"time"
 
 	"cloud.google.com/go/storage"
@@ -98,7 +97,7 @@ func (o *objectStore) Init(config map[string]string) error {
 	return nil
 }
 
-func (o *objectStore) PutObject(bucket string, key string, body io.Reader) error {
+func (o *objectStore) PutObject(bucket, key string, body io.Reader) error {
 	w := o.bucketWriter.getWriteCloser(bucket, key)
 
 	// The writer returned by NewWriter is asynchronous, so errors aren't guaranteed
@@ -114,7 +113,7 @@ func (o *objectStore) PutObject(bucket string, key string, body io.Reader) error
 	return closeErr
 }
 
-func (o *objectStore) GetObject(bucket string, key string) (io.ReadCloser, error) {
+func (o *objectStore) GetObject(bucket, key string) (io.ReadCloser, error) {
 	r, err := o.client.Bucket(bucket).Object(key).NewReader(context.Background())
 	if err != nil {
 		return nil, errors.WithStack(err)
@@ -123,28 +122,30 @@ func (o *objectStore) GetObject(bucket string, key string) (io.ReadCloser, error
 	return r, nil
 }
 
-func (o *objectStore) ListCommonPrefixes(bucket string, delimiter string) ([]string, error) {
+func (o *objectStore) ListCommonPrefixes(bucket, prefix, delimiter string) ([]string, error) {
 	q := &storage.Query{
+		Prefix:    prefix,
 		Delimiter: delimiter,
 	}
 
-	var res []string
-
 	iter := o.client.Bucket(bucket).Objects(context.Background(), q)
 
+	var res []string
 	for {
 		obj, err := iter.Next()
-		if err == iterator.Done {
-			return res, nil
-		}
-		if err != nil {
+		if err != nil && err != iterator.Done {
 			return nil, errors.WithStack(err)
 		}
+		if err == iterator.Done {
+			break
+		}
 
 		if obj.Prefix != "" {
-			res = append(res, obj.Prefix[0:strings.LastIndex(obj.Prefix, delimiter)])
+			res = append(res, obj.Prefix)
 		}
 	}
+
+	return res, nil
 }
 
 func (o *objectStore) ListObjects(bucket, prefix string) ([]string, error) {
@@ -169,7 +170,7 @@ func (o *objectStore) ListObjects(bucket, prefix string) ([]string, error) {
 	}
 }
 
-func (o *objectStore) DeleteObject(bucket string, key string) error {
+func (o *objectStore) DeleteObject(bucket, key string) error {
 	return errors.Wrapf(o.client.Bucket(bucket).Object(key).Delete(context.Background()), "error deleting object %s", key)
 }
 

pkg/cloudprovider/in_memory_object_store.go

@@ -0,0 +1,168 @@
+/*
+Copyright 2018 the Heptio Ark contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cloudprovider
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"io/ioutil"
+	"strings"
+	"time"
+)
+
+type BucketData map[string][]byte
+
+// InMemoryObjectStore is a simple implementation of the ObjectStore interface
+// that stores its data in-memory/in-proc. This is mainly intended to be used
+// as a test fake.
+type InMemoryObjectStore struct {
+	Data map[string]BucketData
+}
+
+func NewInMemoryObjectStore(buckets ...string) *InMemoryObjectStore {
+	o := &InMemoryObjectStore{
+		Data: make(map[string]BucketData),
+	}
+
+	for _, bucket := range buckets {
+		o.Data[bucket] = make(map[string][]byte)
+	}
+
+	return o
+}
+
+//
+// Interface Implementation
+//
+
+func (o *InMemoryObjectStore) Init(config map[string]string) error {
+	return nil
+}
+
+func (o *InMemoryObjectStore) PutObject(bucket, key string, body io.Reader) error {
+	bucketData, ok := o.Data[bucket]
+	if !ok {
+		return errors.New("bucket not found")
+	}
+
+	obj, err := ioutil.ReadAll(body)
+	if err != nil {
+		return err
+	}
+
+	bucketData[key] = obj
+
+	return nil
+}
+
+func (o *InMemoryObjectStore) GetObject(bucket, key string) (io.ReadCloser, error) {
+	bucketData, ok := o.Data[bucket]
+	if !ok {
+		return nil, errors.New("bucket not found")
+	}
+
+	obj, ok := bucketData[key]
+	if !ok {
+		return nil, errors.New("key not found")
+	}
+
+	return ioutil.NopCloser(bytes.NewReader(obj)), nil
+}
+
+func (o *InMemoryObjectStore) ListCommonPrefixes(bucket, prefix, delimiter string) ([]string, error) {
+	keys, err := o.ListObjects(bucket, prefix)
+	if err != nil {
+		return nil, err
+	}
+
+	// For each key, check if it has an instance of the delimiter *after* the prefix.
+	// If not, skip it; if so, return the prefix of the key up to/including the delimiter.
+
+	var prefixes []string
+	for _, key := range keys {
+		// everything after 'prefix'
+		afterPrefix := key[len(prefix):]
+
+		// index of the *start* of 'delimiter' in 'afterPrefix'
+		delimiterStart := strings.Index(afterPrefix, delimiter)
+		if delimiterStart == -1 {
+			continue
+		}
+
+		// return the prefix, plus everything after the prefix and before
+		// the delimiter, plus the delimiter
+		fullPrefix := prefix + afterPrefix[0:delimiterStart] + delimiter
+
+		prefixes = append(prefixes, fullPrefix)
+	}
+
+	return prefixes, nil
+}
+
+func (o *InMemoryObjectStore) ListObjects(bucket, prefix string) ([]string, error) {
+	bucketData, ok := o.Data[bucket]
+	if !ok {
+		return nil, errors.New("bucket not found")
+	}
+
+	var objs []string
+	for key := range bucketData {
+		if strings.HasPrefix(key, prefix) {
+			objs = append(objs, key)
+		}
+	}
+
+	return objs, nil
+}
+
+func (o *InMemoryObjectStore) DeleteObject(bucket, key string) error {
+	bucketData, ok := o.Data[bucket]
+	if !ok {
+		return errors.New("bucket not found")
+	}
+
+	delete(bucketData, key)
+
+	return nil
+}
+
+func (o *InMemoryObjectStore) CreateSignedURL(bucket, key string, ttl time.Duration) (string, error) {
+	bucketData, ok := o.Data[bucket]
+	if !ok {
+		return "", errors.New("bucket not found")
+	}
+
+	_, ok = bucketData[key]
+	if !ok {
+		return "", errors.New("key not found")
+	}
+
+	return "a-url", nil
+}
+
+//
+// Test Helper Methods
+//
+
+func (o *InMemoryObjectStore) ClearBucket(bucket string) {
+	if _, ok := o.Data[bucket]; !ok {
+		return
+	}
+
+	o.Data[bucket] = make(map[string][]byte)
+}

pkg/cloudprovider/mocks/backup_lister.go

@@ -1,48 +0,0 @@
-/*
-Copyright 2018 the Heptio Ark contributors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-// Code generated by mockery v1.0.0. DO NOT EDIT.
-package mocks
-
-import mock "github.com/stretchr/testify/mock"
-import v1 "github.com/heptio/ark/pkg/apis/ark/v1"
-
-// BackupLister is an autogenerated mock type for the BackupLister type
-type BackupLister struct {
-	mock.Mock
-}
-
-// ListBackups provides a mock function with given fields: bucket
-func (_m *BackupLister) ListBackups(bucket string) ([]*v1.Backup, error) {
-	ret := _m.Called(bucket)
-
-	var r0 []*v1.Backup
-	if rf, ok := ret.Get(0).(func(string) []*v1.Backup); ok {
-		r0 = rf(bucket)
-	} else {
-		if ret.Get(0) != nil {
-			r0 = ret.Get(0).([]*v1.Backup)
-		}
-	}
-
-	var r1 error
-	if rf, ok := ret.Get(1).(func(string) error); ok {
-		r1 = rf(bucket)
-	} else {
-		r1 = ret.Error(1)
-	}
-
-	return r0, r1
-}

pkg/cloudprovider/object_store.go

@@ -31,17 +31,23 @@ type ObjectStore interface {
 
 	// PutObject creates a new object using the data in body within the specified
 	// object storage bucket with the given key.
-	PutObject(bucket string, key string, body io.Reader) error
+	PutObject(bucket, key string, body io.Reader) error
 
 	// GetObject retrieves the object with the given key from the specified
 	// bucket in object storage.
-	GetObject(bucket string, key string) (io.ReadCloser, error)
+	GetObject(bucket, key string) (io.ReadCloser, error)
 
-	// ListCommonPrefixes gets a list of all object key prefixes that come
-	// before the provided delimiter. For example, if the bucket contains
-	// the keys "foo-1/bar", "foo-1/baz", and "foo-2/baz", and the delimiter
-	// is "/", this will return the slice {"foo-1", "foo-2"}.
-	ListCommonPrefixes(bucket string, delimiter string) ([]string, error)
+	// ListCommonPrefixes gets a list of all object key prefixes that start with
+	// the specified prefix and stop at the next instance of the provided delimiter.
+	//
+	// For example, if the bucket contains the following keys:
+	//		a-prefix/foo-1/bar
+	// 		a-prefix/foo-1/baz
+	//		a-prefix/foo-2/baz
+	// 		some-other-prefix/foo-3/bar
+	// and the provided prefix arg is "a-prefix/", and the delimiter is "/",
+	// this will return the slice {"a-prefix/foo-1/", "a-prefix/foo-2/"}.
+	ListCommonPrefixes(bucket, prefix, delimiter string) ([]string, error)
 
 	// ListObjects gets a list of all keys in the specified bucket
 	// that have the given prefix.
@@ -49,7 +55,7 @@ type ObjectStore interface {
 
 	// DeleteObject removes the object with the specified key from the given
 	// bucket.
-	DeleteObject(bucket string, key string) error
+	DeleteObject(bucket, key string) error
 
 	// CreateSignedURL creates a pre-signed URL for the given bucket and key that expires after ttl.
 	CreateSignedURL(bucket, key string, ttl time.Duration) (string, error)