mirror of
https://github.com/vmware-tanzu/velero.git
synced 2026-09-19 06:26:44 +00:00
add a BackupStore to pkg/persistence that supports prefixes
Signed-off-by: Steve Kriss <steve@heptio.com>
This commit is contained in:
Steve Kriss
@@ -0,0 +1,158 @@
|
||||
// Code generated by mockery v1.0.0
|
||||
package mocks
|
||||
|
||||
import io "io"
|
||||
import mock "github.com/stretchr/testify/mock"
|
||||
|
||||
import v1 "github.com/heptio/ark/pkg/apis/ark/v1"
|
||||
|
||||
// BackupStore is an autogenerated mock type for the BackupStore type
|
||||
type BackupStore struct {
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
// DeleteBackup provides a mock function with given fields: name
|
||||
func (_m *BackupStore) DeleteBackup(name string) error {
|
||||
ret := _m.Called(name)
|
||||
|
||||
var r0 error
|
||||
if rf, ok := ret.Get(0).(func(string) error); ok {
|
||||
r0 = rf(name)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// GetBackupContents provides a mock function with given fields: name
|
||||
func (_m *BackupStore) GetBackupContents(name string) (io.ReadCloser, error) {
|
||||
ret := _m.Called(name)
|
||||
|
||||
var r0 io.ReadCloser
|
||||
if rf, ok := ret.Get(0).(func(string) io.ReadCloser); ok {
|
||||
r0 = rf(name)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(io.ReadCloser)
|
||||
}
|
||||
}
|
||||
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(1).(func(string) error); ok {
|
||||
r1 = rf(name)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// GetBackupMetadata provides a mock function with given fields: name
|
||||
func (_m *BackupStore) GetBackupMetadata(name string) (*v1.Backup, error) {
|
||||
ret := _m.Called(name)
|
||||
|
||||
var r0 *v1.Backup
|
||||
if rf, ok := ret.Get(0).(func(string) *v1.Backup); ok {
|
||||
r0 = rf(name)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*v1.Backup)
|
||||
}
|
||||
}
|
||||
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(1).(func(string) error); ok {
|
||||
r1 = rf(name)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// GetDownloadURL provides a mock function with given fields: backup, target
|
||||
func (_m *BackupStore) GetDownloadURL(backup string, target v1.DownloadTarget) (string, error) {
|
||||
ret := _m.Called(backup, target)
|
||||
|
||||
var r0 string
|
||||
if rf, ok := ret.Get(0).(func(string, v1.DownloadTarget) string); ok {
|
||||
r0 = rf(backup, target)
|
||||
} else {
|
||||
r0 = ret.Get(0).(string)
|
||||
}
|
||||
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(1).(func(string, v1.DownloadTarget) error); ok {
|
||||
r1 = rf(backup, target)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// ListBackups provides a mock function with given fields:
|
||||
func (_m *BackupStore) ListBackups() ([]*v1.Backup, error) {
|
||||
ret := _m.Called()
|
||||
|
||||
var r0 []*v1.Backup
|
||||
if rf, ok := ret.Get(0).(func() []*v1.Backup); ok {
|
||||
r0 = rf()
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).([]*v1.Backup)
|
||||
}
|
||||
}
|
||||
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(1).(func() error); ok {
|
||||
r1 = rf()
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// PutBackup provides a mock function with given fields: name, metadata, contents, log
|
||||
func (_m *BackupStore) PutBackup(name string, metadata io.Reader, contents io.Reader, log io.Reader) error {
|
||||
ret := _m.Called(name, metadata, contents, log)
|
||||
|
||||
var r0 error
|
||||
if rf, ok := ret.Get(0).(func(string, io.Reader, io.Reader, io.Reader) error); ok {
|
||||
r0 = rf(name, metadata, contents, log)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// PutRestoreLog provides a mock function with given fields: backup, restore, log
|
||||
func (_m *BackupStore) PutRestoreLog(backup string, restore string, log io.Reader) error {
|
||||
ret := _m.Called(backup, restore, log)
|
||||
|
||||
var r0 error
|
||||
if rf, ok := ret.Get(0).(func(string, string, io.Reader) error); ok {
|
||||
r0 = rf(backup, restore, log)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// PutRestoreResults provides a mock function with given fields: backup, restore, results
|
||||
func (_m *BackupStore) PutRestoreResults(backup string, restore string, results io.Reader) error {
|
||||
ret := _m.Called(backup, restore, results)
|
||||
|
||||
var r0 error
|
||||
if rf, ok := ret.Get(0).(func(string, string, io.Reader) error); ok {
|
||||
r0 = rf(backup, restore, results)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
@@ -1,267 +0,0 @@
|
||||
/*
|
||||
Copyright 2017 the Heptio Ark contributors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package persistence
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
kerrors "k8s.io/apimachinery/pkg/util/errors"
|
||||
|
||||
api "github.com/heptio/ark/pkg/apis/ark/v1"
|
||||
"github.com/heptio/ark/pkg/cloudprovider"
|
||||
"github.com/heptio/ark/pkg/generated/clientset/versioned/scheme"
|
||||
)
|
||||
|
||||
// BackupLister knows how to list backups in object storage.
|
||||
type BackupLister interface {
|
||||
// ListBackups lists all the api.Backups in object storage for the given bucket.
|
||||
ListBackups(bucket string) ([]*api.Backup, error)
|
||||
}
|
||||
|
||||
const (
|
||||
metadataFileFormatString = "%s/ark-backup.json"
|
||||
backupFileFormatString = "%s/%s.tar.gz"
|
||||
backupLogFileFormatString = "%s/%s-logs.gz"
|
||||
restoreLogFileFormatString = "%s/restore-%s-logs.gz"
|
||||
restoreResultsFileFormatString = "%s/restore-%s-results.gz"
|
||||
)
|
||||
|
||||
func getMetadataKey(directory string) string {
|
||||
return fmt.Sprintf(metadataFileFormatString, directory)
|
||||
}
|
||||
|
||||
func getBackupContentsKey(directory, backup string) string {
|
||||
return fmt.Sprintf(backupFileFormatString, directory, backup)
|
||||
}
|
||||
|
||||
func getBackupLogKey(directory, backup string) string {
|
||||
return fmt.Sprintf(backupLogFileFormatString, directory, backup)
|
||||
}
|
||||
|
||||
func getRestoreLogKey(directory, restore string) string {
|
||||
return fmt.Sprintf(restoreLogFileFormatString, directory, restore)
|
||||
}
|
||||
|
||||
func getRestoreResultsKey(directory, restore string) string {
|
||||
return fmt.Sprintf(restoreResultsFileFormatString, directory, restore)
|
||||
}
|
||||
|
||||
func seekToBeginning(r io.Reader) error {
|
||||
seeker, ok := r.(io.Seeker)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
_, err := seeker.Seek(0, 0)
|
||||
return err
|
||||
}
|
||||
|
||||
func seekAndPutObject(objectStore cloudprovider.ObjectStore, bucket, key string, file io.Reader) error {
|
||||
if file == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := seekToBeginning(file); err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
return objectStore.PutObject(bucket, key, file)
|
||||
}
|
||||
|
||||
func UploadBackupLog(objectStore cloudprovider.ObjectStore, bucket, backupName string, log io.Reader) error {
|
||||
logKey := getBackupLogKey(backupName, backupName)
|
||||
return seekAndPutObject(objectStore, bucket, logKey, log)
|
||||
}
|
||||
|
||||
func UploadBackupMetadata(objectStore cloudprovider.ObjectStore, bucket, backupName string, metadata io.Reader) error {
|
||||
metadataKey := getMetadataKey(backupName)
|
||||
return seekAndPutObject(objectStore, bucket, metadataKey, metadata)
|
||||
}
|
||||
|
||||
func DeleteBackupMetadata(objectStore cloudprovider.ObjectStore, bucket, backupName string) error {
|
||||
metadataKey := getMetadataKey(backupName)
|
||||
return objectStore.DeleteObject(bucket, metadataKey)
|
||||
}
|
||||
|
||||
func UploadBackupData(objectStore cloudprovider.ObjectStore, bucket, backupName string, backup io.Reader) error {
|
||||
backupKey := getBackupContentsKey(backupName, backupName)
|
||||
return seekAndPutObject(objectStore, bucket, backupKey, backup)
|
||||
}
|
||||
|
||||
func UploadBackup(logger logrus.FieldLogger, objectStore cloudprovider.ObjectStore, bucket, backupName string, metadata, backup, log io.Reader) error {
|
||||
if err := UploadBackupLog(objectStore, bucket, backupName, log); err != nil {
|
||||
// Uploading the log file is best-effort; if it fails, we log the error but it doesn't impact the
|
||||
// backup's status.
|
||||
logger.WithError(err).WithField("bucket", bucket).Error("Error uploading log file")
|
||||
}
|
||||
|
||||
if metadata == nil {
|
||||
// If we don't have metadata, something failed, and there's no point in continuing. An object
|
||||
// storage bucket that is missing the metadata file can't be restored, nor can its logs be
|
||||
// viewed.
|
||||
return nil
|
||||
}
|
||||
|
||||
// upload metadata file
|
||||
if err := UploadBackupMetadata(objectStore, bucket, backupName, metadata); err != nil {
|
||||
// failure to upload metadata file is a hard-stop
|
||||
return err
|
||||
}
|
||||
|
||||
// upload tar file
|
||||
if err := UploadBackupData(objectStore, bucket, backupName, backup); err != nil {
|
||||
// try to delete the metadata file since the data upload failed
|
||||
deleteErr := DeleteBackupMetadata(objectStore, bucket, backupName)
|
||||
return kerrors.NewAggregate([]error{err, deleteErr})
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// DownloadBackupFunc is a function that can download backup metadata from a bucket in object storage.
|
||||
type DownloadBackupFunc func(objectStore cloudprovider.ObjectStore, bucket, backupName string) (io.ReadCloser, error)
|
||||
|
||||
// DownloadBackup downloads an Ark backup with the specified object key from object storage via the cloud API.
|
||||
// It returns the snapshot metadata and data (separately), or an error if a problem is encountered
|
||||
// downloading or reading the file from the cloud API.
|
||||
func DownloadBackup(objectStore cloudprovider.ObjectStore, bucket, backupName string) (io.ReadCloser, error) {
|
||||
return objectStore.GetObject(bucket, getBackupContentsKey(backupName, backupName))
|
||||
}
|
||||
|
||||
func ListBackups(logger logrus.FieldLogger, objectStore cloudprovider.ObjectStore, bucket string) ([]*api.Backup, error) {
|
||||
prefixes, err := objectStore.ListCommonPrefixes(bucket, "/")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(prefixes) == 0 {
|
||||
return []*api.Backup{}, nil
|
||||
}
|
||||
|
||||
output := make([]*api.Backup, 0, len(prefixes))
|
||||
|
||||
for _, backupDir := range prefixes {
|
||||
backup, err := GetBackup(objectStore, bucket, backupDir)
|
||||
if err != nil {
|
||||
logger.WithError(err).WithField("dir", backupDir).Error("Error reading backup directory")
|
||||
continue
|
||||
}
|
||||
|
||||
output = append(output, backup)
|
||||
}
|
||||
|
||||
return output, nil
|
||||
}
|
||||
|
||||
//GetBackupFunc is a function that can retrieve backup metadata from an object store
|
||||
type GetBackupFunc func(objectStore cloudprovider.ObjectStore, bucket, backupName string) (*api.Backup, error)
|
||||
|
||||
// GetBackup gets the specified api.Backup from the given bucket in object storage.
|
||||
func GetBackup(objectStore cloudprovider.ObjectStore, bucket, backupName string) (*api.Backup, error) {
|
||||
key := getMetadataKey(backupName)
|
||||
|
||||
res, err := objectStore.GetObject(bucket, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer res.Close()
|
||||
|
||||
data, err := ioutil.ReadAll(res)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
decoder := scheme.Codecs.UniversalDecoder(api.SchemeGroupVersion)
|
||||
obj, _, err := decoder.Decode(data, nil, nil)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
backup, ok := obj.(*api.Backup)
|
||||
if !ok {
|
||||
return nil, errors.Errorf("unexpected type for %s/%s: %T", bucket, key, obj)
|
||||
}
|
||||
|
||||
return backup, nil
|
||||
}
|
||||
|
||||
// DeleteBackupDirFunc is a function that can delete a backup directory from a bucket in object storage.
|
||||
type DeleteBackupDirFunc func(logger logrus.FieldLogger, objectStore cloudprovider.ObjectStore, bucket, backupName string) error
|
||||
|
||||
// DeleteBackupDir deletes all files in object storage for the given backup.
|
||||
func DeleteBackupDir(logger logrus.FieldLogger, objectStore cloudprovider.ObjectStore, bucket, backupName string) error {
|
||||
objects, err := objectStore.ListObjects(bucket, backupName+"/")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var errs []error
|
||||
for _, key := range objects {
|
||||
logger.WithFields(logrus.Fields{
|
||||
"bucket": bucket,
|
||||
"key": key,
|
||||
}).Debug("Trying to delete object")
|
||||
if err := objectStore.DeleteObject(bucket, key); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
|
||||
return errors.WithStack(kerrors.NewAggregate(errs))
|
||||
}
|
||||
|
||||
// CreateSignedURLFunc is a function that can create a signed URL for an object in object storage.
|
||||
type CreateSignedURLFunc func(objectStore cloudprovider.ObjectStore, target api.DownloadTarget, bucket, directory string, ttl time.Duration) (string, error)
|
||||
|
||||
// CreateSignedURL creates a pre-signed URL that can be used to download a file from object
|
||||
// storage. The URL expires after ttl.
|
||||
func CreateSignedURL(objectStore cloudprovider.ObjectStore, target api.DownloadTarget, bucket, directory string, ttl time.Duration) (string, error) {
|
||||
switch target.Kind {
|
||||
case api.DownloadTargetKindBackupContents:
|
||||
return objectStore.CreateSignedURL(bucket, getBackupContentsKey(directory, target.Name), ttl)
|
||||
case api.DownloadTargetKindBackupLog:
|
||||
return objectStore.CreateSignedURL(bucket, getBackupLogKey(directory, target.Name), ttl)
|
||||
case api.DownloadTargetKindRestoreLog:
|
||||
return objectStore.CreateSignedURL(bucket, getRestoreLogKey(directory, target.Name), ttl)
|
||||
case api.DownloadTargetKindRestoreResults:
|
||||
return objectStore.CreateSignedURL(bucket, getRestoreResultsKey(directory, target.Name), ttl)
|
||||
default:
|
||||
return "", errors.Errorf("unsupported download target kind %q", target.Kind)
|
||||
}
|
||||
}
|
||||
|
||||
// UploadRestoreLogFunc is a function that can upload a restore log to a bucket in object storage.
|
||||
type UploadRestoreLogFunc func(objectStore cloudprovider.ObjectStore, bucket, backup, restore string, log io.Reader) error
|
||||
|
||||
// UploadRestoreLog uploads the restore's log file to object storage.
|
||||
func UploadRestoreLog(objectStore cloudprovider.ObjectStore, bucket, backup, restore string, log io.Reader) error {
|
||||
key := getRestoreLogKey(backup, restore)
|
||||
return objectStore.PutObject(bucket, key, log)
|
||||
}
|
||||
|
||||
// UploadRestoreResultsFunc is a function that can upload restore results to a bucket in object storage.
|
||||
type UploadRestoreResultsFunc func(objectStore cloudprovider.ObjectStore, bucket, backup, restore string, results io.Reader) error
|
||||
|
||||
// UploadRestoreResults uploads the restore's results file to object storage.
|
||||
func UploadRestoreResults(objectStore cloudprovider.ObjectStore, bucket, backup, restore string, results io.Reader) error {
|
||||
key := getRestoreResultsKey(backup, restore)
|
||||
return objectStore.PutObject(bucket, key, results)
|
||||
}
|
||||
@@ -1,366 +0,0 @@
|
||||
/*
|
||||
Copyright 2017 the Heptio Ark contributors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package persistence
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
testutil "github.com/heptio/ark/pkg/util/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
|
||||
api "github.com/heptio/ark/pkg/apis/ark/v1"
|
||||
"github.com/heptio/ark/pkg/util/encode"
|
||||
arktest "github.com/heptio/ark/pkg/util/test"
|
||||
)
|
||||
|
||||
func TestUploadBackup(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
metadata io.ReadSeeker
|
||||
metadataError error
|
||||
expectMetadataDelete bool
|
||||
backup io.ReadSeeker
|
||||
backupError error
|
||||
expectBackupUpload bool
|
||||
log io.ReadSeeker
|
||||
logError error
|
||||
expectedErr string
|
||||
}{
|
||||
{
|
||||
name: "normal case",
|
||||
metadata: newStringReadSeeker("foo"),
|
||||
backup: newStringReadSeeker("bar"),
|
||||
expectBackupUpload: true,
|
||||
log: newStringReadSeeker("baz"),
|
||||
},
|
||||
{
|
||||
name: "error on metadata upload does not upload data",
|
||||
metadata: newStringReadSeeker("foo"),
|
||||
metadataError: errors.New("md"),
|
||||
log: newStringReadSeeker("baz"),
|
||||
expectedErr: "md",
|
||||
},
|
||||
{
|
||||
name: "error on data upload deletes metadata",
|
||||
metadata: newStringReadSeeker("foo"),
|
||||
backup: newStringReadSeeker("bar"),
|
||||
expectBackupUpload: true,
|
||||
backupError: errors.New("backup"),
|
||||
expectMetadataDelete: true,
|
||||
expectedErr: "backup",
|
||||
},
|
||||
{
|
||||
name: "error on log upload is ok",
|
||||
metadata: newStringReadSeeker("foo"),
|
||||
backup: newStringReadSeeker("bar"),
|
||||
expectBackupUpload: true,
|
||||
log: newStringReadSeeker("baz"),
|
||||
logError: errors.New("log"),
|
||||
},
|
||||
{
|
||||
name: "don't upload data when metadata is nil",
|
||||
backup: newStringReadSeeker("bar"),
|
||||
log: newStringReadSeeker("baz"),
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
var (
|
||||
objectStore = &testutil.ObjectStore{}
|
||||
bucket = "test-bucket"
|
||||
backupName = "test-backup"
|
||||
logger = arktest.NewLogger()
|
||||
)
|
||||
defer objectStore.AssertExpectations(t)
|
||||
|
||||
if test.metadata != nil {
|
||||
objectStore.On("PutObject", bucket, backupName+"/ark-backup.json", test.metadata).Return(test.metadataError)
|
||||
}
|
||||
if test.backup != nil && test.expectBackupUpload {
|
||||
objectStore.On("PutObject", bucket, backupName+"/"+backupName+".tar.gz", test.backup).Return(test.backupError)
|
||||
}
|
||||
if test.log != nil {
|
||||
objectStore.On("PutObject", bucket, backupName+"/"+backupName+"-logs.gz", test.log).Return(test.logError)
|
||||
}
|
||||
if test.expectMetadataDelete {
|
||||
objectStore.On("DeleteObject", bucket, backupName+"/ark-backup.json").Return(nil)
|
||||
}
|
||||
|
||||
err := UploadBackup(logger, objectStore, bucket, backupName, test.metadata, test.backup, test.log)
|
||||
|
||||
if test.expectedErr != "" {
|
||||
assert.EqualError(t, err, test.expectedErr)
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadBackup(t *testing.T) {
|
||||
var (
|
||||
objectStore = &testutil.ObjectStore{}
|
||||
bucket = "b"
|
||||
backup = "bak"
|
||||
)
|
||||
objectStore.On("GetObject", bucket, backup+"/"+backup+".tar.gz").Return(ioutil.NopCloser(strings.NewReader("foo")), nil)
|
||||
|
||||
rc, err := DownloadBackup(objectStore, bucket, backup)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, rc)
|
||||
data, err := ioutil.ReadAll(rc)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "foo", string(data))
|
||||
objectStore.AssertExpectations(t)
|
||||
}
|
||||
|
||||
func TestDeleteBackup(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
listObjectsError error
|
||||
deleteErrors []error
|
||||
expectedErr string
|
||||
}{
|
||||
{
|
||||
name: "normal case",
|
||||
},
|
||||
{
|
||||
name: "some delete errors, do as much as we can",
|
||||
deleteErrors: []error{errors.New("a"), nil, errors.New("c")},
|
||||
expectedErr: "[a, c]",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
var (
|
||||
bucket = "bucket"
|
||||
backup = "bak"
|
||||
objects = []string{"bak/ark-backup.json", "bak/bak.tar.gz", "bak/bak.log.gz"}
|
||||
objectStore = &testutil.ObjectStore{}
|
||||
logger = arktest.NewLogger()
|
||||
)
|
||||
|
||||
objectStore.On("ListObjects", bucket, backup+"/").Return(objects, test.listObjectsError)
|
||||
for i, obj := range objects {
|
||||
var err error
|
||||
if i < len(test.deleteErrors) {
|
||||
err = test.deleteErrors[i]
|
||||
}
|
||||
|
||||
objectStore.On("DeleteObject", bucket, obj).Return(err)
|
||||
}
|
||||
|
||||
err := DeleteBackupDir(logger, objectStore, bucket, backup)
|
||||
|
||||
if test.expectedErr != "" {
|
||||
assert.EqualError(t, err, test.expectedErr)
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
objectStore.AssertExpectations(t)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetAllBackups(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
storageData map[string][]byte
|
||||
expectedRes []*api.Backup
|
||||
expectedErr string
|
||||
}{
|
||||
{
|
||||
name: "normal case",
|
||||
storageData: map[string][]byte{
|
||||
"backup-1/ark-backup.json": encodeToBytes(&api.Backup{ObjectMeta: metav1.ObjectMeta{Name: "backup-1"}}),
|
||||
"backup-2/ark-backup.json": encodeToBytes(&api.Backup{ObjectMeta: metav1.ObjectMeta{Name: "backup-2"}}),
|
||||
},
|
||||
expectedRes: []*api.Backup{
|
||||
{
|
||||
TypeMeta: metav1.TypeMeta{Kind: "Backup", APIVersion: "ark.heptio.com/v1"},
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "backup-1"},
|
||||
},
|
||||
{
|
||||
TypeMeta: metav1.TypeMeta{Kind: "Backup", APIVersion: "ark.heptio.com/v1"},
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "backup-2"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "backup that can't be decoded is ignored",
|
||||
storageData: map[string][]byte{
|
||||
"backup-1/ark-backup.json": encodeToBytes(&api.Backup{ObjectMeta: metav1.ObjectMeta{Name: "backup-1"}}),
|
||||
"backup-2/ark-backup.json": []byte("this is not valid backup JSON"),
|
||||
},
|
||||
expectedRes: []*api.Backup{
|
||||
{
|
||||
TypeMeta: metav1.TypeMeta{Kind: "Backup", APIVersion: "ark.heptio.com/v1"},
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "backup-1"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
var (
|
||||
bucket = "bucket"
|
||||
objectStore = &testutil.ObjectStore{}
|
||||
logger = arktest.NewLogger()
|
||||
)
|
||||
|
||||
objectStore.On("ListCommonPrefixes", bucket, "/").Return([]string{"backup-1", "backup-2"}, nil)
|
||||
objectStore.On("GetObject", bucket, "backup-1/ark-backup.json").Return(ioutil.NopCloser(bytes.NewReader(test.storageData["backup-1/ark-backup.json"])), nil)
|
||||
objectStore.On("GetObject", bucket, "backup-2/ark-backup.json").Return(ioutil.NopCloser(bytes.NewReader(test.storageData["backup-2/ark-backup.json"])), nil)
|
||||
|
||||
res, err := ListBackups(logger, objectStore, bucket)
|
||||
|
||||
if test.expectedErr != "" {
|
||||
assert.EqualError(t, err, test.expectedErr)
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
assert.Equal(t, test.expectedRes, res)
|
||||
|
||||
objectStore.AssertExpectations(t)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateSignedURL(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
targetKind api.DownloadTargetKind
|
||||
targetName string
|
||||
directory string
|
||||
expectedKey string
|
||||
}{
|
||||
{
|
||||
name: "backup contents",
|
||||
targetKind: api.DownloadTargetKindBackupContents,
|
||||
targetName: "my-backup",
|
||||
directory: "my-backup",
|
||||
expectedKey: "my-backup/my-backup.tar.gz",
|
||||
},
|
||||
{
|
||||
name: "backup log",
|
||||
targetKind: api.DownloadTargetKindBackupLog,
|
||||
targetName: "my-backup",
|
||||
directory: "my-backup",
|
||||
expectedKey: "my-backup/my-backup-logs.gz",
|
||||
},
|
||||
{
|
||||
name: "scheduled backup contents",
|
||||
targetKind: api.DownloadTargetKindBackupContents,
|
||||
targetName: "my-backup-20170913154901",
|
||||
directory: "my-backup-20170913154901",
|
||||
expectedKey: "my-backup-20170913154901/my-backup-20170913154901.tar.gz",
|
||||
},
|
||||
{
|
||||
name: "scheduled backup log",
|
||||
targetKind: api.DownloadTargetKindBackupLog,
|
||||
targetName: "my-backup-20170913154901",
|
||||
directory: "my-backup-20170913154901",
|
||||
expectedKey: "my-backup-20170913154901/my-backup-20170913154901-logs.gz",
|
||||
},
|
||||
{
|
||||
name: "restore log",
|
||||
targetKind: api.DownloadTargetKindRestoreLog,
|
||||
targetName: "b-20170913154901",
|
||||
directory: "b",
|
||||
expectedKey: "b/restore-b-20170913154901-logs.gz",
|
||||
},
|
||||
{
|
||||
name: "restore results",
|
||||
targetKind: api.DownloadTargetKindRestoreResults,
|
||||
targetName: "b-20170913154901",
|
||||
directory: "b",
|
||||
expectedKey: "b/restore-b-20170913154901-results.gz",
|
||||
},
|
||||
{
|
||||
name: "restore results - backup has multiple dashes (e.g. restore of scheduled backup)",
|
||||
targetKind: api.DownloadTargetKindRestoreResults,
|
||||
targetName: "b-cool-20170913154901-20170913154902",
|
||||
directory: "b-cool-20170913154901",
|
||||
expectedKey: "b-cool-20170913154901/restore-b-cool-20170913154901-20170913154902-results.gz",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
var (
|
||||
objectStore = &testutil.ObjectStore{}
|
||||
)
|
||||
defer objectStore.AssertExpectations(t)
|
||||
|
||||
target := api.DownloadTarget{
|
||||
Kind: test.targetKind,
|
||||
Name: test.targetName,
|
||||
}
|
||||
objectStore.On("CreateSignedURL", "bucket", test.expectedKey, time.Duration(0)).Return("url", nil)
|
||||
url, err := CreateSignedURL(objectStore, target, "bucket", test.directory, 0)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "url", url)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func jsonMarshal(obj interface{}) []byte {
|
||||
res, err := json.Marshal(obj)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func encodeToBytes(obj runtime.Object) []byte {
|
||||
res, err := encode.Encode(obj, "json")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
type stringReadSeeker struct {
|
||||
*strings.Reader
|
||||
}
|
||||
|
||||
func newStringReadSeeker(s string) *stringReadSeeker {
|
||||
return &stringReadSeeker{
|
||||
Reader: strings.NewReader(s),
|
||||
}
|
||||
}
|
||||
|
||||
func (srs *stringReadSeeker) Seek(offset int64, whence int) (int64, error) {
|
||||
return 0, nil
|
||||
}
|
||||
@@ -0,0 +1,299 @@
|
||||
/*
|
||||
Copyright 2018 the Heptio Ark contributors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package persistence
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
kerrors "k8s.io/apimachinery/pkg/util/errors"
|
||||
|
||||
arkv1api "github.com/heptio/ark/pkg/apis/ark/v1"
|
||||
"github.com/heptio/ark/pkg/cloudprovider"
|
||||
"github.com/heptio/ark/pkg/generated/clientset/versioned/scheme"
|
||||
)
|
||||
|
||||
// BackupStore defines operations for creating, retrieving, and deleting
|
||||
// Ark backup and restore data in/from a persistent backup store.
|
||||
type BackupStore interface {
|
||||
ListBackups() ([]*arkv1api.Backup, error)
|
||||
|
||||
PutBackup(name string, metadata, contents, log io.Reader) error
|
||||
GetBackupMetadata(name string) (*arkv1api.Backup, error)
|
||||
GetBackupContents(name string) (io.ReadCloser, error)
|
||||
DeleteBackup(name string) error
|
||||
|
||||
PutRestoreLog(backup, restore string, log io.Reader) error
|
||||
PutRestoreResults(backup, restore string, results io.Reader) error
|
||||
|
||||
GetDownloadURL(backup string, target arkv1api.DownloadTarget) (string, error)
|
||||
}
|
||||
|
||||
const (
|
||||
// DownloadURLTTL is how long a download URL is valid for.
|
||||
DownloadURLTTL = 10 * time.Minute
|
||||
|
||||
backupMetadataFileFormatString = "%s/ark-backup.json"
|
||||
backupFileFormatString = "%s/%s.tar.gz"
|
||||
backupLogFileFormatString = "%s/%s-logs.gz"
|
||||
restoreLogFileFormatString = "%s/restore-%s-logs.gz"
|
||||
restoreResultsFileFormatString = "%s/restore-%s-results.gz"
|
||||
)
|
||||
|
||||
func getPrefix(prefix string) string {
|
||||
if prefix == "" || strings.HasSuffix(prefix, "/") {
|
||||
return prefix
|
||||
}
|
||||
|
||||
return prefix + "/"
|
||||
}
|
||||
|
||||
func getBackupMetadataKey(prefix, backup string) string {
|
||||
return prefix + fmt.Sprintf(backupMetadataFileFormatString, backup)
|
||||
}
|
||||
|
||||
func getBackupContentsKey(prefix, backup string) string {
|
||||
return prefix + fmt.Sprintf(backupFileFormatString, backup, backup)
|
||||
}
|
||||
|
||||
func getBackupLogKey(prefix, backup string) string {
|
||||
return prefix + fmt.Sprintf(backupLogFileFormatString, backup, backup)
|
||||
}
|
||||
|
||||
func getRestoreLogKey(prefix, backup, restore string) string {
|
||||
return prefix + fmt.Sprintf(restoreLogFileFormatString, backup, restore)
|
||||
}
|
||||
|
||||
func getRestoreResultsKey(prefix, backup, restore string) string {
|
||||
return prefix + fmt.Sprintf(restoreResultsFileFormatString, backup, restore)
|
||||
}
|
||||
|
||||
type objectBackupStore struct {
|
||||
objectStore cloudprovider.ObjectStore
|
||||
bucket string
|
||||
prefix string
|
||||
logger logrus.FieldLogger
|
||||
}
|
||||
|
||||
// ObjectStoreGetter is a type that can get a cloudprovider.ObjectStore
|
||||
// from a provider name.
|
||||
type ObjectStoreGetter interface {
|
||||
GetObjectStore(provider string) (cloudprovider.ObjectStore, error)
|
||||
}
|
||||
|
||||
func NewObjectBackupStore(location *arkv1api.BackupStorageLocation, objectStoreGetter ObjectStoreGetter, logger logrus.FieldLogger) (BackupStore, error) {
|
||||
if location.Spec.ObjectStorage == nil {
|
||||
return nil, errors.New("backup storage location does not use object storage")
|
||||
}
|
||||
|
||||
if location.Spec.Provider == "" {
|
||||
return nil, errors.New("object storage provider name must not be empty")
|
||||
}
|
||||
|
||||
objectStore, err := objectStoreGetter.GetObjectStore(location.Spec.Provider)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// add the bucket name to the config map so that object stores can use
|
||||
// it when initializing. The AWS object store uses this to determine the
|
||||
// bucket's region when setting up its client.
|
||||
if location.Spec.ObjectStorage != nil {
|
||||
if location.Spec.Config == nil {
|
||||
location.Spec.Config = make(map[string]string)
|
||||
}
|
||||
location.Spec.Config["bucket"] = location.Spec.ObjectStorage.Bucket
|
||||
}
|
||||
|
||||
if err := objectStore.Init(location.Spec.Config); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
prefix := getPrefix(location.Spec.ObjectStorage.Prefix)
|
||||
|
||||
log := logger.WithFields(logrus.Fields(map[string]interface{}{
|
||||
"bucket": location.Spec.ObjectStorage.Bucket,
|
||||
"prefix": prefix,
|
||||
}))
|
||||
|
||||
return &objectBackupStore{
|
||||
objectStore: objectStore,
|
||||
bucket: location.Spec.ObjectStorage.Bucket,
|
||||
prefix: prefix,
|
||||
logger: log,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *objectBackupStore) ListBackups() ([]*arkv1api.Backup, error) {
|
||||
prefixes, err := s.objectStore.ListCommonPrefixes(s.bucket, s.prefix, "/")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(prefixes) == 0 {
|
||||
return []*arkv1api.Backup{}, nil
|
||||
}
|
||||
|
||||
output := make([]*arkv1api.Backup, 0, len(prefixes))
|
||||
|
||||
for _, prefix := range prefixes {
|
||||
// values returned from a call to cloudprovider.ObjectStore's
|
||||
// ListcommonPrefixes method return the *full* prefix, inclusive
|
||||
// of s.prefix, and include the delimiter ("/") as a suffix. Trim
|
||||
// each of those off to get the backup name.
|
||||
backupName := strings.TrimSuffix(strings.TrimPrefix(prefix, s.prefix), "/")
|
||||
|
||||
backup, err := s.GetBackupMetadata(backupName)
|
||||
if err != nil {
|
||||
s.logger.WithError(err).WithField("dir", backupName).Error("Error reading backup directory")
|
||||
continue
|
||||
}
|
||||
|
||||
output = append(output, backup)
|
||||
}
|
||||
|
||||
return output, nil
|
||||
}
|
||||
|
||||
func (s *objectBackupStore) PutBackup(name string, metadata io.Reader, contents io.Reader, log io.Reader) error {
|
||||
if err := seekAndPutObject(s.objectStore, s.bucket, getBackupLogKey(s.prefix, name), log); err != nil {
|
||||
// Uploading the log file is best-effort; if it fails, we log the error but it doesn't impact the
|
||||
// backup's status.
|
||||
s.logger.WithError(err).WithField("backup", name).Error("Error uploading log file")
|
||||
}
|
||||
|
||||
if metadata == nil {
|
||||
// If we don't have metadata, something failed, and there's no point in continuing. An object
|
||||
// storage bucket that is missing the metadata file can't be restored, nor can its logs be
|
||||
// viewed.
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := seekAndPutObject(s.objectStore, s.bucket, getBackupMetadataKey(s.prefix, name), metadata); err != nil {
|
||||
// failure to upload metadata file is a hard-stop
|
||||
return err
|
||||
}
|
||||
|
||||
if err := seekAndPutObject(s.objectStore, s.bucket, getBackupContentsKey(s.prefix, name), contents); err != nil {
|
||||
deleteErr := s.objectStore.DeleteObject(s.bucket, getBackupMetadataKey(s.prefix, name))
|
||||
return kerrors.NewAggregate([]error{err, deleteErr})
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *objectBackupStore) GetBackupMetadata(name string) (*arkv1api.Backup, error) {
|
||||
key := getBackupMetadataKey(s.prefix, name)
|
||||
|
||||
res, err := s.objectStore.GetObject(s.bucket, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer res.Close()
|
||||
|
||||
data, err := ioutil.ReadAll(res)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
decoder := scheme.Codecs.UniversalDecoder(arkv1api.SchemeGroupVersion)
|
||||
obj, _, err := decoder.Decode(data, nil, nil)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
backupObj, ok := obj.(*arkv1api.Backup)
|
||||
if !ok {
|
||||
return nil, errors.Errorf("unexpected type for %s/%s: %T", s.bucket, key, obj)
|
||||
}
|
||||
|
||||
return backupObj, nil
|
||||
|
||||
}
|
||||
|
||||
func (s *objectBackupStore) GetBackupContents(name string) (io.ReadCloser, error) {
|
||||
return s.objectStore.GetObject(s.bucket, getBackupContentsKey(s.prefix, name))
|
||||
}
|
||||
|
||||
func (s *objectBackupStore) DeleteBackup(name string) error {
|
||||
objects, err := s.objectStore.ListObjects(s.bucket, s.prefix+name+"/")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var errs []error
|
||||
for _, key := range objects {
|
||||
s.logger.WithFields(logrus.Fields{
|
||||
"key": key,
|
||||
}).Debug("Trying to delete object")
|
||||
if err := s.objectStore.DeleteObject(s.bucket, key); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
|
||||
return errors.WithStack(kerrors.NewAggregate(errs))
|
||||
}
|
||||
|
||||
func (s *objectBackupStore) PutRestoreLog(backup string, restore string, log io.Reader) error {
|
||||
return s.objectStore.PutObject(s.bucket, getRestoreLogKey(s.prefix, backup, restore), log)
|
||||
}
|
||||
|
||||
func (s *objectBackupStore) PutRestoreResults(backup string, restore string, results io.Reader) error {
|
||||
return s.objectStore.PutObject(s.bucket, getRestoreResultsKey(s.prefix, backup, restore), results)
|
||||
}
|
||||
|
||||
func (s *objectBackupStore) GetDownloadURL(backup string, target arkv1api.DownloadTarget) (string, error) {
|
||||
switch target.Kind {
|
||||
case arkv1api.DownloadTargetKindBackupContents:
|
||||
return s.objectStore.CreateSignedURL(s.bucket, getBackupContentsKey(s.prefix, backup), DownloadURLTTL)
|
||||
case arkv1api.DownloadTargetKindBackupLog:
|
||||
return s.objectStore.CreateSignedURL(s.bucket, getBackupLogKey(s.prefix, backup), DownloadURLTTL)
|
||||
case arkv1api.DownloadTargetKindRestoreLog:
|
||||
return s.objectStore.CreateSignedURL(s.bucket, getRestoreLogKey(s.prefix, backup, target.Name), DownloadURLTTL)
|
||||
case arkv1api.DownloadTargetKindRestoreResults:
|
||||
return s.objectStore.CreateSignedURL(s.bucket, getRestoreResultsKey(s.prefix, backup, target.Name), DownloadURLTTL)
|
||||
default:
|
||||
return "", errors.Errorf("unsupported download target kind %q", target.Kind)
|
||||
}
|
||||
}
|
||||
|
||||
func seekToBeginning(r io.Reader) error {
|
||||
seeker, ok := r.(io.Seeker)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
_, err := seeker.Seek(0, 0)
|
||||
return err
|
||||
}
|
||||
|
||||
func seekAndPutObject(objectStore cloudprovider.ObjectStore, bucket, key string, file io.Reader) error {
|
||||
if file == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := seekToBeginning(file); err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
return objectStore.PutObject(bucket, key, file)
|
||||
}
|
||||
@@ -0,0 +1,403 @@
|
||||
/*
|
||||
Copyright 2017 the Heptio Ark contributors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package persistence
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
|
||||
api "github.com/heptio/ark/pkg/apis/ark/v1"
|
||||
"github.com/heptio/ark/pkg/cloudprovider"
|
||||
"github.com/heptio/ark/pkg/util/encode"
|
||||
arktest "github.com/heptio/ark/pkg/util/test"
|
||||
)
|
||||
|
||||
type objectBackupStoreTestHarness struct {
|
||||
// embedded to reduce verbosity when calling methods
|
||||
*objectBackupStore
|
||||
|
||||
objectStore *cloudprovider.InMemoryObjectStore
|
||||
bucket, prefix string
|
||||
}
|
||||
|
||||
func newObjectBackupStoreTestHarness(bucket, prefix string) *objectBackupStoreTestHarness {
|
||||
objectStore := cloudprovider.NewInMemoryObjectStore(bucket)
|
||||
|
||||
return &objectBackupStoreTestHarness{
|
||||
objectBackupStore: &objectBackupStore{
|
||||
objectStore: objectStore,
|
||||
bucket: bucket,
|
||||
prefix: prefix,
|
||||
logger: arktest.NewLogger(),
|
||||
},
|
||||
objectStore: objectStore,
|
||||
bucket: bucket,
|
||||
prefix: prefix,
|
||||
}
|
||||
}
|
||||
|
||||
func TestListBackups(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
prefix string
|
||||
storageData cloudprovider.BucketData
|
||||
expectedRes []*api.Backup
|
||||
expectedErr string
|
||||
}{
|
||||
{
|
||||
name: "normal case",
|
||||
storageData: map[string][]byte{
|
||||
"backup-1/ark-backup.json": encodeToBytes(&api.Backup{ObjectMeta: metav1.ObjectMeta{Name: "backup-1"}}),
|
||||
"backup-2/ark-backup.json": encodeToBytes(&api.Backup{ObjectMeta: metav1.ObjectMeta{Name: "backup-2"}}),
|
||||
},
|
||||
expectedRes: []*api.Backup{
|
||||
{
|
||||
TypeMeta: metav1.TypeMeta{Kind: "Backup", APIVersion: "ark.heptio.com/v1"},
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "backup-1"},
|
||||
},
|
||||
{
|
||||
TypeMeta: metav1.TypeMeta{Kind: "Backup", APIVersion: "ark.heptio.com/v1"},
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "backup-2"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "normal case with backup store prefix",
|
||||
prefix: "ark-backups/",
|
||||
storageData: map[string][]byte{
|
||||
"ark-backups/backup-1/ark-backup.json": encodeToBytes(&api.Backup{ObjectMeta: metav1.ObjectMeta{Name: "backup-1"}}),
|
||||
"ark-backups/backup-2/ark-backup.json": encodeToBytes(&api.Backup{ObjectMeta: metav1.ObjectMeta{Name: "backup-2"}}),
|
||||
},
|
||||
expectedRes: []*api.Backup{
|
||||
{
|
||||
TypeMeta: metav1.TypeMeta{Kind: "Backup", APIVersion: "ark.heptio.com/v1"},
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "backup-1"},
|
||||
},
|
||||
{
|
||||
TypeMeta: metav1.TypeMeta{Kind: "Backup", APIVersion: "ark.heptio.com/v1"},
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "backup-2"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "backup that can't be decoded is ignored",
|
||||
storageData: map[string][]byte{
|
||||
"backup-1/ark-backup.json": encodeToBytes(&api.Backup{ObjectMeta: metav1.ObjectMeta{Name: "backup-1"}}),
|
||||
"backup-2/ark-backup.json": []byte("this is not valid backup JSON"),
|
||||
},
|
||||
expectedRes: []*api.Backup{
|
||||
{
|
||||
TypeMeta: metav1.TypeMeta{Kind: "Backup", APIVersion: "ark.heptio.com/v1"},
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "backup-1"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
harness := newObjectBackupStoreTestHarness("foo", tc.prefix)
|
||||
|
||||
for key, obj := range tc.storageData {
|
||||
require.NoError(t, harness.objectStore.PutObject(harness.bucket, key, bytes.NewReader(obj)))
|
||||
}
|
||||
|
||||
res, err := harness.ListBackups()
|
||||
|
||||
arktest.AssertErrorMatches(t, tc.expectedErr, err)
|
||||
|
||||
getComparer := func(obj []*api.Backup) func(i, j int) bool {
|
||||
return func(i, j int) bool {
|
||||
switch strings.Compare(obj[i].Namespace, obj[j].Namespace) {
|
||||
case -1:
|
||||
return true
|
||||
case 1:
|
||||
return false
|
||||
default:
|
||||
// namespaces are the same: compare by name
|
||||
return obj[i].Name < obj[j].Name
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sort.Slice(tc.expectedRes, getComparer(tc.expectedRes))
|
||||
sort.Slice(res, getComparer(res))
|
||||
|
||||
assert.Equal(t, tc.expectedRes, res)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPutBackup(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
prefix string
|
||||
metadata io.Reader
|
||||
contents io.Reader
|
||||
log io.Reader
|
||||
expectedErr string
|
||||
expectedKeys []string
|
||||
}{
|
||||
{
|
||||
name: "normal case",
|
||||
metadata: newStringReadSeeker("metadata"),
|
||||
contents: newStringReadSeeker("contents"),
|
||||
log: newStringReadSeeker("log"),
|
||||
expectedErr: "",
|
||||
expectedKeys: []string{"backup-1/ark-backup.json", "backup-1/backup-1.tar.gz", "backup-1/backup-1-logs.gz"},
|
||||
},
|
||||
{
|
||||
name: "normal case with backup store prefix",
|
||||
prefix: "prefix-1/",
|
||||
metadata: newStringReadSeeker("metadata"),
|
||||
contents: newStringReadSeeker("contents"),
|
||||
log: newStringReadSeeker("log"),
|
||||
expectedErr: "",
|
||||
expectedKeys: []string{"prefix-1/backup-1/ark-backup.json", "prefix-1/backup-1/backup-1.tar.gz", "prefix-1/backup-1/backup-1-logs.gz"},
|
||||
},
|
||||
{
|
||||
name: "error on metadata upload does not upload data",
|
||||
metadata: new(errorReader),
|
||||
contents: newStringReadSeeker("contents"),
|
||||
log: newStringReadSeeker("log"),
|
||||
expectedErr: "error readers return errors",
|
||||
expectedKeys: []string{"backup-1/backup-1-logs.gz"},
|
||||
},
|
||||
{
|
||||
name: "error on data upload deletes metadata",
|
||||
metadata: newStringReadSeeker("metadata"),
|
||||
contents: new(errorReader),
|
||||
log: newStringReadSeeker("log"),
|
||||
expectedErr: "error readers return errors",
|
||||
expectedKeys: []string{"backup-1/backup-1-logs.gz"},
|
||||
},
|
||||
{
|
||||
name: "error on log upload is ok",
|
||||
metadata: newStringReadSeeker("foo"),
|
||||
contents: newStringReadSeeker("bar"),
|
||||
log: new(errorReader),
|
||||
expectedErr: "",
|
||||
expectedKeys: []string{"backup-1/ark-backup.json", "backup-1/backup-1.tar.gz"},
|
||||
},
|
||||
{
|
||||
name: "don't upload data when metadata is nil",
|
||||
metadata: nil,
|
||||
contents: newStringReadSeeker("contents"),
|
||||
log: newStringReadSeeker("log"),
|
||||
expectedErr: "",
|
||||
expectedKeys: []string{"backup-1/backup-1-logs.gz"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
harness := newObjectBackupStoreTestHarness("foo", tc.prefix)
|
||||
|
||||
err := harness.PutBackup("backup-1", tc.metadata, tc.contents, tc.log)
|
||||
|
||||
arktest.AssertErrorMatches(t, tc.expectedErr, err)
|
||||
assert.Len(t, harness.objectStore.Data[harness.bucket], len(tc.expectedKeys))
|
||||
for _, key := range tc.expectedKeys {
|
||||
assert.Contains(t, harness.objectStore.Data[harness.bucket], key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetBackupContents(t *testing.T) {
|
||||
harness := newObjectBackupStoreTestHarness("test-bucket", "")
|
||||
|
||||
harness.objectStore.PutObject(harness.bucket, "test-backup/test-backup.tar.gz", newStringReadSeeker("foo"))
|
||||
|
||||
rc, err := harness.GetBackupContents("test-backup")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, rc)
|
||||
|
||||
data, err := ioutil.ReadAll(rc)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "foo", string(data))
|
||||
}
|
||||
|
||||
func TestDeleteBackup(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
prefix string
|
||||
listObjectsError error
|
||||
deleteErrors []error
|
||||
expectedErr string
|
||||
}{
|
||||
{
|
||||
name: "normal case",
|
||||
},
|
||||
{
|
||||
name: "normal case with backup store prefix",
|
||||
prefix: "ark-backups/",
|
||||
},
|
||||
{
|
||||
name: "some delete errors, do as much as we can",
|
||||
deleteErrors: []error{errors.New("a"), nil, errors.New("c")},
|
||||
expectedErr: "[a, c]",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
objectStore := new(arktest.ObjectStore)
|
||||
backupStore := &objectBackupStore{
|
||||
objectStore: objectStore,
|
||||
bucket: "test-bucket",
|
||||
prefix: test.prefix,
|
||||
logger: arktest.NewLogger(),
|
||||
}
|
||||
defer objectStore.AssertExpectations(t)
|
||||
|
||||
objects := []string{test.prefix + "bak/ark-backup.json", test.prefix + "bak/bak.tar.gz", test.prefix + "bak/bak.log.gz"}
|
||||
|
||||
objectStore.On("ListObjects", backupStore.bucket, test.prefix+"bak/").Return(objects, test.listObjectsError)
|
||||
for i, obj := range objects {
|
||||
var err error
|
||||
if i < len(test.deleteErrors) {
|
||||
err = test.deleteErrors[i]
|
||||
}
|
||||
|
||||
objectStore.On("DeleteObject", backupStore.bucket, obj).Return(err)
|
||||
}
|
||||
|
||||
err := backupStore.DeleteBackup("bak")
|
||||
|
||||
arktest.AssertErrorMatches(t, test.expectedErr, err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetDownloadURL(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
targetKind api.DownloadTargetKind
|
||||
targetName string
|
||||
directory string
|
||||
prefix string
|
||||
expectedKey string
|
||||
}{
|
||||
{
|
||||
name: "backup contents",
|
||||
targetKind: api.DownloadTargetKindBackupContents,
|
||||
targetName: "my-backup",
|
||||
directory: "my-backup",
|
||||
expectedKey: "my-backup/my-backup.tar.gz",
|
||||
},
|
||||
{
|
||||
name: "backup log",
|
||||
targetKind: api.DownloadTargetKindBackupLog,
|
||||
targetName: "my-backup",
|
||||
directory: "my-backup",
|
||||
expectedKey: "my-backup/my-backup-logs.gz",
|
||||
},
|
||||
{
|
||||
name: "scheduled backup contents",
|
||||
targetKind: api.DownloadTargetKindBackupContents,
|
||||
targetName: "my-backup-20170913154901",
|
||||
directory: "my-backup-20170913154901",
|
||||
expectedKey: "my-backup-20170913154901/my-backup-20170913154901.tar.gz",
|
||||
},
|
||||
{
|
||||
name: "scheduled backup log",
|
||||
targetKind: api.DownloadTargetKindBackupLog,
|
||||
targetName: "my-backup-20170913154901",
|
||||
directory: "my-backup-20170913154901",
|
||||
expectedKey: "my-backup-20170913154901/my-backup-20170913154901-logs.gz",
|
||||
},
|
||||
{
|
||||
name: "backup contents with backup store prefix",
|
||||
targetKind: api.DownloadTargetKindBackupContents,
|
||||
targetName: "my-backup",
|
||||
directory: "my-backup",
|
||||
prefix: "ark-backups/",
|
||||
expectedKey: "ark-backups/my-backup/my-backup.tar.gz",
|
||||
},
|
||||
{
|
||||
name: "restore log",
|
||||
targetKind: api.DownloadTargetKindRestoreLog,
|
||||
targetName: "b-20170913154901",
|
||||
directory: "b",
|
||||
expectedKey: "b/restore-b-20170913154901-logs.gz",
|
||||
},
|
||||
{
|
||||
name: "restore results",
|
||||
targetKind: api.DownloadTargetKindRestoreResults,
|
||||
targetName: "b-20170913154901",
|
||||
directory: "b",
|
||||
expectedKey: "b/restore-b-20170913154901-results.gz",
|
||||
},
|
||||
{
|
||||
name: "restore results - backup has multiple dashes (e.g. restore of scheduled backup)",
|
||||
targetKind: api.DownloadTargetKindRestoreResults,
|
||||
targetName: "b-cool-20170913154901-20170913154902",
|
||||
directory: "b-cool-20170913154901",
|
||||
expectedKey: "b-cool-20170913154901/restore-b-cool-20170913154901-20170913154902-results.gz",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
harness := newObjectBackupStoreTestHarness("test-bucket", test.prefix)
|
||||
|
||||
require.NoError(t, harness.objectStore.PutObject("test-bucket", test.expectedKey, newStringReadSeeker("foo")))
|
||||
|
||||
url, err := harness.GetDownloadURL(test.directory, api.DownloadTarget{Kind: test.targetKind, Name: test.targetName})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "a-url", url)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func encodeToBytes(obj runtime.Object) []byte {
|
||||
res, err := encode.Encode(obj, "json")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
type stringReadSeeker struct {
|
||||
*strings.Reader
|
||||
}
|
||||
|
||||
func newStringReadSeeker(s string) *stringReadSeeker {
|
||||
return &stringReadSeeker{
|
||||
Reader: strings.NewReader(s),
|
||||
}
|
||||
}
|
||||
|
||||
func (srs *stringReadSeeker) Seek(offset int64, whence int) (int64, error) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
type errorReader struct{}
|
||||
|
||||
func (r *errorReader) Read([]byte) (int, error) {
|
||||
return 0, errors.New("error readers return errors")
|
||||
}
|
||||
Reference in New Issue
Block a user