From af64069d65f587fb5d144ac27fa407ea8f682f58 Mon Sep 17 00:00:00 2001 From: Steve Kriss Date: Mon, 20 Aug 2018 11:47:16 -0700 Subject: [PATCH] create pkg/persistence and move relevant code from pkg/cloudprovider into it Signed-off-by: Steve Kriss --- pkg/controller/backup_controller.go | 3 +- pkg/controller/backup_deletion_controller.go | 5 ++- pkg/controller/backup_sync_controller.go | 3 +- pkg/controller/download_request_controller.go | 6 +-- pkg/controller/restore_controller.go | 19 +++++---- .../object_storage_persistence.go} | 41 ++++++++++--------- .../object_storage_persistence_test.go} | 2 +- 7 files changed, 42 insertions(+), 37 deletions(-) rename pkg/{cloudprovider/backup_service.go => persistence/object_storage_persistence.go} (76%) rename pkg/{cloudprovider/backup_service_test.go => persistence/object_storage_persistence_test.go} (99%) diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index 1ffd4a81a..43a731b84 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -47,6 +47,7 @@ import ( informers "github.com/heptio/ark/pkg/generated/informers/externalversions/ark/v1" listers "github.com/heptio/ark/pkg/generated/listers/ark/v1" "github.com/heptio/ark/pkg/metrics" + "github.com/heptio/ark/pkg/persistence" "github.com/heptio/ark/pkg/plugin" "github.com/heptio/ark/pkg/util/collections" "github.com/heptio/ark/pkg/util/encode" @@ -437,7 +438,7 @@ func (controller *backupController) runBackup(backup *api.Backup, backupLocation controller.logger.WithError(err).Error("error closing gzippedLogFile") } - if err := cloudprovider.UploadBackup(log, objectStore, backupLocation.Spec.ObjectStorage.Bucket, backup.Name, backupJSONToUpload, backupFileToUpload, logFile); err != nil { + if err := persistence.UploadBackup(log, objectStore, backupLocation.Spec.ObjectStorage.Bucket, backup.Name, backupJSONToUpload, backupFileToUpload, logFile); err != nil { errs = append(errs, err) } diff --git a/pkg/controller/backup_deletion_controller.go b/pkg/controller/backup_deletion_controller.go index 3fd77d4c2..9c63e0c3e 100644 --- a/pkg/controller/backup_deletion_controller.go +++ b/pkg/controller/backup_deletion_controller.go @@ -28,6 +28,7 @@ import ( arkv1client "github.com/heptio/ark/pkg/generated/clientset/versioned/typed/ark/v1" informers "github.com/heptio/ark/pkg/generated/informers/externalversions/ark/v1" listers "github.com/heptio/ark/pkg/generated/listers/ark/v1" + "github.com/heptio/ark/pkg/persistence" "github.com/heptio/ark/pkg/plugin" "github.com/heptio/ark/pkg/restic" "github.com/heptio/ark/pkg/util/kube" @@ -57,7 +58,7 @@ type backupDeletionController struct { resticMgr restic.RepositoryManager podvolumeBackupLister listers.PodVolumeBackupLister backupLocationLister listers.BackupStorageLocationLister - deleteBackupDir cloudprovider.DeleteBackupDirFunc + deleteBackupDir persistence.DeleteBackupDirFunc processRequestFunc func(*v1.DeleteBackupRequest) error clock clock.Clock newPluginManager func(logrus.FieldLogger) plugin.Manager @@ -94,7 +95,7 @@ func NewBackupDeletionController( // use variables to refer to these functions so they can be // replaced with fakes for testing. newPluginManager: newPluginManager, - deleteBackupDir: cloudprovider.DeleteBackupDir, + deleteBackupDir: persistence.DeleteBackupDir, clock: &clock.RealClock{}, } diff --git a/pkg/controller/backup_sync_controller.go b/pkg/controller/backup_sync_controller.go index 38904051d..f1f1700e6 100644 --- a/pkg/controller/backup_sync_controller.go +++ b/pkg/controller/backup_sync_controller.go @@ -33,6 +33,7 @@ import ( arkv1client "github.com/heptio/ark/pkg/generated/clientset/versioned/typed/ark/v1" informers "github.com/heptio/ark/pkg/generated/informers/externalversions/ark/v1" listers "github.com/heptio/ark/pkg/generated/listers/ark/v1" + "github.com/heptio/ark/pkg/persistence" "github.com/heptio/ark/pkg/plugin" "github.com/heptio/ark/pkg/util/kube" "github.com/heptio/ark/pkg/util/stringslice" @@ -76,7 +77,7 @@ func NewBackupSyncController( // use variables to refer to these functions so they can be // replaced with fakes for testing. newPluginManager: newPluginManager, - listCloudBackups: cloudprovider.ListBackups, + listCloudBackups: persistence.ListBackups, } c.resyncFunc = c.run diff --git a/pkg/controller/download_request_controller.go b/pkg/controller/download_request_controller.go index 7a4c025f3..747e6ce40 100644 --- a/pkg/controller/download_request_controller.go +++ b/pkg/controller/download_request_controller.go @@ -32,10 +32,10 @@ import ( "k8s.io/client-go/tools/cache" "github.com/heptio/ark/pkg/apis/ark/v1" - "github.com/heptio/ark/pkg/cloudprovider" arkv1client "github.com/heptio/ark/pkg/generated/clientset/versioned/typed/ark/v1" informers "github.com/heptio/ark/pkg/generated/informers/externalversions/ark/v1" listers "github.com/heptio/ark/pkg/generated/listers/ark/v1" + "github.com/heptio/ark/pkg/persistence" "github.com/heptio/ark/pkg/plugin" "github.com/heptio/ark/pkg/util/kube" ) @@ -47,7 +47,7 @@ type downloadRequestController struct { downloadRequestLister listers.DownloadRequestLister restoreLister listers.RestoreLister clock clock.Clock - createSignedURL cloudprovider.CreateSignedURLFunc + createSignedURL persistence.CreateSignedURLFunc backupLocationLister listers.BackupStorageLocationLister backupLister listers.BackupLister newPluginManager func(logrus.FieldLogger) plugin.Manager @@ -73,7 +73,7 @@ func NewDownloadRequestController( // use variables to refer to these functions so they can be // replaced with fakes for testing. - createSignedURL: cloudprovider.CreateSignedURL, + createSignedURL: persistence.CreateSignedURL, newPluginManager: newPluginManager, clock: &clock.RealClock{}, diff --git a/pkg/controller/restore_controller.go b/pkg/controller/restore_controller.go index fead34d3a..f4b01d2ff 100644 --- a/pkg/controller/restore_controller.go +++ b/pkg/controller/restore_controller.go @@ -46,6 +46,7 @@ import ( informers "github.com/heptio/ark/pkg/generated/informers/externalversions/ark/v1" listers "github.com/heptio/ark/pkg/generated/listers/ark/v1" "github.com/heptio/ark/pkg/metrics" + "github.com/heptio/ark/pkg/persistence" "github.com/heptio/ark/pkg/plugin" "github.com/heptio/ark/pkg/restore" "github.com/heptio/ark/pkg/util/boolptr" @@ -89,10 +90,10 @@ type restoreController struct { defaultBackupLocation string metrics *metrics.ServerMetrics - getBackup cloudprovider.GetBackupFunc - downloadBackup cloudprovider.DownloadBackupFunc - uploadRestoreLog cloudprovider.UploadRestoreLogFunc - uploadRestoreResults cloudprovider.UploadRestoreResultsFunc + getBackup persistence.GetBackupFunc + downloadBackup persistence.DownloadBackupFunc + uploadRestoreLog persistence.UploadRestoreLogFunc + uploadRestoreResults persistence.UploadRestoreResultsFunc newPluginManager func(logger logrus.FieldLogger) plugin.Manager } @@ -132,10 +133,10 @@ func NewRestoreController( // use variables to refer to these functions so they can be // replaced with fakes for testing. newPluginManager: newPluginManager, - getBackup: cloudprovider.GetBackup, - downloadBackup: cloudprovider.DownloadBackup, - uploadRestoreLog: cloudprovider.UploadRestoreLog, - uploadRestoreResults: cloudprovider.UploadRestoreResults, + getBackup: persistence.GetBackup, + downloadBackup: persistence.DownloadBackup, + uploadRestoreLog: persistence.UploadRestoreLog, + uploadRestoreResults: persistence.UploadRestoreResults, } c.syncHandler = c.processRestore @@ -667,7 +668,7 @@ func (c *restoreController) runRestore( func downloadToTempFile( objectStore cloudprovider.ObjectStore, bucket, backupName string, - downloadBackup cloudprovider.DownloadBackupFunc, + downloadBackup persistence.DownloadBackupFunc, logger logrus.FieldLogger, ) (*os.File, error) { readCloser, err := downloadBackup(objectStore, bucket, backupName) diff --git a/pkg/cloudprovider/backup_service.go b/pkg/persistence/object_storage_persistence.go similarity index 76% rename from pkg/cloudprovider/backup_service.go rename to pkg/persistence/object_storage_persistence.go index e17859af6..151ceeb62 100644 --- a/pkg/cloudprovider/backup_service.go +++ b/pkg/persistence/object_storage_persistence.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package cloudprovider +package persistence import ( "fmt" @@ -28,6 +28,7 @@ import ( kerrors "k8s.io/apimachinery/pkg/util/errors" api "github.com/heptio/ark/pkg/apis/ark/v1" + "github.com/heptio/ark/pkg/cloudprovider" "github.com/heptio/ark/pkg/generated/clientset/versioned/scheme" ) @@ -75,7 +76,7 @@ func seekToBeginning(r io.Reader) error { return err } -func seekAndPutObject(objectStore ObjectStore, bucket, key string, file io.Reader) error { +func seekAndPutObject(objectStore cloudprovider.ObjectStore, bucket, key string, file io.Reader) error { if file == nil { return nil } @@ -87,27 +88,27 @@ func seekAndPutObject(objectStore ObjectStore, bucket, key string, file io.Reade return objectStore.PutObject(bucket, key, file) } -func UploadBackupLog(objectStore ObjectStore, bucket, backupName string, log io.Reader) error { +func UploadBackupLog(objectStore cloudprovider.ObjectStore, bucket, backupName string, log io.Reader) error { logKey := getBackupLogKey(backupName, backupName) return seekAndPutObject(objectStore, bucket, logKey, log) } -func UploadBackupMetadata(objectStore ObjectStore, bucket, backupName string, metadata io.Reader) error { +func UploadBackupMetadata(objectStore cloudprovider.ObjectStore, bucket, backupName string, metadata io.Reader) error { metadataKey := getMetadataKey(backupName) return seekAndPutObject(objectStore, bucket, metadataKey, metadata) } -func DeleteBackupMetadata(objectStore ObjectStore, bucket, backupName string) error { +func DeleteBackupMetadata(objectStore cloudprovider.ObjectStore, bucket, backupName string) error { metadataKey := getMetadataKey(backupName) return objectStore.DeleteObject(bucket, metadataKey) } -func UploadBackupData(objectStore ObjectStore, bucket, backupName string, backup io.Reader) error { +func UploadBackupData(objectStore cloudprovider.ObjectStore, bucket, backupName string, backup io.Reader) error { backupKey := getBackupContentsKey(backupName, backupName) return seekAndPutObject(objectStore, bucket, backupKey, backup) } -func UploadBackup(logger logrus.FieldLogger, objectStore ObjectStore, bucket, backupName string, metadata, backup, log io.Reader) error { +func UploadBackup(logger logrus.FieldLogger, objectStore cloudprovider.ObjectStore, bucket, backupName string, metadata, backup, log io.Reader) error { if err := UploadBackupLog(objectStore, bucket, backupName, log); err != nil { // Uploading the log file is best-effort; if it fails, we log the error but it doesn't impact the // backup's status. @@ -138,16 +139,16 @@ func UploadBackup(logger logrus.FieldLogger, objectStore ObjectStore, bucket, ba } // DownloadBackupFunc is a function that can download backup metadata from a bucket in object storage. -type DownloadBackupFunc func(objectStore ObjectStore, bucket, backupName string) (io.ReadCloser, error) +type DownloadBackupFunc func(objectStore cloudprovider.ObjectStore, bucket, backupName string) (io.ReadCloser, error) // DownloadBackup downloads an Ark backup with the specified object key from object storage via the cloud API. // It returns the snapshot metadata and data (separately), or an error if a problem is encountered // downloading or reading the file from the cloud API. -func DownloadBackup(objectStore ObjectStore, bucket, backupName string) (io.ReadCloser, error) { +func DownloadBackup(objectStore cloudprovider.ObjectStore, bucket, backupName string) (io.ReadCloser, error) { return objectStore.GetObject(bucket, getBackupContentsKey(backupName, backupName)) } -func ListBackups(logger logrus.FieldLogger, objectStore ObjectStore, bucket string) ([]*api.Backup, error) { +func ListBackups(logger logrus.FieldLogger, objectStore cloudprovider.ObjectStore, bucket string) ([]*api.Backup, error) { prefixes, err := objectStore.ListCommonPrefixes(bucket, "/") if err != nil { return nil, err @@ -172,10 +173,10 @@ func ListBackups(logger logrus.FieldLogger, objectStore ObjectStore, bucket stri } //GetBackupFunc is a function that can retrieve backup metadata from an object store -type GetBackupFunc func(objectStore ObjectStore, bucket, backupName string) (*api.Backup, error) +type GetBackupFunc func(objectStore cloudprovider.ObjectStore, bucket, backupName string) (*api.Backup, error) // GetBackup gets the specified api.Backup from the given bucket in object storage. -func GetBackup(objectStore ObjectStore, bucket, backupName string) (*api.Backup, error) { +func GetBackup(objectStore cloudprovider.ObjectStore, bucket, backupName string) (*api.Backup, error) { key := getMetadataKey(backupName) res, err := objectStore.GetObject(bucket, key) @@ -204,10 +205,10 @@ func GetBackup(objectStore ObjectStore, bucket, backupName string) (*api.Backup, } // DeleteBackupDirFunc is a function that can delete a backup directory from a bucket in object storage. -type DeleteBackupDirFunc func(logger logrus.FieldLogger, objectStore ObjectStore, bucket, backupName string) error +type DeleteBackupDirFunc func(logger logrus.FieldLogger, objectStore cloudprovider.ObjectStore, bucket, backupName string) error // DeleteBackupDir deletes all files in object storage for the given backup. -func DeleteBackupDir(logger logrus.FieldLogger, objectStore ObjectStore, bucket, backupName string) error { +func DeleteBackupDir(logger logrus.FieldLogger, objectStore cloudprovider.ObjectStore, bucket, backupName string) error { objects, err := objectStore.ListObjects(bucket, backupName+"/") if err != nil { return err @@ -228,11 +229,11 @@ func DeleteBackupDir(logger logrus.FieldLogger, objectStore ObjectStore, bucket, } // CreateSignedURLFunc is a function that can create a signed URL for an object in object storage. -type CreateSignedURLFunc func(objectStore ObjectStore, target api.DownloadTarget, bucket, directory string, ttl time.Duration) (string, error) +type CreateSignedURLFunc func(objectStore cloudprovider.ObjectStore, target api.DownloadTarget, bucket, directory string, ttl time.Duration) (string, error) // CreateSignedURL creates a pre-signed URL that can be used to download a file from object // storage. The URL expires after ttl. -func CreateSignedURL(objectStore ObjectStore, target api.DownloadTarget, bucket, directory string, ttl time.Duration) (string, error) { +func CreateSignedURL(objectStore cloudprovider.ObjectStore, target api.DownloadTarget, bucket, directory string, ttl time.Duration) (string, error) { switch target.Kind { case api.DownloadTargetKindBackupContents: return objectStore.CreateSignedURL(bucket, getBackupContentsKey(directory, target.Name), ttl) @@ -248,19 +249,19 @@ func CreateSignedURL(objectStore ObjectStore, target api.DownloadTarget, bucket, } // UploadRestoreLogFunc is a function that can upload a restore log to a bucket in object storage. -type UploadRestoreLogFunc func(objectStore ObjectStore, bucket, backup, restore string, log io.Reader) error +type UploadRestoreLogFunc func(objectStore cloudprovider.ObjectStore, bucket, backup, restore string, log io.Reader) error // UploadRestoreLog uploads the restore's log file to object storage. -func UploadRestoreLog(objectStore ObjectStore, bucket, backup, restore string, log io.Reader) error { +func UploadRestoreLog(objectStore cloudprovider.ObjectStore, bucket, backup, restore string, log io.Reader) error { key := getRestoreLogKey(backup, restore) return objectStore.PutObject(bucket, key, log) } // UploadRestoreResultsFunc is a function that can upload restore results to a bucket in object storage. -type UploadRestoreResultsFunc func(objectStore ObjectStore, bucket, backup, restore string, results io.Reader) error +type UploadRestoreResultsFunc func(objectStore cloudprovider.ObjectStore, bucket, backup, restore string, results io.Reader) error // UploadRestoreResults uploads the restore's results file to object storage. -func UploadRestoreResults(objectStore ObjectStore, bucket, backup, restore string, results io.Reader) error { +func UploadRestoreResults(objectStore cloudprovider.ObjectStore, bucket, backup, restore string, results io.Reader) error { key := getRestoreResultsKey(backup, restore) return objectStore.PutObject(bucket, key, results) } diff --git a/pkg/cloudprovider/backup_service_test.go b/pkg/persistence/object_storage_persistence_test.go similarity index 99% rename from pkg/cloudprovider/backup_service_test.go rename to pkg/persistence/object_storage_persistence_test.go index 17bd8adc3..fc59cc3bd 100644 --- a/pkg/cloudprovider/backup_service_test.go +++ b/pkg/persistence/object_storage_persistence_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package cloudprovider +package persistence import ( "bytes"