mirrors/velero
1
0
Fork 0
mirror of https://github.com/vmware-tanzu/velero.git synced 2026-08-17 12:46:05 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add per-restore logs

Browse Source 
Signed-off-by: Andy Goldstein <andy.goldstein@gmail.com>
This commit is contained in:
Andy Goldstein
2017-09-13 09:51:40 -04:00
parent d0b7880881
commit 273b563c86
13 changed files with 481 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/controller/download_request_controller.go
+15 -24
View File
@@ -46,8 +46,6 @@ type downloadRequestController struct {
downloadRequestClient arkv1client.DownloadRequestsGetter
downloadRequestLister listers.DownloadRequestLister
downloadRequestListerSynced cache.InformerSynced
backupLister listers.BackupLister
backupListerSynced cache.InformerSynced
backupService cloudprovider.BackupService
bucket string
@@ -62,7 +60,6 @@ type downloadRequestController struct {
func NewDownloadRequestController(
downloadRequestClient arkv1client.DownloadRequestsGetter,
downloadRequestInformer informers.DownloadRequestInformer,
backupInformer informers.BackupInformer,
backupService cloudprovider.BackupService,
bucket string,
) Interface {
@@ -70,13 +67,11 @@ func NewDownloadRequestController(
downloadRequestClient: downloadRequestClient,
downloadRequestLister: downloadRequestInformer.Lister(),
downloadRequestListerSynced: downloadRequestInformer.Informer().HasSynced,
backupLister: backupInformer.Lister(),
backupListerSynced: backupInformer.Informer().HasSynced,
backupService: backupService,
bucket: bucket,
queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "backup"),
queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "downloadrequest"),
clock: &clock.RealClock{},
}
@@ -122,7 +117,7 @@ func (c *downloadRequestController) Run(ctx context.Context, numWorkers int) err
defer glog.Infof("Shutting down DownloadRequestController")
glog.Info("Waiting for caches to sync")
if !cache.WaitForCacheSync(ctx.Done(), c.downloadRequestListerSynced, c.backupListerSynced) {
if !cache.WaitForCacheSync(ctx.Done(), c.downloadRequestListerSynced) {
return errors.New("timed out waiting for caches to sync")
}
glog.Info("Caches are synced")
@@ -215,25 +210,21 @@ const signedURLTTL = 10 * time.Minute
// generatePreSignedURL generates a pre-signed URL for downloadRequest, changes the phase to
// Processed, and persists the changes to storage.
func (c *downloadRequestController) generatePreSignedURL(downloadRequest *v1.DownloadRequest) error {
switch downloadRequest.Spec.Target.Kind {
case v1.DownloadTargetKindBackupLog, v1.DownloadTargetKindBackupContents:
update, err := cloneDownloadRequest(downloadRequest)
if err != nil {
return err
}
update.Status.DownloadURL, err = c.backupService.CreateBackupSignedURL(downloadRequest.Spec.Target.Kind, c.bucket, update.Spec.Target.Name, signedURLTTL)
if err != nil {
return err
}
update.Status.Phase = v1.DownloadRequestPhaseProcessed
update.Status.Expiration = metav1.NewTime(c.clock.Now().Add(signedURLTTL))
_, err = c.downloadRequestClient.DownloadRequests(update.Namespace).Update(update)
update, err := cloneDownloadRequest(downloadRequest)
if err != nil {
return err
}
return fmt.Errorf("unsupported download target kind %q", downloadRequest.Spec.Target.Kind)
update.Status.DownloadURL, err = c.backupService.CreateSignedURL(downloadRequest.Spec.Target, c.bucket, signedURLTTL)
if err != nil {
return err
}
update.Status.Phase = v1.DownloadRequestPhaseProcessed
update.Status.Expiration = metav1.NewTime(c.clock.Now().Add(signedURLTTL))
_, err = c.downloadRequestClient.DownloadRequests(update.Namespace).Update(update)
return err
}
pkg/controller/download_request_controller_test.go
+161
View File
@@ -0,0 +1,161 @@
/*
Copyright 2017 Heptio Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controller
import (
"testing"
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
core "k8s.io/client-go/testing"
"github.com/heptio/ark/pkg/apis/ark/v1"
"github.com/heptio/ark/pkg/generated/clientset/fake"
informers "github.com/heptio/ark/pkg/generated/informers/externalversions"
"github.com/heptio/ark/pkg/util/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestProcessDownloadRequest(t *testing.T) {
tests := []struct {
name string
key string
phase v1.DownloadRequestPhase
targetKind v1.DownloadTargetKind
targetName string
expectedError string
expectedPhase v1.DownloadRequestPhase
expectedURL string
}{
{
name: "empty key",
key: "",
},
{
name: "bad key format",
key: "a/b/c",
expectedError: `unexpected key format: "a/b/c"`,
},
{
name: "backup log request with phase '' gets a url",
key: "heptio-ark/dr1",
phase: "",
targetKind: v1.DownloadTargetKindBackupLog,
targetName: "backup1",
expectedPhase: v1.DownloadRequestPhaseProcessed,
expectedURL: "signedURL",
},
{
name: "backup log request with phase 'New' gets a url",
key: "heptio-ark/dr1",
phase: v1.DownloadRequestPhaseNew,
targetKind: v1.DownloadTargetKindBackupLog,
targetName: "backup1",
expectedPhase: v1.DownloadRequestPhaseProcessed,
expectedURL: "signedURL",
},
{
name: "restore log request with phase '' gets a url",
key: "heptio-ark/dr1",
phase: "",
targetKind: v1.DownloadTargetKindRestoreLog,
targetName: "backup1-20170912150214",
expectedPhase: v1.DownloadRequestPhaseProcessed,
expectedURL: "signedURL",
},
{
name: "restore log request with phase New gets a url",
key: "heptio-ark/dr1",
phase: v1.DownloadRequestPhaseNew,
targetKind: v1.DownloadTargetKindRestoreLog,
targetName: "backup1-20170912150214",
expectedPhase: v1.DownloadRequestPhaseProcessed,
expectedURL: "signedURL",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
client := fake.NewSimpleClientset()
sharedInformers := informers.NewSharedInformerFactory(client, 0)
downloadRequestsInformer := sharedInformers.Ark().V1().DownloadRequests()
backupService := &test.BackupService{}
defer backupService.AssertExpectations(t)
c := NewDownloadRequestController(
client.ArkV1(),
downloadRequestsInformer,
backupService,
"bucket",
).(*downloadRequestController)
if tc.expectedPhase == v1.DownloadRequestPhaseProcessed {
target := v1.DownloadTarget{
Kind: tc.targetKind,
Name: tc.targetName,
}
downloadRequestsInformer.Informer().GetStore().Add(
&v1.DownloadRequest{
ObjectMeta: metav1.ObjectMeta{
Namespace: v1.DefaultNamespace,
Name: "dr1",
},
Spec: v1.DownloadRequestSpec{
Target: target,
},
},
)
backupService.On("CreateSignedURL", target, "bucket", 10*time.Minute).Return("signedURL", nil)
}
var updatedRequest *v1.DownloadRequest
client.PrependReactor("update", "downloadrequests", func(action core.Action) (bool, runtime.Object, error) {
obj := action.(core.UpdateAction).GetObject()
r, ok := obj.(*v1.DownloadRequest)
require.True(t, ok)
updatedRequest = r
return true, obj, nil
})
// method under test
err := c.processDownloadRequest(tc.key)
if tc.expectedError != "" {
assert.EqualError(t, err, tc.expectedError)
return
}
require.NoError(t, err)
var (
updatedPhase v1.DownloadRequestPhase
updatedURL string
)
if updatedRequest != nil {
updatedPhase = updatedRequest.Status.Phase
updatedURL = updatedRequest.Status.DownloadURL
}
assert.Equal(t, tc.expectedPhase, updatedPhase)
assert.Equal(t, tc.expectedURL, updatedURL)
})
}
}
pkg/controller/restore_controller.go
+34 -5
View File
@@ -333,28 +333,57 @@ func (controller *restoreController) runRestore(restore *api.Restore, bucket str
backup, err := controller.fetchBackup(bucket, restore.Spec.BackupName)
if err != nil {
glog.Errorf("error getting backup: %v", err)
errors.Cluster = append(errors.Ark, err.Error())
errors.Ark = append(errors.Ark, err.Error())
return
}
tmpFile, err := downloadToTempFile(restore.Spec.BackupName, controller.backupService, bucket)
if err != nil {
glog.Errorf("error downloading backup: %v", err)
errors.Cluster = append(errors.Ark, err.Error())
errors.Ark = append(errors.Ark, err.Error())
return
}
logFile, err := ioutil.TempFile("", "")
if err != nil {
glog.Errorf("error creating log temp file: %v", err)
errors.Ark = append(errors.Ark, err.Error())
return
}
defer func() {
if err := tmpFile.Close(); err != nil {
errors.Cluster = append(errors.Ark, err.Error())
glog.Errorf("error closing %q: %v", tmpFile.Name(), err)
}
if err := os.Remove(tmpFile.Name()); err != nil {
errors.Cluster = append(errors.Ark, err.Error())
glog.Errorf("error removing %q: %v", tmpFile.Name(), err)
}
if err := logFile.Close(); err != nil {
glog.Errorf("error closing %q: %v", logFile.Name(), err)
}
if err := os.Remove(logFile.Name()); err != nil {
glog.Errorf("error removing %q: %v", logFile.Name(), err)
}
}()
return controller.restorer.Restore(restore, backup, tmpFile)
warnings, errors = controller.restorer.Restore(restore, backup, tmpFile, logFile)
// Try to upload the log file. This is best-effort. If we fail, we'll add to the ark errors.
// Reset the offset to 0 for reading
if _, err = logFile.Seek(0, 0); err != nil {
errors.Ark = append(errors.Ark, fmt.Sprintf("error resetting log file offset to 0: %v", err))
return
}
if err := controller.backupService.UploadRestoreLog(bucket, restore.Spec.BackupName, restore.Name, logFile); err != nil {
errors.Ark = append(errors.Ark, fmt.Sprintf("error uploading log file to object storage: %v", err))
}
return
}
func downloadToTempFile(backupName string, backupService cloudprovider.BackupService, bucket string) (*os.File, error) {
pkg/controller/restore_controller_test.go
+18 -12
View File
@@ -19,7 +19,6 @@ package controller
import (
"bytes"
"errors"
"fmt"
"io"
"io/ioutil"
"testing"
@@ -119,6 +118,7 @@ func TestProcessRestore(t *testing.T) {
expectedRestoreUpdates []*api.Restore
expectedRestorerCall *api.Restore
backupServiceGetBackupError error
uploadLogError error
}{
{
name: "invalid key returns error",
@@ -187,7 +187,7 @@ func TestProcessRestore(t *testing.T) {
NewRestore("foo", "bar", "backup-1", "ns-1", "*", api.RestorePhaseInProgress).Restore,
NewRestore("foo", "bar", "backup-1", "ns-1", "*", api.RestorePhaseCompleted).
WithErrors(api.RestoreResult{
Cluster: []string{"no backup here"},
Ark: []string{"no backup here"},
}).
Restore,
},
@@ -260,13 +260,15 @@ func TestProcessRestore(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
fmt.Println(test.name)
var (
client = fake.NewSimpleClientset()
restorer = &fakeRestorer{}
sharedInformers = informers.NewSharedInformerFactory(client, 0)
backupSvc = &BackupService{}
)
client := fake.NewSimpleClientset()
restorer := &fakeRestorer{}
defer restorer.AssertExpectations(t)
sharedInformers := informers.NewSharedInformerFactory(client, 0)
backupSvc := &BackupService{}
defer backupSvc.AssertExpectations(t)
c := NewRestoreController(
sharedInformers.Ark().V1().Restores(),
@@ -303,10 +305,14 @@ func TestProcessRestore(t *testing.T) {
if test.restorerError != nil {
errors.Namespaces = map[string][]string{"ns-1": {test.restorerError.Error()}}
}
if test.uploadLogError != nil {
errors.Ark = append(errors.Ark, "error uploading log file to object storage: "+test.uploadLogError.Error())
}
if test.expectedRestorerCall != nil {
downloadedBackup := ioutil.NopCloser(bytes.NewReader([]byte("hello world")))
backupSvc.On("DownloadBackup", mock.Anything, mock.Anything).Return(downloadedBackup, nil)
restorer.On("Restore", mock.Anything, mock.Anything, mock.Anything).Return(warnings, errors)
restorer.On("Restore", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(warnings, errors)
backupSvc.On("UploadRestoreLog", "bucket", test.restore.Spec.BackupName, test.restore.Name, mock.Anything).Return(test.uploadLogError)
}
var (
@@ -379,8 +385,8 @@ type fakeRestorer struct {
calledWithArg api.Restore
}
func (r *fakeRestorer) Restore(restore *api.Restore, backup *api.Backup, backupReader io.Reader) (api.RestoreResult, api.RestoreResult) {
res := r.Called(restore, backup, backupReader)
func (r *fakeRestorer) Restore(restore *api.Restore, backup *api.Backup, backupReader io.Reader, logger io.Writer) (api.RestoreResult, api.RestoreResult) {
res := r.Called(restore, backup, backupReader, logger)
r.calledWithArg = *restore