mirror of
https://github.com/vmware-tanzu/velero.git
synced 2026-01-08 14:21:18 +00:00
74f60b1ee1
We ran into a lot of problems using a finalizer on the backup to allow the Ark server to clean up all associated backup data when deleting a backup. Users also found it less than desirable that deleting the heptio-ark namespace resulted in all the backup data being deleted. This removes the finalizer and replaces it with an explicit DeleteBackupRequest that is created as a means of requesting the deletion of a backup and all its associated data. This is what `ark backup delete` does. If you use kubectl to delete a backup or to delete the heptio-ark namespace, this no longer deletes associated backups. Additionally, as long as the heptio-ark namespace still exists, the Ark server's BackupSyncController will continually sync backups into the heptio-ark namespace from object storage. Signed-off-by: Andy Goldstein <andy.goldstein@gmail.com>
392 lines
12 KiB
Go
392 lines
12 KiB
Go
/*
|
|
Copyright 2018 the Heptio Ark contributors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package controller
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"reflect"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/heptio/ark/pkg/apis/ark/v1"
|
|
pkgbackup "github.com/heptio/ark/pkg/backup"
|
|
"github.com/heptio/ark/pkg/generated/clientset/versioned/fake"
|
|
informers "github.com/heptio/ark/pkg/generated/informers/externalversions"
|
|
"github.com/heptio/ark/pkg/util/kube"
|
|
arktest "github.com/heptio/ark/pkg/util/test"
|
|
"github.com/pkg/errors"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
"k8s.io/apimachinery/pkg/util/sets"
|
|
"k8s.io/apimachinery/pkg/watch"
|
|
core "k8s.io/client-go/testing"
|
|
)
|
|
|
|
func TestBackupDeletionControllerControllerHasUpdateFunc(t *testing.T) {
|
|
req := pkgbackup.NewDeleteBackupRequest("foo")
|
|
req.Namespace = "heptio-ark"
|
|
expected := kube.NamespaceAndName(req)
|
|
|
|
client := fake.NewSimpleClientset(req)
|
|
|
|
fakeWatch := watch.NewFake()
|
|
defer fakeWatch.Stop()
|
|
client.PrependWatchReactor("deletebackuprequests", core.DefaultWatchReactor(fakeWatch, nil))
|
|
|
|
sharedInformers := informers.NewSharedInformerFactory(client, 0)
|
|
|
|
controller := NewBackupDeletionController(
|
|
arktest.NewLogger(),
|
|
sharedInformers.Ark().V1().DeleteBackupRequests(),
|
|
client.ArkV1(), // deleteBackupRequestClient
|
|
client.ArkV1(), // backupClient
|
|
nil, // snapshotService
|
|
nil, // backupService
|
|
"bucket",
|
|
sharedInformers.Ark().V1().Restores(),
|
|
client.ArkV1(), // restoreClient
|
|
).(*backupDeletionController)
|
|
|
|
keys := make(chan string)
|
|
|
|
controller.syncHandler = func(key string) error {
|
|
keys <- key
|
|
return nil
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
|
|
go sharedInformers.Start(ctx.Done())
|
|
go controller.Run(ctx, 1)
|
|
|
|
// wait for the AddFunc
|
|
select {
|
|
case <-ctx.Done():
|
|
t.Fatal("test timed out waiting for AddFunc")
|
|
case key := <-keys:
|
|
assert.Equal(t, expected, key)
|
|
}
|
|
|
|
req.Status.Phase = v1.DeleteBackupRequestPhaseProcessed
|
|
fakeWatch.Add(req)
|
|
|
|
// wait for the UpdateFunc
|
|
select {
|
|
case <-ctx.Done():
|
|
t.Fatal("test timed out waiting for UpdateFunc")
|
|
case key := <-keys:
|
|
assert.Equal(t, expected, key)
|
|
}
|
|
}
|
|
|
|
func TestBackupDeletionControllerProcessQueueItem(t *testing.T) {
|
|
client := fake.NewSimpleClientset()
|
|
sharedInformers := informers.NewSharedInformerFactory(client, 0)
|
|
|
|
controller := NewBackupDeletionController(
|
|
arktest.NewLogger(),
|
|
sharedInformers.Ark().V1().DeleteBackupRequests(),
|
|
client.ArkV1(), // deleteBackupRequestClient
|
|
client.ArkV1(), // backupClient
|
|
nil, // snapshotService
|
|
nil, // backupService
|
|
"bucket",
|
|
sharedInformers.Ark().V1().Restores(),
|
|
client.ArkV1(), // restoreClient
|
|
).(*backupDeletionController)
|
|
|
|
// Error splitting key
|
|
err := controller.processQueueItem("foo/bar/baz")
|
|
assert.Error(t, err)
|
|
|
|
// Can't find DeleteBackupRequest
|
|
err = controller.processQueueItem("foo/bar")
|
|
assert.NoError(t, err)
|
|
|
|
// Already processed
|
|
req := pkgbackup.NewDeleteBackupRequest("foo")
|
|
req.Namespace = "foo"
|
|
req.Name = "foo-abcde"
|
|
req.Status.Phase = v1.DeleteBackupRequestPhaseProcessed
|
|
|
|
err = controller.processQueueItem("foo/bar")
|
|
assert.NoError(t, err)
|
|
|
|
// Invoke processRequestFunc
|
|
for _, phase := range []v1.DeleteBackupRequestPhase{"", v1.DeleteBackupRequestPhaseNew, v1.DeleteBackupRequestPhaseInProgress} {
|
|
t.Run(fmt.Sprintf("phase=%s", phase), func(t *testing.T) {
|
|
req.Status.Phase = phase
|
|
sharedInformers.Ark().V1().DeleteBackupRequests().Informer().GetStore().Add(req)
|
|
|
|
var errorToReturn error
|
|
var actual *v1.DeleteBackupRequest
|
|
var called bool
|
|
controller.processRequestFunc = func(r *v1.DeleteBackupRequest) error {
|
|
called = true
|
|
actual = r
|
|
return errorToReturn
|
|
}
|
|
|
|
// No error
|
|
err = controller.processQueueItem("foo/foo-abcde")
|
|
require.True(t, called, "processRequestFunc wasn't called")
|
|
assert.Equal(t, err, errorToReturn)
|
|
assert.Equal(t, req, actual)
|
|
|
|
// Error
|
|
errorToReturn = errors.New("bar")
|
|
err = controller.processQueueItem("foo/foo-abcde")
|
|
require.True(t, called, "processRequestFunc wasn't called")
|
|
assert.Equal(t, err, errorToReturn)
|
|
})
|
|
}
|
|
}
|
|
|
|
type backupDeletionControllerTestData struct {
|
|
client *fake.Clientset
|
|
sharedInformers informers.SharedInformerFactory
|
|
backupService *arktest.BackupService
|
|
snapshotService *arktest.FakeSnapshotService
|
|
controller *backupDeletionController
|
|
req *v1.DeleteBackupRequest
|
|
}
|
|
|
|
func setupBackupDeletionControllerTest(objects ...runtime.Object) *backupDeletionControllerTestData {
|
|
client := fake.NewSimpleClientset(objects...)
|
|
sharedInformers := informers.NewSharedInformerFactory(client, 0)
|
|
backupService := &arktest.BackupService{}
|
|
snapshotService := &arktest.FakeSnapshotService{SnapshotsTaken: sets.NewString()}
|
|
req := pkgbackup.NewDeleteBackupRequest("foo")
|
|
|
|
data := &backupDeletionControllerTestData{
|
|
client: client,
|
|
sharedInformers: sharedInformers,
|
|
backupService: backupService,
|
|
snapshotService: snapshotService,
|
|
controller: NewBackupDeletionController(
|
|
arktest.NewLogger(),
|
|
sharedInformers.Ark().V1().DeleteBackupRequests(),
|
|
client.ArkV1(), // deleteBackupRequestClient
|
|
client.ArkV1(), // backupClient
|
|
snapshotService,
|
|
backupService,
|
|
"bucket",
|
|
sharedInformers.Ark().V1().Restores(),
|
|
client.ArkV1(), // restoreClient
|
|
).(*backupDeletionController),
|
|
|
|
req: req,
|
|
}
|
|
req.Namespace = "heptio-ark"
|
|
req.Name = "foo-abcde"
|
|
|
|
return data
|
|
}
|
|
|
|
func TestBackupDeletionControllerProcessRequest(t *testing.T) {
|
|
t.Run("patching to InProgress fails", func(t *testing.T) {
|
|
td := setupBackupDeletionControllerTest()
|
|
defer td.backupService.AssertExpectations(t)
|
|
|
|
td.client.PrependReactor("patch", "deletebackuprequests", func(action core.Action) (bool, runtime.Object, error) {
|
|
return true, nil, errors.New("bad")
|
|
})
|
|
|
|
err := td.controller.processRequest(td.req)
|
|
assert.EqualError(t, err, "error patching DeleteBackupRequest: bad")
|
|
})
|
|
|
|
t.Run("unable to find backup", func(t *testing.T) {
|
|
td := setupBackupDeletionControllerTest()
|
|
defer td.backupService.AssertExpectations(t)
|
|
|
|
td.client.PrependReactor("get", "backups", func(action core.Action) (bool, runtime.Object, error) {
|
|
return true, nil, apierrors.NewNotFound(v1.SchemeGroupVersion.WithResource("backups").GroupResource(), "foo")
|
|
})
|
|
|
|
td.client.PrependReactor("patch", "deletebackuprequests", func(action core.Action) (bool, runtime.Object, error) {
|
|
return true, td.req, nil
|
|
})
|
|
|
|
err := td.controller.processRequest(td.req)
|
|
require.NoError(t, err)
|
|
|
|
expectedActions := []core.Action{
|
|
core.NewPatchAction(
|
|
v1.SchemeGroupVersion.WithResource("deletebackuprequests"),
|
|
td.req.Namespace,
|
|
td.req.Name,
|
|
[]byte(`{"status":{"phase":"InProgress"}}`),
|
|
),
|
|
core.NewGetAction(
|
|
v1.SchemeGroupVersion.WithResource("backups"),
|
|
td.req.Namespace,
|
|
td.req.Spec.BackupName,
|
|
),
|
|
core.NewPatchAction(
|
|
v1.SchemeGroupVersion.WithResource("deletebackuprequests"),
|
|
td.req.Namespace,
|
|
td.req.Name,
|
|
[]byte(`{"status":{"errors":["backup not found"],"phase":"Processed"}}`),
|
|
),
|
|
}
|
|
|
|
assert.Equal(t, expectedActions, td.client.Actions())
|
|
})
|
|
|
|
t.Run("no snapshot service, backup has snapshots", func(t *testing.T) {
|
|
td := setupBackupDeletionControllerTest()
|
|
td.controller.snapshotService = nil
|
|
defer td.backupService.AssertExpectations(t)
|
|
|
|
td.client.PrependReactor("get", "backups", func(action core.Action) (bool, runtime.Object, error) {
|
|
backup := arktest.NewTestBackup().WithName("backup-1").WithSnapshot("pv-1", "snap-1").Backup
|
|
return true, backup, nil
|
|
})
|
|
|
|
td.client.PrependReactor("patch", "deletebackuprequests", func(action core.Action) (bool, runtime.Object, error) {
|
|
return true, td.req, nil
|
|
})
|
|
|
|
err := td.controller.processRequest(td.req)
|
|
require.NoError(t, err)
|
|
|
|
expectedActions := []core.Action{
|
|
core.NewPatchAction(
|
|
v1.SchemeGroupVersion.WithResource("deletebackuprequests"),
|
|
td.req.Namespace,
|
|
td.req.Name,
|
|
[]byte(`{"status":{"phase":"InProgress"}}`),
|
|
),
|
|
core.NewGetAction(
|
|
v1.SchemeGroupVersion.WithResource("backups"),
|
|
td.req.Namespace,
|
|
td.req.Spec.BackupName,
|
|
),
|
|
core.NewPatchAction(
|
|
v1.SchemeGroupVersion.WithResource("deletebackuprequests"),
|
|
td.req.Namespace,
|
|
td.req.Name,
|
|
[]byte(`{"status":{"errors":["unable to delete backup because it includes PV snapshots and Ark is not configured with a PersistentVolumeProvider"],"phase":"Processed"}}`),
|
|
),
|
|
}
|
|
|
|
assert.Equal(t, expectedActions, td.client.Actions())
|
|
})
|
|
|
|
t.Run("full delete, no errors", func(t *testing.T) {
|
|
backup := arktest.NewTestBackup().WithName("foo").WithSnapshot("pv-1", "snap-1").Backup
|
|
|
|
restore1 := arktest.NewTestRestore("heptio-ark", "restore-1", v1.RestorePhaseCompleted).WithBackup("foo").Restore
|
|
restore2 := arktest.NewTestRestore("heptio-ark", "restore-2", v1.RestorePhaseCompleted).WithBackup("foo").Restore
|
|
restore3 := arktest.NewTestRestore("heptio-ark", "restore-3", v1.RestorePhaseCompleted).WithBackup("some-other-backup").Restore
|
|
|
|
td := setupBackupDeletionControllerTest(backup, restore1, restore2, restore3)
|
|
|
|
td.sharedInformers.Ark().V1().Restores().Informer().GetStore().Add(restore1)
|
|
td.sharedInformers.Ark().V1().Restores().Informer().GetStore().Add(restore2)
|
|
td.sharedInformers.Ark().V1().Restores().Informer().GetStore().Add(restore3)
|
|
|
|
defer td.backupService.AssertExpectations(t)
|
|
|
|
td.client.PrependReactor("get", "backups", func(action core.Action) (bool, runtime.Object, error) {
|
|
return true, backup, nil
|
|
})
|
|
td.snapshotService.SnapshotsTaken.Insert("snap-1")
|
|
|
|
td.client.PrependReactor("patch", "deletebackuprequests", func(action core.Action) (bool, runtime.Object, error) {
|
|
return true, td.req, nil
|
|
})
|
|
|
|
td.client.PrependReactor("patch", "backups", func(action core.Action) (bool, runtime.Object, error) {
|
|
return true, backup, nil
|
|
})
|
|
|
|
td.backupService.On("DeleteBackupDir", td.controller.bucket, td.req.Spec.BackupName).Return(nil)
|
|
|
|
err := td.controller.processRequest(td.req)
|
|
require.NoError(t, err)
|
|
|
|
expectedActions := []core.Action{
|
|
core.NewPatchAction(
|
|
v1.SchemeGroupVersion.WithResource("deletebackuprequests"),
|
|
td.req.Namespace,
|
|
td.req.Name,
|
|
[]byte(`{"status":{"phase":"InProgress"}}`),
|
|
),
|
|
core.NewGetAction(
|
|
v1.SchemeGroupVersion.WithResource("backups"),
|
|
td.req.Namespace,
|
|
td.req.Spec.BackupName,
|
|
),
|
|
core.NewPatchAction(
|
|
v1.SchemeGroupVersion.WithResource("backups"),
|
|
td.req.Namespace,
|
|
td.req.Spec.BackupName,
|
|
[]byte(`{"status":{"phase":"Deleting"}}`),
|
|
),
|
|
core.NewDeleteAction(
|
|
v1.SchemeGroupVersion.WithResource("restores"),
|
|
td.req.Namespace,
|
|
"restore-1",
|
|
),
|
|
core.NewDeleteAction(
|
|
v1.SchemeGroupVersion.WithResource("restores"),
|
|
td.req.Namespace,
|
|
"restore-2",
|
|
),
|
|
core.NewDeleteAction(
|
|
v1.SchemeGroupVersion.WithResource("backups"),
|
|
td.req.Namespace,
|
|
td.req.Spec.BackupName,
|
|
),
|
|
core.NewPatchAction(
|
|
v1.SchemeGroupVersion.WithResource("deletebackuprequests"),
|
|
td.req.Namespace,
|
|
td.req.Name,
|
|
[]byte(`{"status":{"phase":"Processed"}}`),
|
|
),
|
|
core.NewDeleteCollectionAction(
|
|
v1.SchemeGroupVersion.WithResource("deletebackuprequests"),
|
|
td.req.Namespace,
|
|
pkgbackup.NewDeleteBackupRequestListOptions(td.req.Spec.BackupName),
|
|
),
|
|
}
|
|
|
|
assert.Len(t, td.client.Actions(), len(expectedActions))
|
|
for _, e := range expectedActions {
|
|
found := false
|
|
for _, a := range td.client.Actions() {
|
|
if reflect.DeepEqual(e, a) {
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
t.Errorf("missing expected action %#v", e)
|
|
}
|
|
}
|
|
|
|
// Make sure snapshot was deleted
|
|
assert.Equal(t, 0, td.snapshotService.SnapshotsTaken.Len())
|
|
})
|
|
}
|