/* Copyright 2018 the Heptio Ark contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "context" "fmt" "testing" "time" "github.com/heptio/ark/pkg/apis/ark/v1" pkgbackup "github.com/heptio/ark/pkg/backup" "github.com/heptio/ark/pkg/generated/clientset/versioned/fake" informers "github.com/heptio/ark/pkg/generated/informers/externalversions" "github.com/heptio/ark/pkg/util/kube" arktest "github.com/heptio/ark/pkg/util/test" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/clock" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/watch" core "k8s.io/client-go/testing" ) func TestBackupDeletionControllerControllerHasUpdateFunc(t *testing.T) { req := pkgbackup.NewDeleteBackupRequest("foo", "uid") req.Namespace = "heptio-ark" expected := kube.NamespaceAndName(req) client := fake.NewSimpleClientset(req) fakeWatch := watch.NewFake() defer fakeWatch.Stop() client.PrependWatchReactor("deletebackuprequests", core.DefaultWatchReactor(fakeWatch, nil)) sharedInformers := informers.NewSharedInformerFactory(client, 0) controller := NewBackupDeletionController( arktest.NewLogger(), sharedInformers.Ark().V1().DeleteBackupRequests(), client.ArkV1(), // deleteBackupRequestClient client.ArkV1(), // backupClient nil, // snapshotService nil, // backupService "bucket", sharedInformers.Ark().V1().Restores(), client.ArkV1(), // restoreClient NewBackupTracker(), nil, // restic repository manager sharedInformers.Ark().V1().PodVolumeBackups(), ).(*backupDeletionController) // disable resync handler since we don't want to test it here controller.resyncFunc = nil keys := make(chan string) controller.syncHandler = func(key string) error { keys <- key return nil } ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() go sharedInformers.Start(ctx.Done()) go controller.Run(ctx, 1) // wait for the AddFunc select { case <-ctx.Done(): t.Fatal("test timed out waiting for AddFunc") case key := <-keys: assert.Equal(t, expected, key) } req.Status.Phase = v1.DeleteBackupRequestPhaseProcessed fakeWatch.Add(req) // wait for the UpdateFunc select { case <-ctx.Done(): t.Fatal("test timed out waiting for UpdateFunc") case key := <-keys: assert.Equal(t, expected, key) } } func TestBackupDeletionControllerProcessQueueItem(t *testing.T) { client := fake.NewSimpleClientset() sharedInformers := informers.NewSharedInformerFactory(client, 0) controller := NewBackupDeletionController( arktest.NewLogger(), sharedInformers.Ark().V1().DeleteBackupRequests(), client.ArkV1(), // deleteBackupRequestClient client.ArkV1(), // backupClient nil, // snapshotService nil, // backupService "bucket", sharedInformers.Ark().V1().Restores(), client.ArkV1(), // restoreClient NewBackupTracker(), nil, // restic repository manager sharedInformers.Ark().V1().PodVolumeBackups(), ).(*backupDeletionController) // Error splitting key err := controller.processQueueItem("foo/bar/baz") assert.Error(t, err) // Can't find DeleteBackupRequest err = controller.processQueueItem("foo/bar") assert.NoError(t, err) // Already processed req := pkgbackup.NewDeleteBackupRequest("foo", "uid") req.Namespace = "foo" req.Name = "foo-abcde" req.Status.Phase = v1.DeleteBackupRequestPhaseProcessed err = controller.processQueueItem("foo/bar") assert.NoError(t, err) // Invoke processRequestFunc for _, phase := range []v1.DeleteBackupRequestPhase{"", v1.DeleteBackupRequestPhaseNew, v1.DeleteBackupRequestPhaseInProgress} { t.Run(fmt.Sprintf("phase=%s", phase), func(t *testing.T) { req.Status.Phase = phase sharedInformers.Ark().V1().DeleteBackupRequests().Informer().GetStore().Add(req) var errorToReturn error var actual *v1.DeleteBackupRequest var called bool controller.processRequestFunc = func(r *v1.DeleteBackupRequest) error { called = true actual = r return errorToReturn } // No error err = controller.processQueueItem("foo/foo-abcde") require.True(t, called, "processRequestFunc wasn't called") assert.Equal(t, err, errorToReturn) assert.Equal(t, req, actual) // Error errorToReturn = errors.New("bar") err = controller.processQueueItem("foo/foo-abcde") require.True(t, called, "processRequestFunc wasn't called") assert.Equal(t, err, errorToReturn) }) } } type backupDeletionControllerTestData struct { client *fake.Clientset sharedInformers informers.SharedInformerFactory backupService *arktest.BackupService snapshotService *arktest.FakeSnapshotService controller *backupDeletionController req *v1.DeleteBackupRequest } func setupBackupDeletionControllerTest(objects ...runtime.Object) *backupDeletionControllerTestData { client := fake.NewSimpleClientset(objects...) sharedInformers := informers.NewSharedInformerFactory(client, 0) backupService := &arktest.BackupService{} snapshotService := &arktest.FakeSnapshotService{SnapshotsTaken: sets.NewString()} req := pkgbackup.NewDeleteBackupRequest("foo", "uid") data := &backupDeletionControllerTestData{ client: client, sharedInformers: sharedInformers, backupService: backupService, snapshotService: snapshotService, controller: NewBackupDeletionController( arktest.NewLogger(), sharedInformers.Ark().V1().DeleteBackupRequests(), client.ArkV1(), // deleteBackupRequestClient client.ArkV1(), // backupClient snapshotService, backupService, "bucket", sharedInformers.Ark().V1().Restores(), client.ArkV1(), // restoreClient NewBackupTracker(), nil, // restic repository manager sharedInformers.Ark().V1().PodVolumeBackups(), ).(*backupDeletionController), req: req, } req.Namespace = "heptio-ark" req.Name = "foo-abcde" return data } func TestBackupDeletionControllerProcessRequest(t *testing.T) { t.Run("missing spec.backupName", func(t *testing.T) { td := setupBackupDeletionControllerTest() defer td.backupService.AssertExpectations(t) td.req.Spec.BackupName = "" err := td.controller.processRequest(td.req) require.NoError(t, err) expectedActions := []core.Action{ core.NewPatchAction( v1.SchemeGroupVersion.WithResource("deletebackuprequests"), td.req.Namespace, td.req.Name, []byte(`{"status":{"errors":["spec.backupName is required"],"phase":"Processed"}}`), ), } assert.Equal(t, expectedActions, td.client.Actions()) }) t.Run("deleting an in progress backup isn't allowed", func(t *testing.T) { td := setupBackupDeletionControllerTest() defer td.backupService.AssertExpectations(t) td.controller.backupTracker.Add(td.req.Namespace, td.req.Spec.BackupName) err := td.controller.processRequest(td.req) require.NoError(t, err) expectedActions := []core.Action{ core.NewPatchAction( v1.SchemeGroupVersion.WithResource("deletebackuprequests"), td.req.Namespace, td.req.Name, []byte(`{"status":{"errors":["backup is still in progress"],"phase":"Processed"}}`), ), } assert.Equal(t, expectedActions, td.client.Actions()) }) t.Run("patching to InProgress fails", func(t *testing.T) { td := setupBackupDeletionControllerTest() defer td.backupService.AssertExpectations(t) td.client.PrependReactor("patch", "deletebackuprequests", func(action core.Action) (bool, runtime.Object, error) { return true, nil, errors.New("bad") }) err := td.controller.processRequest(td.req) assert.EqualError(t, err, "error patching DeleteBackupRequest: bad") }) t.Run("patching backup to Deleting fails", func(t *testing.T) { backup := arktest.NewTestBackup().WithName("foo").WithSnapshot("pv-1", "snap-1").Backup td := setupBackupDeletionControllerTest(backup) defer td.backupService.AssertExpectations(t) td.client.PrependReactor("patch", "deletebackuprequests", func(action core.Action) (bool, runtime.Object, error) { return true, td.req, nil }) td.client.PrependReactor("patch", "backups", func(action core.Action) (bool, runtime.Object, error) { return true, nil, errors.New("bad") }) err := td.controller.processRequest(td.req) assert.EqualError(t, err, "error patching Backup: bad") }) t.Run("unable to find backup", func(t *testing.T) { td := setupBackupDeletionControllerTest() defer td.backupService.AssertExpectations(t) td.client.PrependReactor("get", "backups", func(action core.Action) (bool, runtime.Object, error) { return true, nil, apierrors.NewNotFound(v1.SchemeGroupVersion.WithResource("backups").GroupResource(), "foo") }) td.client.PrependReactor("patch", "deletebackuprequests", func(action core.Action) (bool, runtime.Object, error) { return true, td.req, nil }) err := td.controller.processRequest(td.req) require.NoError(t, err) expectedActions := []core.Action{ core.NewPatchAction( v1.SchemeGroupVersion.WithResource("deletebackuprequests"), td.req.Namespace, td.req.Name, []byte(`{"status":{"phase":"InProgress"}}`), ), core.NewGetAction( v1.SchemeGroupVersion.WithResource("backups"), td.req.Namespace, td.req.Spec.BackupName, ), core.NewPatchAction( v1.SchemeGroupVersion.WithResource("deletebackuprequests"), td.req.Namespace, td.req.Name, []byte(`{"status":{"errors":["backup not found"],"phase":"Processed"}}`), ), } assert.Equal(t, expectedActions, td.client.Actions()) }) t.Run("no snapshot service, backup has snapshots", func(t *testing.T) { td := setupBackupDeletionControllerTest() td.controller.snapshotService = nil defer td.backupService.AssertExpectations(t) td.client.PrependReactor("get", "backups", func(action core.Action) (bool, runtime.Object, error) { backup := arktest.NewTestBackup().WithName("backup-1").WithSnapshot("pv-1", "snap-1").Backup return true, backup, nil }) td.client.PrependReactor("patch", "deletebackuprequests", func(action core.Action) (bool, runtime.Object, error) { return true, td.req, nil }) err := td.controller.processRequest(td.req) require.NoError(t, err) expectedActions := []core.Action{ core.NewPatchAction( v1.SchemeGroupVersion.WithResource("deletebackuprequests"), td.req.Namespace, td.req.Name, []byte(`{"status":{"phase":"InProgress"}}`), ), core.NewGetAction( v1.SchemeGroupVersion.WithResource("backups"), td.req.Namespace, td.req.Spec.BackupName, ), core.NewPatchAction( v1.SchemeGroupVersion.WithResource("deletebackuprequests"), td.req.Namespace, td.req.Name, []byte(`{"status":{"errors":["unable to delete backup because it includes PV snapshots and Ark is not configured with a PersistentVolumeProvider"],"phase":"Processed"}}`), ), } assert.Equal(t, expectedActions, td.client.Actions()) }) t.Run("full delete, no errors", func(t *testing.T) { backup := arktest.NewTestBackup().WithName("foo").WithSnapshot("pv-1", "snap-1").Backup backup.UID = "uid" restore1 := arktest.NewTestRestore("heptio-ark", "restore-1", v1.RestorePhaseCompleted).WithBackup("foo").Restore restore2 := arktest.NewTestRestore("heptio-ark", "restore-2", v1.RestorePhaseCompleted).WithBackup("foo").Restore restore3 := arktest.NewTestRestore("heptio-ark", "restore-3", v1.RestorePhaseCompleted).WithBackup("some-other-backup").Restore td := setupBackupDeletionControllerTest(backup, restore1, restore2, restore3) td.sharedInformers.Ark().V1().Restores().Informer().GetStore().Add(restore1) td.sharedInformers.Ark().V1().Restores().Informer().GetStore().Add(restore2) td.sharedInformers.Ark().V1().Restores().Informer().GetStore().Add(restore3) defer td.backupService.AssertExpectations(t) // Clear out req labels to make sure the controller adds them td.req.Labels = make(map[string]string) td.client.PrependReactor("get", "backups", func(action core.Action) (bool, runtime.Object, error) { return true, backup, nil }) td.snapshotService.SnapshotsTaken.Insert("snap-1") td.client.PrependReactor("patch", "deletebackuprequests", func(action core.Action) (bool, runtime.Object, error) { return true, td.req, nil }) td.client.PrependReactor("patch", "backups", func(action core.Action) (bool, runtime.Object, error) { return true, backup, nil }) td.backupService.On("DeleteBackupDir", td.controller.bucket, td.req.Spec.BackupName).Return(nil) err := td.controller.processRequest(td.req) require.NoError(t, err) expectedActions := []core.Action{ core.NewPatchAction( v1.SchemeGroupVersion.WithResource("deletebackuprequests"), td.req.Namespace, td.req.Name, []byte(`{"metadata":{"labels":{"ark.heptio.com/backup-name":"foo"}},"status":{"phase":"InProgress"}}`), ), core.NewGetAction( v1.SchemeGroupVersion.WithResource("backups"), td.req.Namespace, td.req.Spec.BackupName, ), core.NewPatchAction( v1.SchemeGroupVersion.WithResource("deletebackuprequests"), td.req.Namespace, td.req.Name, []byte(`{"metadata":{"labels":{"ark.heptio.com/backup-uid":"uid"}}}`), ), core.NewPatchAction( v1.SchemeGroupVersion.WithResource("backups"), td.req.Namespace, td.req.Spec.BackupName, []byte(`{"status":{"phase":"Deleting"}}`), ), core.NewDeleteAction( v1.SchemeGroupVersion.WithResource("restores"), td.req.Namespace, "restore-1", ), core.NewDeleteAction( v1.SchemeGroupVersion.WithResource("restores"), td.req.Namespace, "restore-2", ), core.NewDeleteAction( v1.SchemeGroupVersion.WithResource("backups"), td.req.Namespace, td.req.Spec.BackupName, ), core.NewPatchAction( v1.SchemeGroupVersion.WithResource("deletebackuprequests"), td.req.Namespace, td.req.Name, []byte(`{"status":{"phase":"Processed"}}`), ), core.NewDeleteCollectionAction( v1.SchemeGroupVersion.WithResource("deletebackuprequests"), td.req.Namespace, pkgbackup.NewDeleteBackupRequestListOptions(td.req.Spec.BackupName, "uid"), ), } arktest.CompareActions(t, expectedActions, td.client.Actions()) // Make sure snapshot was deleted assert.Equal(t, 0, td.snapshotService.SnapshotsTaken.Len()) }) } func TestBackupDeletionControllerDeleteExpiredRequests(t *testing.T) { now := time.Date(2018, 4, 4, 12, 0, 0, 0, time.UTC) unexpired1 := time.Date(2018, 4, 4, 11, 0, 0, 0, time.UTC) unexpired2 := time.Date(2018, 4, 3, 12, 0, 1, 0, time.UTC) expired1 := time.Date(2018, 4, 3, 12, 0, 0, 0, time.UTC) expired2 := time.Date(2018, 4, 3, 2, 0, 0, 0, time.UTC) tests := []struct { name string requests []*v1.DeleteBackupRequest expectedDeletions []string }{ { name: "no requests", }, { name: "older than max age, phase = '', don't delete", requests: []*v1.DeleteBackupRequest{ { ObjectMeta: metav1.ObjectMeta{ Namespace: "ns", Name: "name", CreationTimestamp: metav1.Time{Time: expired1}, }, Status: v1.DeleteBackupRequestStatus{ Phase: "", }, }, }, }, { name: "older than max age, phase = New, don't delete", requests: []*v1.DeleteBackupRequest{ { ObjectMeta: metav1.ObjectMeta{ Namespace: "ns", Name: "name", CreationTimestamp: metav1.Time{Time: expired1}, }, Status: v1.DeleteBackupRequestStatus{ Phase: v1.DeleteBackupRequestPhaseNew, }, }, }, }, { name: "older than max age, phase = InProcess, don't delete", requests: []*v1.DeleteBackupRequest{ { ObjectMeta: metav1.ObjectMeta{ Namespace: "ns", Name: "name", CreationTimestamp: metav1.Time{Time: expired1}, }, Status: v1.DeleteBackupRequestStatus{ Phase: v1.DeleteBackupRequestPhaseInProgress, }, }, }, }, { name: "some expired, some not", requests: []*v1.DeleteBackupRequest{ { ObjectMeta: metav1.ObjectMeta{ Namespace: "ns", Name: "unexpired-1", CreationTimestamp: metav1.Time{Time: unexpired1}, }, Status: v1.DeleteBackupRequestStatus{ Phase: v1.DeleteBackupRequestPhaseProcessed, }, }, { ObjectMeta: metav1.ObjectMeta{ Namespace: "ns", Name: "expired-1", CreationTimestamp: metav1.Time{Time: expired1}, }, Status: v1.DeleteBackupRequestStatus{ Phase: v1.DeleteBackupRequestPhaseProcessed, }, }, { ObjectMeta: metav1.ObjectMeta{ Namespace: "ns", Name: "unexpired-2", CreationTimestamp: metav1.Time{Time: unexpired2}, }, Status: v1.DeleteBackupRequestStatus{ Phase: v1.DeleteBackupRequestPhaseProcessed, }, }, { ObjectMeta: metav1.ObjectMeta{ Namespace: "ns", Name: "expired-2", CreationTimestamp: metav1.Time{Time: expired2}, }, Status: v1.DeleteBackupRequestStatus{ Phase: v1.DeleteBackupRequestPhaseProcessed, }, }, }, expectedDeletions: []string{"expired-1", "expired-2"}, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { client := fake.NewSimpleClientset() sharedInformers := informers.NewSharedInformerFactory(client, 0) controller := NewBackupDeletionController( arktest.NewLogger(), sharedInformers.Ark().V1().DeleteBackupRequests(), client.ArkV1(), // deleteBackupRequestClient client.ArkV1(), // backupClient nil, // snapshotService nil, // backupService "bucket", sharedInformers.Ark().V1().Restores(), client.ArkV1(), // restoreClient NewBackupTracker(), nil, sharedInformers.Ark().V1().PodVolumeBackups(), ).(*backupDeletionController) fakeClock := &clock.FakeClock{} fakeClock.SetTime(now) controller.clock = fakeClock for i := range test.requests { sharedInformers.Ark().V1().DeleteBackupRequests().Informer().GetStore().Add(test.requests[i]) } controller.deleteExpiredRequests() expectedActions := []core.Action{} for _, name := range test.expectedDeletions { expectedActions = append(expectedActions, core.NewDeleteAction(v1.SchemeGroupVersion.WithResource("deletebackuprequests"), "ns", name)) } arktest.CompareActions(t, expectedActions, client.Actions()) }) } }