Add resource policies E2E test

Signed-off-by: Ming <mqiu@vmware.com>

test/e2e/e2e_suite_test.go

@@ -37,6 +37,7 @@ import (
 	. "github.com/vmware-tanzu/velero/test/e2e/privilegesmgmt"
 	. "github.com/vmware-tanzu/velero/test/e2e/pv-backup"
 	. "github.com/vmware-tanzu/velero/test/e2e/resource-filtering"
+	. "github.com/vmware-tanzu/velero/test/e2e/resourcepolicies"
 	. "github.com/vmware-tanzu/velero/test/e2e/scale"
 	. "github.com/vmware-tanzu/velero/test/e2e/schedule"
 	. "github.com/vmware-tanzu/velero/test/e2e/upgrade"
@@ -109,6 +110,7 @@ var _ = Describe("[ResourceFiltering][IncludeNamespaces][Restore] Velero test on
 var _ = Describe("[ResourceFiltering][IncludeResources][Backup] Velero test on include resources from the cluster backup", BackupWithIncludeResources)
 var _ = Describe("[ResourceFiltering][IncludeResources][Restore] Velero test on include resources from the cluster restore", RestoreWithIncludeResources)
 var _ = Describe("[ResourceFiltering][LabelSelector] Velero test on backup include resources matching the label selector", BackupWithLabelSelector)
+var _ = Describe("[ResourceFiltering][ResourcePolicies] Velero test on skip backup of volume by resource policies", ResourcePoliciesTest)
 
 var _ = Describe("[Backups][Deletion][Restic] Velero tests of Restic backup deletion", BackupDeletionWithRestic)
 var _ = Describe("[Backups][Deletion][Snapshot] Velero tests of snapshot backup deletion", BackupDeletionWithSnapshots)

test/e2e/pv-backup/pv-backup-filter.go

@@ -48,7 +48,7 @@ func (p *PVBackupFiltering) Init() error {
 }
 
 func (p *PVBackupFiltering) StartRun() error {
-	err := InstallStorageClass(context.Background(), fmt.Sprintf("testdata/storage-class/%s.yaml", VeleroCfg.CloudProvider))
+	err := InstallStorageClass(p.Ctx, fmt.Sprintf("testdata/storage-class/%s.yaml", VeleroCfg.CloudProvider))
 	if err != nil {
 		return err
 	}
@@ -72,7 +72,6 @@ func (p *PVBackupFiltering) StartRun() error {
 	return nil
 }
 func (p *PVBackupFiltering) CreateResources() error {
-	p.Ctx, _ = context.WithTimeout(context.Background(), 60*time.Minute)
 	for _, ns := range *p.NSIncluded {
 		By(fmt.Sprintf("Create namespaces %s for workload\n", ns), func() {
 			Expect(CreateNamespace(p.Ctx, p.Client, ns)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", ns))
@@ -124,7 +123,7 @@ func (p *PVBackupFiltering) CreateResources() error {
 				WaitForPods(p.Ctx, p.Client, ns, p.podsList[index])
 				for i, pod := range p.podsList[index] {
 					for j := range p.volumesList[i] {
-						Expect(CreateFileToPod(p.Ctx, ns, pod, p.volumesList[i][j],
+						Expect(CreateFileToPod(p.Ctx, ns, pod, pod, p.volumesList[i][j],
 							FILE_NAME, fileContent(ns, pod, p.volumesList[i][j]))).To(Succeed())
 					}
 				}
@@ -135,7 +134,6 @@ func (p *PVBackupFiltering) CreateResources() error {
 }
 
 func (p *PVBackupFiltering) Verify() error {
-	p.Ctx, _ = context.WithTimeout(context.Background(), 60*time.Minute)
 	By(fmt.Sprintf("Waiting for all pods to start %s", p.podsList), func() {
 		for index, ns := range *p.NSIncluded {
 			By(fmt.Sprintf("Waiting for all pods to start %d in namespace %s", index, ns), func() {
@@ -182,7 +180,7 @@ func fileContent(namespace, podName, volume string) string {
 }
 
 func fileExist(ctx context.Context, namespace, podName, volume string) error {
-	c, err := ReadFileFromPodVolume(ctx, namespace, podName, volume, FILE_NAME)
+	c, err := ReadFileFromPodVolume(ctx, namespace, podName, podName, volume, FILE_NAME)
 	if err != nil {
 		return errors.Wrap(err, fmt.Sprintf("Fail to read file %s from volume %s of pod %s in %s ",
 			FILE_NAME, volume, podName, namespace))
@@ -197,7 +195,7 @@ func fileExist(ctx context.Context, namespace, podName, volume string) error {
 	}
 }
 func fileNotExist(ctx context.Context, namespace, podName, volume string) error {
-	_, err := ReadFileFromPodVolume(ctx, namespace, podName, volume, FILE_NAME)
+	_, err := ReadFileFromPodVolume(ctx, namespace, podName, podName, volume, FILE_NAME)
 	if err != nil {
 		return nil
 	} else {

test/e2e/resource-filtering/base.go

@@ -84,7 +84,7 @@ func (f *FilteringCase) CreateResources() error {
 		}
 		//Create deployment
 		fmt.Printf("Creating deployment in namespaces ...%s\n", namespace)
-		deployment := NewDeployment(f.NSBaseName, namespace, f.replica, f.labels, nil)
+		deployment := NewDeployment(f.NSBaseName, namespace, f.replica, f.labels, nil).Result()
 		deployment, err := CreateDeployment(f.Client.ClientGo, namespace, deployment)
 		if err != nil {
 			return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace))

test/e2e/resource-filtering/exclude_label.go

@@ -100,7 +100,7 @@ func (e *ExcludeFromBackup) CreateResources() error {
 	}
 	//Create deployment: to be included
 	fmt.Printf("Creating deployment in namespaces ...%s\n", namespace)
-	deployment := NewDeployment(e.NSBaseName, namespace, e.replica, label2, nil)
+	deployment := NewDeployment(e.NSBaseName, namespace, e.replica, label2, nil).Result()
 	deployment, err := CreateDeployment(e.Client.ClientGo, namespace, deployment)
 	if err != nil {
 		return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace))

test/e2e/resource-filtering/label_selector.go

@@ -101,7 +101,7 @@ func (l *LabelSelector) CreateResources() error {
 		//Create deployment
 		fmt.Printf("Creating deployment in namespaces ...%s\n", namespace)
 
-		deployment := NewDeployment(l.NSBaseName, namespace, l.replica, labels, nil)
+		deployment := NewDeployment(l.NSBaseName, namespace, l.replica, labels, nil).Result()
 		deployment, err := CreateDeployment(l.Client.ClientGo, namespace, deployment)
 		if err != nil {
 			return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace))

test/e2e/resourcepolicies/resource_policies.go

@@ -0,0 +1,281 @@
+/*
+Copyright 2021 the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package filtering
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"strings"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/google/uuid"
+	"github.com/pkg/errors"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+
+	. "github.com/vmware-tanzu/velero/test/e2e"
+	. "github.com/vmware-tanzu/velero/test/e2e/test"
+	. "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
+)
+
+const FileName = "test-data.txt"
+
+var yamlData = `version: v1
+volumePolicies:
+- conditions:
+    capacity: "2Gi,3Gi"
+  action:
+    type: skip
+- conditions:
+    storageClass:
+    - e2e-storage-class
+  action:
+    type: skip
+`
+
+type ResourcePoliciesCase struct {
+	TestCase
+	cmName, yamlConfig string
+}
+
+var ResourcePoliciesTest func() = TestFunc(&ResourcePoliciesCase{})
+
+func (r *ResourcePoliciesCase) Init() error {
+	rand.Seed(time.Now().UnixNano())
+	UUIDgen, _ = uuid.NewRandom()
+	r.yamlConfig = yamlData
+	r.VeleroCfg = VeleroCfg
+	r.Client = *r.VeleroCfg.ClientToInstallVelero
+	r.VeleroCfg.UseVolumeSnapshots = false
+	r.VeleroCfg.UseNodeAgent = true
+	r.NamespacesTotal = 3
+	r.NSBaseName = "resource-policies-" + UUIDgen.String()
+	r.cmName = "cm-resource-policies-sc"
+	r.NSIncluded = &[]string{}
+	for nsNum := 0; nsNum < r.NamespacesTotal; nsNum++ {
+		createNSName := fmt.Sprintf("%s-%00000d", r.NSBaseName, nsNum)
+		*r.NSIncluded = append(*r.NSIncluded, createNSName)
+	}
+
+	r.BackupName = "backup-resource-policies-" + UUIDgen.String()
+	r.RestoreName = "restore-resource-policies-" + UUIDgen.String()
+
+	r.BackupArgs = []string{
+		"create", "--namespace", VeleroCfg.VeleroNamespace, "backup", r.BackupName,
+		"--resource-policies-configmap", r.cmName,
+		"--include-namespaces", strings.Join(*r.NSIncluded, ","),
+		"--default-volumes-to-fs-backup", "--wait",
+	}
+
+	r.RestoreArgs = []string{
+		"create", "--namespace", VeleroCfg.VeleroNamespace, "restore", r.RestoreName,
+		"--from-backup", r.BackupName, "--wait",
+	}
+
+	r.TestMsg = &TestMSG{
+		Desc:      "Skip backup of volume by resource policies",
+		FailedMSG: "Failed to skip backup of volume by resource policies",
+		Text:      fmt.Sprintf("Should backup PVs in namespace %s respect to resource policies rules", *r.NSIncluded),
+	}
+	return nil
+}
+
+func (r *ResourcePoliciesCase) CreateResources() error {
+	r.Ctx, _ = context.WithTimeout(context.Background(), 60*time.Minute)
+
+	By(("Installing storage class..."), func() {
+		Expect(r.installTestStorageClass(fmt.Sprintf("testdata/storage-class/%s.yaml", VeleroCfg.CloudProvider))).To(Succeed(), "Failed to install storage class")
+	})
+
+	By(fmt.Sprintf("Create configmap %s in namespaces %s for workload\n", r.cmName, r.VeleroCfg.VeleroNamespace), func() {
+		Expect(CreateConfigMapFromYAMLData(r.Client.ClientGo, r.yamlConfig, r.cmName, r.VeleroCfg.VeleroNamespace)).To(Succeed(), fmt.Sprintf("Failed to create configmap %s in namespaces %s for workload\n", r.cmName, r.VeleroCfg.VeleroNamespace))
+	})
+
+	By(fmt.Sprintf("Waiting for configmap %s in namespaces %s ready\n", r.cmName, r.VeleroCfg.VeleroNamespace), func() {
+		Expect(WaitForConfigMapComplete(r.Client.ClientGo, r.VeleroCfg.VeleroNamespace, r.cmName)).To(Succeed(), fmt.Sprintf("Failed to wait configmap %s in namespaces %s ready\n", r.cmName, r.VeleroCfg.VeleroNamespace))
+	})
+
+	for nsNum := 0; nsNum < r.NamespacesTotal; nsNum++ {
+		namespace := fmt.Sprintf("%s-%00000d", r.NSBaseName, nsNum)
+		By(fmt.Sprintf("Create namespaces %s for workload\n", namespace), func() {
+			Expect(CreateNamespace(r.Ctx, r.Client, namespace)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", namespace))
+		})
+
+		volName := fmt.Sprintf("vol-%s-%00000d", r.NSBaseName, nsNum)
+		volList := PrepareVolumeList([]string{volName})
+
+		// Create PVC
+		By(fmt.Sprintf("Creating pvc in namespaces ...%s\n", namespace), func() {
+			Expect(r.createPVC(nsNum, namespace, volList)).To(Succeed(), fmt.Sprintf("Failed to create pvc in namespace %s", namespace))
+		})
+
+		// Create deployment
+		fmt.Printf("Creating deployment in namespaces ...%s\n", namespace)
+		By(fmt.Sprintf("Creating deployment in namespaces ...%s\n", namespace), func() {
+			Expect(r.createDeploymentWithVolume(namespace, volList)).To(Succeed(), fmt.Sprintf("Failed to create deployment namespace %s", namespace))
+		})
+
+		//Write data into pods
+		By(fmt.Sprintf("Writing data into pod in namespaces ...%s\n", namespace), func() {
+			Expect(r.writeDataIntoPods(namespace, volName)).To(Succeed(), fmt.Sprintf("Failed to write data into pod in namespace %s", namespace))
+		})
+	}
+
+	return nil
+}
+
+func (r *ResourcePoliciesCase) Verify() error {
+	for i, ns := range *r.NSIncluded {
+		By(fmt.Sprintf("Verify pod data in namespace %s", ns), func() {
+			By(fmt.Sprintf("Waiting for deployment %s in namespace %s ready", r.NSBaseName, ns), func() {
+				Expect(WaitForReadyDeployment(r.Client.ClientGo, ns, r.NSBaseName)).To(Succeed(), fmt.Sprintf("Failed to waiting for deployment %s in namespace %s ready", r.NSBaseName, ns))
+			})
+			podList, err := ListPods(r.Ctx, r.Client, ns)
+			Expect(err).To(Succeed(), fmt.Sprintf("failed to list pods in namespace: %q with error %v", ns, err))
+
+			volName := fmt.Sprintf("vol-%s-%00000d", r.NSBaseName, i)
+			for _, pod := range podList.Items {
+				for _, vol := range pod.Spec.Volumes {
+					if vol.Name != volName {
+						continue
+					}
+					content, err := ReadFileFromPodVolume(r.Ctx, ns, pod.Name, "container-busybox", vol.Name, FileName)
+					if i%2 == 0 {
+						Expect(err).To(HaveOccurred(), "Expected file not found") // File should not exist
+					} else {
+						Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Fail to read file %s from volume %s of pod %s in namespace %s",
+							FileName, vol.Name, pod.Name, ns))
+
+						content = strings.Replace(content, "\n", "", -1)
+						originContent := strings.Replace(fmt.Sprintf("ns-%s pod-%s volume-%s", ns, pod.Name, vol.Name), "\n", "", -1)
+
+						Expect(content == originContent).To(BeTrue(), fmt.Sprintf("File %s does not exist in volume %s of pod %s in namespace %s",
+							FileName, vol.Name, pod.Name, ns))
+					}
+				}
+			}
+		})
+
+	}
+	return nil
+}
+
+func (r *ResourcePoliciesCase) Clean() error {
+	if err := r.deleteTestStorageClassList([]string{"e2e-storage-class", "e2e-storage-class-2"}); err != nil {
+		return err
+	}
+
+	if err := DeleteConfigmap(r.Client.ClientGo, r.VeleroCfg.VeleroNamespace, r.cmName); err != nil {
+		return err
+	}
+
+	return r.GetTestCase().Clean()
+}
+
+func (r *ResourcePoliciesCase) createPVC(index int, namespace string, volList []*v1.Volume) error {
+	var err error
+	for i := range volList {
+		pvcName := fmt.Sprintf("pvc-%d", i)
+		fmt.Printf("Creating PVC %s in namespaces ...%s\n", pvcName, namespace)
+		if index%3 == 0 {
+			pvcBuilder := NewPVC(namespace, pvcName).WithStorageClass("e2e-storage-class") // Testing sc should not backup
+			err = CreatePvc(r.Client, pvcBuilder)
+		} else if index%3 == 1 {
+			pvcBuilder := NewPVC(namespace, pvcName).WithStorageClass("e2e-storage-class-2") // Testing sc should backup
+			err = CreatePvc(r.Client, pvcBuilder)
+		} else if index%3 == 2 {
+			pvcBuilder := NewPVC(namespace, pvcName).WithStorageClass("e2e-storage-class-2").WithResourceStorage(resource.MustParse("2Gi")) // Testing capacity should not backup
+			err = CreatePvc(r.Client, pvcBuilder)
+		}
+		if err != nil {
+			return errors.Wrapf(err, "failed to create pvc %s in namespace %s", pvcName, namespace)
+		}
+	}
+	return nil
+}
+
+func (r *ResourcePoliciesCase) createDeploymentWithVolume(namespace string, volList []*v1.Volume) error {
+	deployment := NewDeployment(r.NSBaseName, namespace, 1, map[string]string{"resource-policies": "resource-policies"}, nil).WithVolume(volList).Result()
+	deployment, err := CreateDeployment(r.Client.ClientGo, namespace, deployment)
+	if err != nil {
+		return errors.Wrap(err, fmt.Sprintf("failed to create deloyment %s the namespace %q", deployment.Name, namespace))
+	}
+	err = WaitForReadyDeployment(r.Client.ClientGo, namespace, deployment.Name)
+	if err != nil {
+		return errors.Wrap(err, fmt.Sprintf("failed to wait for deployment %s to be ready in namespace: %q", deployment.Name, namespace))
+	}
+	return nil
+}
+
+func (r *ResourcePoliciesCase) writeDataIntoPods(namespace, volName string) error {
+	podList, err := ListPods(r.Ctx, r.Client, namespace)
+	if err != nil {
+		return errors.Wrap(err, fmt.Sprintf("failed to list pods in namespace: %q with error %v", namespace, err))
+	}
+	for _, pod := range podList.Items {
+		for _, vol := range pod.Spec.Volumes {
+			if vol.Name != volName {
+				continue
+			}
+			err := CreateFileToPod(r.Ctx, namespace, pod.Name, "container-busybox", vol.Name, FileName, fmt.Sprintf("ns-%s pod-%s volume-%s", namespace, pod.Name, vol.Name))
+			if err != nil {
+				return errors.Wrap(err, fmt.Sprintf("failed to create file into pod %s in namespace: %q", pod.Name, namespace))
+			}
+		}
+	}
+	return nil
+}
+
+func (r *ResourcePoliciesCase) deleteTestStorageClassList(scList []string) error {
+	for _, v := range scList {
+		if err := DeleteStorageClass(r.Ctx, r.Client, v); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (r *ResourcePoliciesCase) installTestStorageClass(path string) error {
+	err := InstallStorageClass(r.Ctx, path)
+	if err != nil {
+		return err
+	}
+	content, err := ioutil.ReadFile(path)
+	if err != nil {
+		return errors.Wrapf(err, "failed to get %s when install storage class", path)
+	}
+
+	// replace sc to new value
+	newContent := strings.ReplaceAll(string(content), "name: e2e-storage-class", "name: e2e-storage-class-2")
+
+	tmpFile, err := ioutil.TempFile("", "sc-file")
+	if err != nil {
+		return errors.Wrapf(err, "failed to create temp file  when install storage class")
+	}
+
+	defer os.Remove(tmpFile.Name())
+	if _, err := tmpFile.WriteString(newContent); err != nil {
+		return errors.Wrapf(err, "failed to write content into temp file %s when install storage class", tmpFile.Name())
+	}
+	return InstallStorageClass(r.Ctx, tmpFile.Name())
+}

test/e2e/schedule/ordered_resources.go

@@ -159,7 +159,7 @@ func (o *OrderedResources) CreateResources() error {
 	//Create deployment
 	deploymentName := fmt.Sprintf("deploy-%s", o.NSBaseName)
 	fmt.Printf("Creating deployment %s in %s namespaces ...\n", deploymentName, o.Namespace)
-	deployment := NewDeployment(deploymentName, o.Namespace, 1, label, nil)
+	deployment := NewDeployment(deploymentName, o.Namespace, 1, label, nil).Result()
 	deployment, err := CreateDeployment(o.Client.ClientGo, o.Namespace, deployment)
 	if err != nil {
 		return errors.Wrap(err, fmt.Sprintf("failed to create namespace %q with err %v", o.Namespace, err))

test/e2e/util/k8s/common.go

@@ -255,6 +255,22 @@ func GetPVByPodName(client TestClient, namespace, podName string) (string, error
 	}
 	return pv_value.Name, nil
 }
+
+func PrepareVolumeList(volumeNameList []string) (vols []*corev1.Volume) {
+	for i, volume := range volumeNameList {
+		vols = append(vols, &corev1.Volume{
+			Name: volume,
+			VolumeSource: corev1.VolumeSource{
+				PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
+					ClaimName: fmt.Sprintf("pvc-%d", i),
+					ReadOnly:  false,
+				},
+			},
+		})
+	}
+	return
+}
+
 func CreatePodWithPVC(client TestClient, ns, podName, sc, pvcName string, volumeNameList []string, pvcAnn map[string]string) (*corev1.Pod, error) {
 	volumes := []corev1.Volume{}
 	for _, volume := range volumeNameList {
@@ -285,15 +301,15 @@ func CreatePodWithPVC(client TestClient, ns, podName, sc, pvcName string, volume
 	return pod, nil
 }
 
-func CreateFileToPod(ctx context.Context, namespace, podName, volume, filename, content string) error {
-	arg := []string{"exec", "-n", namespace, "-c", podName, podName,
+func CreateFileToPod(ctx context.Context, namespace, podName, containerName, volume, filename, content string) error {
+	arg := []string{"exec", "-n", namespace, "-c", containerName, podName,
 		"--", "/bin/sh", "-c", fmt.Sprintf("echo ns-%s pod-%s volume-%s  > /%s/%s", namespace, podName, volume, volume, filename)}
 	cmd := exec.CommandContext(ctx, "kubectl", arg...)
 	fmt.Printf("Kubectl exec cmd =%v\n", cmd)
 	return cmd.Run()
 }
-func ReadFileFromPodVolume(ctx context.Context, namespace, podName, volume, filename string) (string, error) {
-	arg := []string{"exec", "-n", namespace, "-c", podName, podName,
+func ReadFileFromPodVolume(ctx context.Context, namespace, podName, containerName, volume, filename string) (string, error) {
+	arg := []string{"exec", "-n", namespace, "-c", containerName, podName,
 		"--", "cat", fmt.Sprintf("/%s/%s", volume, filename)}
 	cmd := exec.CommandContext(ctx, "kubectl", arg...)
 	fmt.Printf("Kubectl exec cmd =%v\n", cmd)

test/e2e/util/k8s/configmap.go

@@ -42,6 +42,20 @@ func CreateConfigMap(c clientset.Interface, ns, name string, labels, data map[st
 	return c.CoreV1().ConfigMaps(ns).Create(context.TODO(), cm, metav1.CreateOptions{})
 }
 
+func CreateConfigMapFromYAMLData(c clientset.Interface, yamlData, cmName, namespace string) error {
+	cmData := make(map[string]string)
+	cmData[cmName] = yamlData
+	cm := &v1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      cmName,
+			Namespace: namespace,
+		},
+		Data: cmData,
+	}
+	_, err := c.CoreV1().ConfigMaps(namespace).Create(context.TODO(), cm, metav1.CreateOptions{})
+	return err
+}
+
 // WaitForConfigMapComplete uses c to wait for completions to complete for the Job jobName in namespace ns.
 func WaitForConfigMapComplete(c clientset.Interface, ns, configmapName string) error {
 	return wait.Poll(PollInterval, PollTimeout, func() (bool, error) {
@@ -57,10 +71,18 @@ func GetConfigmap(c clientset.Interface, ns, secretName string) (*v1.ConfigMap,
 	return c.CoreV1().ConfigMaps(ns).Get(context.TODO(), secretName, metav1.GetOptions{})
 }
 
-func WaitForConfigmapDelete(c clientset.Interface, ns, name string) error {
+func DeleteConfigmap(c clientset.Interface, ns, name string) error {
 	if err := c.CoreV1().ConfigMaps(ns).Delete(context.TODO(), name, metav1.DeleteOptions{}); err != nil {
 		return errors.Wrap(err, fmt.Sprintf("failed to delete  configmap in namespace %q", ns))
 	}
+	return nil
+}
+
+func WaitForConfigmapDelete(c clientset.Interface, ns, name string) error {
+	if err := DeleteConfigmap(c, ns, name); err != nil {
+		return err
+	}
+
 	return waitutil.PollImmediateInfinite(5*time.Second,
 		func() (bool, error) {
 			if _, err := c.CoreV1().ConfigMaps(ns).Get(context.TODO(), ns, metav1.GetOptions{}); err != nil {

test/e2e/util/k8s/deployment.go

@@ -35,46 +35,73 @@ const (
 	PollTimeout  = 15 * time.Minute
 )
 
+// DeploymentBuilder builds Deployment objects.
+type DeploymentBuilder struct {
+	*apps.Deployment
+}
+
+func (d *DeploymentBuilder) Result() *apps.Deployment {
+	return d.Deployment
+}
+
 // newDeployment returns a RollingUpdate Deployment with a fake container image
-func NewDeployment(name, ns string, replicas int32, labels map[string]string, containers []v1.Container) *apps.Deployment {
+func NewDeployment(name, ns string, replicas int32, labels map[string]string, containers []v1.Container) *DeploymentBuilder {
 	if containers == nil {
 		containers = []v1.Container{
 			{
-				Name:    fmt.Sprintf("container-%s", "busybox"),
+				Name:    "container-busybox",
 				Image:   "gcr.io/velero-gcp/busybox:latest",
 				Command: []string{"sleep", "1000000"},
 			},
 		}
 	}
-	return &apps.Deployment{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Deployment",
-			APIVersion: "apps/v1",
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Namespace: ns,
-			Name:      name,
-			Labels:    labels,
-		},
-		Spec: apps.DeploymentSpec{
-			Replicas: &replicas,
-			Selector: &metav1.LabelSelector{MatchLabels: labels},
-			Strategy: apps.DeploymentStrategy{
-				Type:          apps.RollingUpdateDeploymentStrategyType,
-				RollingUpdate: new(apps.RollingUpdateDeployment),
+	return &DeploymentBuilder{
+		&apps.Deployment{
+			TypeMeta: metav1.TypeMeta{
+				Kind:       "Deployment",
+				APIVersion: "apps/v1",
 			},
-			Template: v1.PodTemplateSpec{
-				ObjectMeta: metav1.ObjectMeta{
-					Labels: labels,
+			ObjectMeta: metav1.ObjectMeta{
+				Namespace: ns,
+				Name:      name,
+				Labels:    labels,
+			},
+			Spec: apps.DeploymentSpec{
+				Replicas: &replicas,
+				Selector: &metav1.LabelSelector{MatchLabels: labels},
+				Strategy: apps.DeploymentStrategy{
+					Type:          apps.RollingUpdateDeploymentStrategyType,
+					RollingUpdate: new(apps.RollingUpdateDeployment),
 				},
-				Spec: v1.PodSpec{
-					Containers: containers,
+				Template: v1.PodTemplateSpec{
+					ObjectMeta: metav1.ObjectMeta{
+						Labels: labels,
+					},
+					Spec: v1.PodSpec{
+						Containers: containers,
+					},
 				},
 			},
 		},
 	}
 }
 
+func (d *DeploymentBuilder) WithVolume(vols []*v1.Volume) *DeploymentBuilder {
+	vmList := []v1.VolumeMount{}
+	for i, v := range vols {
+		vmList = append(vmList, v1.VolumeMount{
+			Name:      v.Name,
+			MountPath: "/" + v.Name,
+		})
+		d.Spec.Template.Spec.Volumes = append(d.Spec.Template.Spec.Volumes, *vols[i])
+	}
+
+	// NOTE here just mount volumes to the first container
+	d.Spec.Template.Spec.Containers[0].VolumeMounts = vmList
+
+	return d
+}
+
 func CreateDeploy(c clientset.Interface, ns string, deployment *apps.Deployment) error {
 	_, err := c.AppsV1().Deployments(ns).Create(context.TODO(), deployment, metav1.CreateOptions{})
 	return err

test/e2e/util/k8s/pod.go

@@ -77,3 +77,7 @@ func AddAnnotationToPod(ctx context.Context, client TestClient, namespace, podNa
 
 	return client.ClientGo.CoreV1().Pods(namespace).Update(ctx, newPod, metav1.UpdateOptions{})
 }
+
+func ListPods(ctx context.Context, client TestClient, namespace string) (*corev1.PodList, error) {
+	return client.ClientGo.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{})
+}

test/e2e/util/k8s/pvc.go

@@ -24,28 +24,67 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-func CreatePVC(client TestClient, ns, name, sc string, ann map[string]string) (*corev1.PersistentVolumeClaim, error) {
+// PVCBuilder builds PVC objects.
+type PVCBuilder struct {
+	*corev1.PersistentVolumeClaim
+}
+
+func (p *PVCBuilder) Result() *corev1.PersistentVolumeClaim {
+	return p.PersistentVolumeClaim
+}
+
+func NewPVC(ns, name string) *PVCBuilder {
 	oMeta := metav1.ObjectMeta{}
-	oMeta = metav1.ObjectMeta{Name: name}
-	if ann != nil {
-		oMeta.Annotations = ann
-	}
-	pvc := &corev1.PersistentVolumeClaim{
-		ObjectMeta: oMeta,
-		Spec: corev1.PersistentVolumeClaimSpec{
-			AccessModes: []corev1.PersistentVolumeAccessMode{
-				corev1.ReadWriteOnce,
-			},
-			Resources: corev1.ResourceRequirements{
-				Requests: corev1.ResourceList{
-					corev1.ResourceStorage: resource.MustParse("1Gi"),
+	oMeta = metav1.ObjectMeta{Name: name, Namespace: ns}
+	return &PVCBuilder{
+		&corev1.PersistentVolumeClaim{
+			ObjectMeta: oMeta,
+			Spec: corev1.PersistentVolumeClaimSpec{
+				AccessModes: []corev1.PersistentVolumeAccessMode{
+					corev1.ReadWriteOnce, // Default read write once
+				},
+				Resources: corev1.ResourceRequirements{
+					Requests: corev1.ResourceList{
+						corev1.ResourceStorage: resource.MustParse("1Gi"), // Default 1Gi
+					},
 				},
 			},
-			StorageClassName: &sc,
 		},
 	}
-	return client.ClientGo.CoreV1().PersistentVolumeClaims(ns).Create(context.TODO(), pvc, metav1.CreateOptions{})
 }
+
+func (p *PVCBuilder) WithAnnotation(ann map[string]string) *PVCBuilder {
+	p.Annotations = ann
+	return p
+}
+
+func (p *PVCBuilder) WithStorageClass(sc string) *PVCBuilder {
+	p.Spec.StorageClassName = &sc
+	return p
+}
+
+func (p *PVCBuilder) WithResourceStorage(q resource.Quantity) *PVCBuilder {
+	p.Spec.Resources.Requests[corev1.ResourceStorage] = q
+	return p
+}
+
+func CreatePVC(client TestClient, ns, name, sc string, ann map[string]string) (*corev1.PersistentVolumeClaim, error) {
+	pvcBulder := NewPVC(ns, name)
+	if ann != nil {
+		pvcBulder.WithAnnotation(ann)
+	}
+	if sc != "" {
+		pvcBulder.WithStorageClass(sc)
+	}
+
+	return client.ClientGo.CoreV1().PersistentVolumeClaims(ns).Create(context.TODO(), pvcBulder.Result(), metav1.CreateOptions{})
+}
+
+func CreatePvc(client TestClient, pvcBulder *PVCBuilder) error {
+	_, err := client.ClientGo.CoreV1().PersistentVolumeClaims(pvcBulder.Namespace).Create(context.TODO(), pvcBulder.Result(), metav1.CreateOptions{})
+	return err
+}
+
 func GetPVC(ctx context.Context, client TestClient, namespace string, pvcName string) (*corev1.PersistentVolumeClaim, error) {
 	return client.ClientGo.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{})
 }

test/e2e/util/k8s/sc.go

@@ -3,6 +3,9 @@ package k8s
 import (
 	"context"
 	"fmt"
+
+	"github.com/pkg/errors"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 func InstallStorageClass(ctx context.Context, yaml string) error {
@@ -10,3 +13,10 @@ func InstallStorageClass(ctx context.Context, yaml string) error {
 	err := KubectlApplyByFile(ctx, yaml)
 	return err
 }
+
+func DeleteStorageClass(ctx context.Context, client TestClient, name string) error {
+	if err := client.ClientGo.StorageV1().StorageClasses().Delete(ctx, name, v1.DeleteOptions{}); err != nil {
+		return errors.Wrapf(err, "Could not retrieve storage classes %s", name)
+	}
+	return nil
+}