diff --git a/test/e2e/debug-bundle-1669709736054564976.tar.gz b/test/e2e/debug-bundle-1669709736054564976.tar.gz deleted file mode 100644 index 293844e35..000000000 Binary files a/test/e2e/debug-bundle-1669709736054564976.tar.gz and /dev/null differ diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 4ce7486d2..26c61d821 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -37,6 +37,7 @@ import ( . "github.com/vmware-tanzu/velero/test/e2e/privilegesmgmt" . "github.com/vmware-tanzu/velero/test/e2e/pv-backup" . "github.com/vmware-tanzu/velero/test/e2e/resource-filtering" + . "github.com/vmware-tanzu/velero/test/e2e/resourcepolicies" . "github.com/vmware-tanzu/velero/test/e2e/scale" . "github.com/vmware-tanzu/velero/test/e2e/schedule" . "github.com/vmware-tanzu/velero/test/e2e/upgrade" @@ -109,6 +110,7 @@ var _ = Describe("[ResourceFiltering][IncludeNamespaces][Restore] Velero test on var _ = Describe("[ResourceFiltering][IncludeResources][Backup] Velero test on include resources from the cluster backup", BackupWithIncludeResources) var _ = Describe("[ResourceFiltering][IncludeResources][Restore] Velero test on include resources from the cluster restore", RestoreWithIncludeResources) var _ = Describe("[ResourceFiltering][LabelSelector] Velero test on backup include resources matching the label selector", BackupWithLabelSelector) +var _ = Describe("[ResourceFiltering][ResourcePolicies] Velero test on skip backup of volume by resource policies", ResourcePoliciesTest) var _ = Describe("[Backups][Deletion][Restic] Velero tests of Restic backup deletion", BackupDeletionWithRestic) var _ = Describe("[Backups][Deletion][Snapshot] Velero tests of snapshot backup deletion", BackupDeletionWithSnapshots) diff --git a/test/e2e/pv-backup/pv-backup-filter.go b/test/e2e/pv-backup/pv-backup-filter.go index 47785309c..311503612 100644 --- a/test/e2e/pv-backup/pv-backup-filter.go +++ b/test/e2e/pv-backup/pv-backup-filter.go @@ -48,7 +48,7 @@ func (p *PVBackupFiltering) Init() error { } func (p *PVBackupFiltering) StartRun() error { - err := InstallStorageClass(context.Background(), fmt.Sprintf("testdata/storage-class/%s.yaml", VeleroCfg.CloudProvider)) + err := InstallStorageClass(p.Ctx, fmt.Sprintf("testdata/storage-class/%s.yaml", VeleroCfg.CloudProvider)) if err != nil { return err } @@ -72,7 +72,6 @@ func (p *PVBackupFiltering) StartRun() error { return nil } func (p *PVBackupFiltering) CreateResources() error { - p.Ctx, _ = context.WithTimeout(context.Background(), 60*time.Minute) for _, ns := range *p.NSIncluded { By(fmt.Sprintf("Create namespaces %s for workload\n", ns), func() { Expect(CreateNamespace(p.Ctx, p.Client, ns)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", ns)) @@ -124,7 +123,7 @@ func (p *PVBackupFiltering) CreateResources() error { WaitForPods(p.Ctx, p.Client, ns, p.podsList[index]) for i, pod := range p.podsList[index] { for j := range p.volumesList[i] { - Expect(CreateFileToPod(p.Ctx, ns, pod, p.volumesList[i][j], + Expect(CreateFileToPod(p.Ctx, ns, pod, pod, p.volumesList[i][j], FILE_NAME, fileContent(ns, pod, p.volumesList[i][j]))).To(Succeed()) } } @@ -135,7 +134,6 @@ func (p *PVBackupFiltering) CreateResources() error { } func (p *PVBackupFiltering) Verify() error { - p.Ctx, _ = context.WithTimeout(context.Background(), 60*time.Minute) By(fmt.Sprintf("Waiting for all pods to start %s", p.podsList), func() { for index, ns := range *p.NSIncluded { By(fmt.Sprintf("Waiting for all pods to start %d in namespace %s", index, ns), func() { @@ -182,7 +180,7 @@ func fileContent(namespace, podName, volume string) string { } func fileExist(ctx context.Context, namespace, podName, volume string) error { - c, err := ReadFileFromPodVolume(ctx, namespace, podName, volume, FILE_NAME) + c, err := ReadFileFromPodVolume(ctx, namespace, podName, podName, volume, FILE_NAME) if err != nil { return errors.Wrap(err, fmt.Sprintf("Fail to read file %s from volume %s of pod %s in %s ", FILE_NAME, volume, podName, namespace)) @@ -197,7 +195,7 @@ func fileExist(ctx context.Context, namespace, podName, volume string) error { } } func fileNotExist(ctx context.Context, namespace, podName, volume string) error { - _, err := ReadFileFromPodVolume(ctx, namespace, podName, volume, FILE_NAME) + _, err := ReadFileFromPodVolume(ctx, namespace, podName, podName, volume, FILE_NAME) if err != nil { return nil } else { diff --git a/test/e2e/resource-filtering/base.go b/test/e2e/resource-filtering/base.go index b13bf8702..35f5e3f77 100644 --- a/test/e2e/resource-filtering/base.go +++ b/test/e2e/resource-filtering/base.go @@ -84,7 +84,7 @@ func (f *FilteringCase) CreateResources() error { } //Create deployment fmt.Printf("Creating deployment in namespaces ...%s\n", namespace) - deployment := NewDeployment(f.NSBaseName, namespace, f.replica, f.labels, nil) + deployment := NewDeployment(f.NSBaseName, namespace, f.replica, f.labels, nil).Result() deployment, err := CreateDeployment(f.Client.ClientGo, namespace, deployment) if err != nil { return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace)) diff --git a/test/e2e/resource-filtering/exclude_label.go b/test/e2e/resource-filtering/exclude_label.go index f04308ea1..7644a1c86 100644 --- a/test/e2e/resource-filtering/exclude_label.go +++ b/test/e2e/resource-filtering/exclude_label.go @@ -100,7 +100,7 @@ func (e *ExcludeFromBackup) CreateResources() error { } //Create deployment: to be included fmt.Printf("Creating deployment in namespaces ...%s\n", namespace) - deployment := NewDeployment(e.NSBaseName, namespace, e.replica, label2, nil) + deployment := NewDeployment(e.NSBaseName, namespace, e.replica, label2, nil).Result() deployment, err := CreateDeployment(e.Client.ClientGo, namespace, deployment) if err != nil { return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace)) diff --git a/test/e2e/resource-filtering/label_selector.go b/test/e2e/resource-filtering/label_selector.go index 2641f60b8..2053daed3 100644 --- a/test/e2e/resource-filtering/label_selector.go +++ b/test/e2e/resource-filtering/label_selector.go @@ -101,7 +101,7 @@ func (l *LabelSelector) CreateResources() error { //Create deployment fmt.Printf("Creating deployment in namespaces ...%s\n", namespace) - deployment := NewDeployment(l.NSBaseName, namespace, l.replica, labels, nil) + deployment := NewDeployment(l.NSBaseName, namespace, l.replica, labels, nil).Result() deployment, err := CreateDeployment(l.Client.ClientGo, namespace, deployment) if err != nil { return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace)) diff --git a/test/e2e/resourcepolicies/resource_policies.go b/test/e2e/resourcepolicies/resource_policies.go new file mode 100644 index 000000000..c5b5dfdfc --- /dev/null +++ b/test/e2e/resourcepolicies/resource_policies.go @@ -0,0 +1,281 @@ +/* +Copyright 2021 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package filtering + +import ( + "context" + "fmt" + "io/ioutil" + "math/rand" + "os" + "strings" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/google/uuid" + "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + + . "github.com/vmware-tanzu/velero/test/e2e" + . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" +) + +const FileName = "test-data.txt" + +var yamlData = `version: v1 +volumePolicies: +- conditions: + capacity: "2Gi,3Gi" + action: + type: skip +- conditions: + storageClass: + - e2e-storage-class + action: + type: skip +` + +type ResourcePoliciesCase struct { + TestCase + cmName, yamlConfig string +} + +var ResourcePoliciesTest func() = TestFunc(&ResourcePoliciesCase{}) + +func (r *ResourcePoliciesCase) Init() error { + rand.Seed(time.Now().UnixNano()) + UUIDgen, _ = uuid.NewRandom() + r.yamlConfig = yamlData + r.VeleroCfg = VeleroCfg + r.Client = *r.VeleroCfg.ClientToInstallVelero + r.VeleroCfg.UseVolumeSnapshots = false + r.VeleroCfg.UseNodeAgent = true + r.NamespacesTotal = 3 + r.NSBaseName = "resource-policies-" + UUIDgen.String() + r.cmName = "cm-resource-policies-sc" + r.NSIncluded = &[]string{} + for nsNum := 0; nsNum < r.NamespacesTotal; nsNum++ { + createNSName := fmt.Sprintf("%s-%00000d", r.NSBaseName, nsNum) + *r.NSIncluded = append(*r.NSIncluded, createNSName) + } + + r.BackupName = "backup-resource-policies-" + UUIDgen.String() + r.RestoreName = "restore-resource-policies-" + UUIDgen.String() + + r.BackupArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", r.BackupName, + "--resource-policies-configmap", r.cmName, + "--include-namespaces", strings.Join(*r.NSIncluded, ","), + "--default-volumes-to-fs-backup", + "--snapshot-volumes=false", "--wait", + } + + r.RestoreArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "restore", r.RestoreName, + "--from-backup", r.BackupName, "--wait", + } + + r.TestMsg = &TestMSG{ + Desc: "Skip backup of volume by resource policies", + FailedMSG: "Failed to skip backup of volume by resource policies", + Text: fmt.Sprintf("Should backup PVs in namespace %s respect to resource policies rules", *r.NSIncluded), + } + return nil +} + +func (r *ResourcePoliciesCase) CreateResources() error { + r.Ctx, _ = context.WithTimeout(context.Background(), 60*time.Minute) + + By(("Installing storage class..."), func() { + Expect(r.installTestStorageClasses(fmt.Sprintf("testdata/storage-class/%s.yaml", VeleroCfg.CloudProvider))).To(Succeed(), "Failed to install storage class") + }) + + By(fmt.Sprintf("Create configmap %s in namespaces %s for workload\n", r.cmName, r.VeleroCfg.VeleroNamespace), func() { + Expect(CreateConfigMapFromYAMLData(r.Client.ClientGo, r.yamlConfig, r.cmName, r.VeleroCfg.VeleroNamespace)).To(Succeed(), fmt.Sprintf("Failed to create configmap %s in namespaces %s for workload\n", r.cmName, r.VeleroCfg.VeleroNamespace)) + }) + + By(fmt.Sprintf("Waiting for configmap %s in namespaces %s ready\n", r.cmName, r.VeleroCfg.VeleroNamespace), func() { + Expect(WaitForConfigMapComplete(r.Client.ClientGo, r.VeleroCfg.VeleroNamespace, r.cmName)).To(Succeed(), fmt.Sprintf("Failed to wait configmap %s in namespaces %s ready\n", r.cmName, r.VeleroCfg.VeleroNamespace)) + }) + + for nsNum := 0; nsNum < r.NamespacesTotal; nsNum++ { + namespace := fmt.Sprintf("%s-%00000d", r.NSBaseName, nsNum) + By(fmt.Sprintf("Create namespaces %s for workload\n", namespace), func() { + Expect(CreateNamespace(r.Ctx, r.Client, namespace)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", namespace)) + }) + + volName := fmt.Sprintf("vol-%s-%00000d", r.NSBaseName, nsNum) + volList := PrepareVolumeList([]string{volName}) + + // Create PVC + By(fmt.Sprintf("Creating pvc in namespaces ...%s\n", namespace), func() { + Expect(r.createPVC(nsNum, namespace, volList)).To(Succeed(), fmt.Sprintf("Failed to create pvc in namespace %s", namespace)) + }) + + // Create deployment + By(fmt.Sprintf("Creating deployment in namespaces ...%s\n", namespace), func() { + Expect(r.createDeploymentWithVolume(namespace, volList)).To(Succeed(), fmt.Sprintf("Failed to create deployment namespace %s", namespace)) + }) + + //Write data into pods + By(fmt.Sprintf("Writing data into pod in namespaces ...%s\n", namespace), func() { + Expect(r.writeDataIntoPods(namespace, volName)).To(Succeed(), fmt.Sprintf("Failed to write data into pod in namespace %s", namespace)) + }) + } + + return nil +} + +func (r *ResourcePoliciesCase) Verify() error { + for i, ns := range *r.NSIncluded { + By(fmt.Sprintf("Verify pod data in namespace %s", ns), func() { + By(fmt.Sprintf("Waiting for deployment %s in namespace %s ready", r.NSBaseName, ns), func() { + Expect(WaitForReadyDeployment(r.Client.ClientGo, ns, r.NSBaseName)).To(Succeed(), fmt.Sprintf("Failed to waiting for deployment %s in namespace %s ready", r.NSBaseName, ns)) + }) + podList, err := ListPods(r.Ctx, r.Client, ns) + Expect(err).To(Succeed(), fmt.Sprintf("failed to list pods in namespace: %q with error %v", ns, err)) + + volName := fmt.Sprintf("vol-%s-%00000d", r.NSBaseName, i) + for _, pod := range podList.Items { + for _, vol := range pod.Spec.Volumes { + if vol.Name != volName { + continue + } + content, err := ReadFileFromPodVolume(r.Ctx, ns, pod.Name, "container-busybox", vol.Name, FileName) + if i%2 == 0 { + Expect(err).To(HaveOccurred(), "Expected file not found") // File should not exist + } else { + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Fail to read file %s from volume %s of pod %s in namespace %s", + FileName, vol.Name, pod.Name, ns)) + + content = strings.Replace(content, "\n", "", -1) + originContent := strings.Replace(fmt.Sprintf("ns-%s pod-%s volume-%s", ns, pod.Name, vol.Name), "\n", "", -1) + + Expect(content == originContent).To(BeTrue(), fmt.Sprintf("File %s does not exist in volume %s of pod %s in namespace %s", + FileName, vol.Name, pod.Name, ns)) + } + } + } + }) + + } + return nil +} + +func (r *ResourcePoliciesCase) Clean() error { + if err := r.deleteTestStorageClassList([]string{"e2e-storage-class", "e2e-storage-class-2"}); err != nil { + return err + } + + if err := DeleteConfigmap(r.Client.ClientGo, r.VeleroCfg.VeleroNamespace, r.cmName); err != nil { + return err + } + + return r.GetTestCase().Clean() +} + +func (r *ResourcePoliciesCase) createPVC(index int, namespace string, volList []*v1.Volume) error { + var err error + for i := range volList { + pvcName := fmt.Sprintf("pvc-%d", i) + By(fmt.Sprintf("Creating PVC %s in namespaces ...%s\n", pvcName, namespace)) + if index%3 == 0 { + pvcBuilder := NewPVC(namespace, pvcName).WithStorageClass("e2e-storage-class") // Testing sc should not backup + err = CreatePvc(r.Client, pvcBuilder) + } else if index%3 == 1 { + pvcBuilder := NewPVC(namespace, pvcName).WithStorageClass("e2e-storage-class-2") // Testing sc should backup + err = CreatePvc(r.Client, pvcBuilder) + } else if index%3 == 2 { + pvcBuilder := NewPVC(namespace, pvcName).WithStorageClass("e2e-storage-class-2").WithResourceStorage(resource.MustParse("2Gi")) // Testing capacity should not backup + err = CreatePvc(r.Client, pvcBuilder) + } + if err != nil { + return errors.Wrapf(err, "failed to create pvc %s in namespace %s", pvcName, namespace) + } + } + return nil +} + +func (r *ResourcePoliciesCase) createDeploymentWithVolume(namespace string, volList []*v1.Volume) error { + deployment := NewDeployment(r.NSBaseName, namespace, 1, map[string]string{"resource-policies": "resource-policies"}, nil).WithVolume(volList).Result() + deployment, err := CreateDeployment(r.Client.ClientGo, namespace, deployment) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to create deloyment %s the namespace %q", deployment.Name, namespace)) + } + err = WaitForReadyDeployment(r.Client.ClientGo, namespace, deployment.Name) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to wait for deployment %s to be ready in namespace: %q", deployment.Name, namespace)) + } + return nil +} + +func (r *ResourcePoliciesCase) writeDataIntoPods(namespace, volName string) error { + podList, err := ListPods(r.Ctx, r.Client, namespace) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to list pods in namespace: %q with error %v", namespace, err)) + } + for _, pod := range podList.Items { + for _, vol := range pod.Spec.Volumes { + if vol.Name != volName { + continue + } + err := CreateFileToPod(r.Ctx, namespace, pod.Name, "container-busybox", vol.Name, FileName, fmt.Sprintf("ns-%s pod-%s volume-%s", namespace, pod.Name, vol.Name)) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to create file into pod %s in namespace: %q", pod.Name, namespace)) + } + } + } + return nil +} + +func (r *ResourcePoliciesCase) deleteTestStorageClassList(scList []string) error { + for _, v := range scList { + if err := DeleteStorageClass(r.Ctx, r.Client, v); err != nil { + return err + } + } + return nil +} + +func (r *ResourcePoliciesCase) installTestStorageClasses(path string) error { + err := InstallStorageClass(r.Ctx, path) + if err != nil { + return err + } + content, err := ioutil.ReadFile(path) + if err != nil { + return errors.Wrapf(err, "failed to get %s when install storage class", path) + } + + // replace sc to new value + newContent := strings.ReplaceAll(string(content), "name: e2e-storage-class", "name: e2e-storage-class-2") + + tmpFile, err := ioutil.TempFile("", "sc-file") + if err != nil { + return errors.Wrapf(err, "failed to create temp file when install storage class") + } + + defer os.Remove(tmpFile.Name()) + if _, err := tmpFile.WriteString(newContent); err != nil { + return errors.Wrapf(err, "failed to write content into temp file %s when install storage class", tmpFile.Name()) + } + return InstallStorageClass(r.Ctx, tmpFile.Name()) +} diff --git a/test/e2e/schedule/ordered_resources.go b/test/e2e/schedule/ordered_resources.go index 166913c41..f220d9d44 100644 --- a/test/e2e/schedule/ordered_resources.go +++ b/test/e2e/schedule/ordered_resources.go @@ -159,7 +159,7 @@ func (o *OrderedResources) CreateResources() error { //Create deployment deploymentName := fmt.Sprintf("deploy-%s", o.NSBaseName) fmt.Printf("Creating deployment %s in %s namespaces ...\n", deploymentName, o.Namespace) - deployment := NewDeployment(deploymentName, o.Namespace, 1, label, nil) + deployment := NewDeployment(deploymentName, o.Namespace, 1, label, nil).Result() deployment, err := CreateDeployment(o.Client.ClientGo, o.Namespace, deployment) if err != nil { return errors.Wrap(err, fmt.Sprintf("failed to create namespace %q with err %v", o.Namespace, err)) diff --git a/test/e2e/util/k8s/common.go b/test/e2e/util/k8s/common.go index eb0ec43c6..694051e72 100644 --- a/test/e2e/util/k8s/common.go +++ b/test/e2e/util/k8s/common.go @@ -255,15 +255,30 @@ func GetPVByPodName(client TestClient, namespace, podName string) (string, error return pv_value.Name, nil } -func CreateFileToPod(ctx context.Context, namespace, podName, volume, filename, content string) error { - arg := []string{"exec", "-n", namespace, "-c", podName, podName, +func PrepareVolumeList(volumeNameList []string) (vols []*corev1.Volume) { + for i, volume := range volumeNameList { + vols = append(vols, &corev1.Volume{ + Name: volume, + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: fmt.Sprintf("pvc-%d", i), + ReadOnly: false, + }, + }, + }) + } + return +} + +func CreateFileToPod(ctx context.Context, namespace, podName, containerName, volume, filename, content string) error { + arg := []string{"exec", "-n", namespace, "-c", containerName, podName, "--", "/bin/sh", "-c", fmt.Sprintf("echo ns-%s pod-%s volume-%s > /%s/%s", namespace, podName, volume, volume, filename)} cmd := exec.CommandContext(ctx, "kubectl", arg...) fmt.Printf("Kubectl exec cmd =%v\n", cmd) return cmd.Run() } -func ReadFileFromPodVolume(ctx context.Context, namespace, podName, volume, filename string) (string, error) { - arg := []string{"exec", "-n", namespace, "-c", podName, podName, +func ReadFileFromPodVolume(ctx context.Context, namespace, podName, containerName, volume, filename string) (string, error) { + arg := []string{"exec", "-n", namespace, "-c", containerName, podName, "--", "cat", fmt.Sprintf("/%s/%s", volume, filename)} cmd := exec.CommandContext(ctx, "kubectl", arg...) fmt.Printf("Kubectl exec cmd =%v\n", cmd) diff --git a/test/e2e/util/k8s/configmap.go b/test/e2e/util/k8s/configmap.go index 0f311fa1d..e4edd0667 100644 --- a/test/e2e/util/k8s/configmap.go +++ b/test/e2e/util/k8s/configmap.go @@ -42,6 +42,20 @@ func CreateConfigMap(c clientset.Interface, ns, name string, labels, data map[st return c.CoreV1().ConfigMaps(ns).Create(context.TODO(), cm, metav1.CreateOptions{}) } +func CreateConfigMapFromYAMLData(c clientset.Interface, yamlData, cmName, namespace string) error { + cmData := make(map[string]string) + cmData[cmName] = yamlData + cm := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: cmName, + Namespace: namespace, + }, + Data: cmData, + } + _, err := c.CoreV1().ConfigMaps(namespace).Create(context.TODO(), cm, metav1.CreateOptions{}) + return err +} + // WaitForConfigMapComplete uses c to wait for completions to complete for the Job jobName in namespace ns. func WaitForConfigMapComplete(c clientset.Interface, ns, configmapName string) error { return wait.Poll(PollInterval, PollTimeout, func() (bool, error) { @@ -57,10 +71,18 @@ func GetConfigmap(c clientset.Interface, ns, secretName string) (*v1.ConfigMap, return c.CoreV1().ConfigMaps(ns).Get(context.TODO(), secretName, metav1.GetOptions{}) } -func WaitForConfigmapDelete(c clientset.Interface, ns, name string) error { +func DeleteConfigmap(c clientset.Interface, ns, name string) error { if err := c.CoreV1().ConfigMaps(ns).Delete(context.TODO(), name, metav1.DeleteOptions{}); err != nil { return errors.Wrap(err, fmt.Sprintf("failed to delete configmap in namespace %q", ns)) } + return nil +} + +func WaitForConfigmapDelete(c clientset.Interface, ns, name string) error { + if err := DeleteConfigmap(c, ns, name); err != nil { + return err + } + return waitutil.PollImmediateInfinite(5*time.Second, func() (bool, error) { if _, err := c.CoreV1().ConfigMaps(ns).Get(context.TODO(), ns, metav1.GetOptions{}); err != nil { diff --git a/test/e2e/util/k8s/deployment.go b/test/e2e/util/k8s/deployment.go index e9f5572fb..bc4c30a3d 100644 --- a/test/e2e/util/k8s/deployment.go +++ b/test/e2e/util/k8s/deployment.go @@ -35,46 +35,73 @@ const ( PollTimeout = 15 * time.Minute ) +// DeploymentBuilder builds Deployment objects. +type DeploymentBuilder struct { + *apps.Deployment +} + +func (d *DeploymentBuilder) Result() *apps.Deployment { + return d.Deployment +} + // newDeployment returns a RollingUpdate Deployment with a fake container image -func NewDeployment(name, ns string, replicas int32, labels map[string]string, containers []v1.Container) *apps.Deployment { +func NewDeployment(name, ns string, replicas int32, labels map[string]string, containers []v1.Container) *DeploymentBuilder { if containers == nil { containers = []v1.Container{ { - Name: fmt.Sprintf("container-%s", "busybox"), + Name: "container-busybox", Image: "gcr.io/velero-gcp/busybox:latest", Command: []string{"sleep", "1000000"}, }, } } - return &apps.Deployment{ - TypeMeta: metav1.TypeMeta{ - Kind: "Deployment", - APIVersion: "apps/v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: ns, - Name: name, - Labels: labels, - }, - Spec: apps.DeploymentSpec{ - Replicas: &replicas, - Selector: &metav1.LabelSelector{MatchLabels: labels}, - Strategy: apps.DeploymentStrategy{ - Type: apps.RollingUpdateDeploymentStrategyType, - RollingUpdate: new(apps.RollingUpdateDeployment), + return &DeploymentBuilder{ + &apps.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps/v1", }, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: labels, + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, + Name: name, + Labels: labels, + }, + Spec: apps.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{MatchLabels: labels}, + Strategy: apps.DeploymentStrategy{ + Type: apps.RollingUpdateDeploymentStrategyType, + RollingUpdate: new(apps.RollingUpdateDeployment), }, - Spec: v1.PodSpec{ - Containers: containers, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + Spec: v1.PodSpec{ + Containers: containers, + }, }, }, }, } } +func (d *DeploymentBuilder) WithVolume(vols []*v1.Volume) *DeploymentBuilder { + vmList := []v1.VolumeMount{} + for i, v := range vols { + vmList = append(vmList, v1.VolumeMount{ + Name: v.Name, + MountPath: "/" + v.Name, + }) + d.Spec.Template.Spec.Volumes = append(d.Spec.Template.Spec.Volumes, *vols[i]) + } + + // NOTE here just mount volumes to the first container + d.Spec.Template.Spec.Containers[0].VolumeMounts = vmList + + return d +} + func CreateDeploy(c clientset.Interface, ns string, deployment *apps.Deployment) error { _, err := c.AppsV1().Deployments(ns).Create(context.TODO(), deployment, metav1.CreateOptions{}) return err diff --git a/test/e2e/util/k8s/pod.go b/test/e2e/util/k8s/pod.go index a0daef03a..d730d6d5f 100644 --- a/test/e2e/util/k8s/pod.go +++ b/test/e2e/util/k8s/pod.go @@ -106,3 +106,7 @@ func AddAnnotationToPod(ctx context.Context, client TestClient, namespace, podNa return client.ClientGo.CoreV1().Pods(namespace).Update(ctx, newPod, metav1.UpdateOptions{}) } + +func ListPods(ctx context.Context, client TestClient, namespace string) (*corev1.PodList, error) { + return client.ClientGo.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{}) +} diff --git a/test/e2e/util/k8s/pvc.go b/test/e2e/util/k8s/pvc.go index 32e6b9528..428064e53 100644 --- a/test/e2e/util/k8s/pvc.go +++ b/test/e2e/util/k8s/pvc.go @@ -24,28 +24,67 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func CreatePVC(client TestClient, ns, name, sc string, ann map[string]string) (*corev1.PersistentVolumeClaim, error) { +// PVCBuilder builds PVC objects. +type PVCBuilder struct { + *corev1.PersistentVolumeClaim +} + +func (p *PVCBuilder) Result() *corev1.PersistentVolumeClaim { + return p.PersistentVolumeClaim +} + +func NewPVC(ns, name string) *PVCBuilder { oMeta := metav1.ObjectMeta{} - oMeta = metav1.ObjectMeta{Name: name} - if ann != nil { - oMeta.Annotations = ann - } - pvc := &corev1.PersistentVolumeClaim{ - ObjectMeta: oMeta, - Spec: corev1.PersistentVolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{ - corev1.ReadWriteOnce, - }, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse("1Gi"), + oMeta = metav1.ObjectMeta{Name: name, Namespace: ns} + return &PVCBuilder{ + &corev1.PersistentVolumeClaim{ + ObjectMeta: oMeta, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, // Default read write once + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), // Default 1Gi + }, }, }, - StorageClassName: &sc, }, } - return client.ClientGo.CoreV1().PersistentVolumeClaims(ns).Create(context.TODO(), pvc, metav1.CreateOptions{}) } + +func (p *PVCBuilder) WithAnnotation(ann map[string]string) *PVCBuilder { + p.Annotations = ann + return p +} + +func (p *PVCBuilder) WithStorageClass(sc string) *PVCBuilder { + p.Spec.StorageClassName = &sc + return p +} + +func (p *PVCBuilder) WithResourceStorage(q resource.Quantity) *PVCBuilder { + p.Spec.Resources.Requests[corev1.ResourceStorage] = q + return p +} + +func CreatePVC(client TestClient, ns, name, sc string, ann map[string]string) (*corev1.PersistentVolumeClaim, error) { + pvcBulder := NewPVC(ns, name) + if ann != nil { + pvcBulder.WithAnnotation(ann) + } + if sc != "" { + pvcBulder.WithStorageClass(sc) + } + + return client.ClientGo.CoreV1().PersistentVolumeClaims(ns).Create(context.TODO(), pvcBulder.Result(), metav1.CreateOptions{}) +} + +func CreatePvc(client TestClient, pvcBulder *PVCBuilder) error { + _, err := client.ClientGo.CoreV1().PersistentVolumeClaims(pvcBulder.Namespace).Create(context.TODO(), pvcBulder.Result(), metav1.CreateOptions{}) + return err +} + func GetPVC(ctx context.Context, client TestClient, namespace string, pvcName string) (*corev1.PersistentVolumeClaim, error) { return client.ClientGo.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{}) } diff --git a/test/e2e/util/k8s/sc.go b/test/e2e/util/k8s/sc.go index bcc5cbfed..3878018b2 100644 --- a/test/e2e/util/k8s/sc.go +++ b/test/e2e/util/k8s/sc.go @@ -3,6 +3,9 @@ package k8s import ( "context" "fmt" + + "github.com/pkg/errors" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func InstallStorageClass(ctx context.Context, yaml string) error { @@ -10,3 +13,10 @@ func InstallStorageClass(ctx context.Context, yaml string) error { err := KubectlApplyByFile(ctx, yaml) return err } + +func DeleteStorageClass(ctx context.Context, client TestClient, name string) error { + if err := client.ClientGo.StorageV1().StorageClasses().Delete(ctx, name, v1.DeleteOptions{}); err != nil { + return errors.Wrapf(err, "Could not retrieve storage classes %s", name) + } + return nil +}