Merge pull request #7396 from blackpiglet/backup_volumeinfo_e2e

Add E2E test cases for backup VolumeInfo feature.

cmd/velero-restore-helper/velero-restore-helper.go

@@ -73,7 +73,7 @@ func done() bool {
 			return false
 		}
 
-		fmt.Printf("Found %s", doneFile)
+		fmt.Printf("Found the done file %s\n", doneFile)
 	}
 
 	return true

test/e2e/basic/backup-volume-info/base.go

@@ -0,0 +1,148 @@
+/*
+Copyright the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package basic
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/pkg/errors"
+
+	. "github.com/vmware-tanzu/velero/test"
+	. "github.com/vmware-tanzu/velero/test/e2e/test"
+	. "github.com/vmware-tanzu/velero/test/util/k8s"
+	. "github.com/vmware-tanzu/velero/test/util/velero"
+)
+
+type BackupVolumeInfo struct {
+	TestCase
+	SnapshotVolumes          bool
+	DefaultVolumesToFSBackup bool
+	SnapshotMoveData         bool
+	TimeoutDuration          time.Duration
+}
+
+func (v *BackupVolumeInfo) Init() error {
+	v.TestCase.Init()
+
+	BeforeEach(func() {
+		if v.VeleroCfg.CloudProvider == "vsphere" && (!strings.Contains(v.CaseBaseName, "fs-upload") && !strings.Contains(v.CaseBaseName, "skipped")) {
+			fmt.Printf("Skip snapshot case %s for vsphere environment.\n", v.CaseBaseName)
+			Skip("Skip snapshot case due to vsphere environment doesn't cover the CSI test, and it doesn't have a Velero native snapshot plugin.")
+		}
+
+		if strings.Contains(v.VeleroCfg.Features, "EnableCSI") {
+			if strings.Contains(v.CaseBaseName, "native-snapshot") {
+				fmt.Printf("Skip native snapshot case %s when the CSI feature is enabled.\n", v.CaseBaseName)
+				Skip("Skip native snapshot case due to CSI feature is enabled.")
+			}
+		} else {
+			if strings.Contains(v.CaseBaseName, "csi") {
+				fmt.Printf("Skip CSI related case %s when the CSI feature is not enabled.\n", v.CaseBaseName)
+				Skip("Skip CSI cases due to CSI feature is not enabled.")
+			}
+		}
+	})
+
+	v.CaseBaseName = v.CaseBaseName + v.UUIDgen
+	v.BackupName = "backup-" + v.CaseBaseName
+	v.RestoreName = "restore-" + v.CaseBaseName
+	v.TimeoutDuration = 10 * time.Minute
+	v.NamespacesTotal = 1
+
+	v.VeleroCfg = VeleroCfg
+	v.Client = *v.VeleroCfg.ClientToInstallVelero
+	v.NSIncluded = &[]string{v.CaseBaseName}
+
+	v.TestMsg = &TestMSG{
+		Desc:      "Test backup's VolumeInfo metadata content",
+		Text:      "The VolumeInfo should be generated based on the backup type",
+		FailedMSG: "Failed to verify the backup VolumeInfo's content",
+	}
+
+	v.BackupArgs = []string{
+		"backup", "create", v.BackupName,
+		"--namespace", v.VeleroCfg.VeleroNamespace,
+		"--include-namespaces", v.CaseBaseName,
+		"--snapshot-volumes" + "=" + strconv.FormatBool(v.SnapshotVolumes),
+		"--default-volumes-to-fs-backup" + "=" + strconv.FormatBool(v.DefaultVolumesToFSBackup),
+		"--snapshot-move-data" + "=" + strconv.FormatBool(v.SnapshotMoveData),
+		"--wait",
+	}
+
+	v.RestoreArgs = []string{
+		"create", "--namespace", v.VeleroCfg.VeleroNamespace, "restore", v.RestoreName,
+		"--from-backup", v.BackupName, "--wait",
+	}
+	return nil
+}
+
+func (v *BackupVolumeInfo) CreateResources() error {
+	v.Ctx, v.CtxCancel = context.WithTimeout(context.Background(), v.TimeoutDuration)
+	labels := map[string]string{
+		"volume-info": "true",
+	}
+	for nsNum := 0; nsNum < v.NamespacesTotal; nsNum++ {
+		fmt.Printf("Creating namespaces ...\n")
+		createNSName := v.CaseBaseName
+		if err := CreateNamespaceWithLabel(v.Ctx, v.Client, createNSName, labels); err != nil {
+			return errors.Wrapf(err, "Failed to create namespace %s", createNSName)
+		}
+
+		// Install StorageClass
+		Expect(InstallTestStorageClasses(fmt.Sprintf("../testdata/storage-class/%s-csi.yaml", v.VeleroCfg.CloudProvider))).To(Succeed(), "Failed to install StorageClass")
+
+		pvc, err := CreatePVC(v.Client, createNSName, "volume-info", CSIStorageClassName, nil)
+		Expect(err).To(Succeed())
+		vols := CreateVolumes(pvc.Name, []string{"volume-info"})
+
+		//Create deployment
+		fmt.Printf("Creating deployment in namespaces ...%s\n", createNSName)
+		deployment := NewDeployment(v.CaseBaseName, createNSName, 1, labels, nil).WithVolume(vols).Result()
+		deployment, err = CreateDeployment(v.Client.ClientGo, createNSName, deployment)
+		if err != nil {
+			return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", createNSName))
+		}
+		err = WaitForReadyDeployment(v.Client.ClientGo, createNSName, deployment.Name)
+		if err != nil {
+			return errors.Wrap(err, fmt.Sprintf("failed to ensure job completion in namespace: %q", createNSName))
+		}
+	}
+	return nil
+}
+
+func (v *BackupVolumeInfo) Destroy() error {
+	err := CleanupNamespaces(v.Ctx, v.Client, v.CaseBaseName)
+	if err != nil {
+		return errors.Wrap(err, "Could cleanup retrieve namespaces")
+	}
+
+	return WaitAllSelectedNSDeleted(v.Ctx, v.Client, "ns-test=true")
+}
+
+func (v *BackupVolumeInfo) cleanResource() error {
+	if err := DeleteStorageClass(v.Ctx, v.Client, CSIStorageClassName); err != nil {
+		return errors.Wrap(err, "fail to delete the StorageClass")
+	}
+
+	return nil
+}

test/e2e/basic/backup-volume-info/csi_data_mover.go

@@ -0,0 +1,66 @@
+/*
+Copyright the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package basic
+
+import (
+	"fmt"
+
+	. "github.com/onsi/gomega"
+
+	. "github.com/vmware-tanzu/velero/test/e2e/test"
+	. "github.com/vmware-tanzu/velero/test/util/providers"
+	. "github.com/vmware-tanzu/velero/test/util/velero"
+)
+
+var CSIDataMoverVolumeInfoTest func() = TestFunc(&CSIDataMoverVolumeInfo{
+	BackupVolumeInfo{
+		SnapshotMoveData: true,
+		SnapshotVolumes:  true,
+		TestCase: TestCase{
+			CaseBaseName: "csi-data-mover-volumeinfo",
+			TestMsg: &TestMSG{
+				Desc: "Test backup's VolumeInfo metadata content for CSI data mover case.",
+				Text: "The VolumeInfo should be generated, and the SnapshotDataMovementInfo structure should not be nil.",
+			},
+		},
+	},
+})
+
+type CSIDataMoverVolumeInfo struct {
+	BackupVolumeInfo
+}
+
+func (c *CSIDataMoverVolumeInfo) Verify() error {
+	volumeInfo, err := GetVolumeInfo(
+		c.VeleroCfg.ObjectStoreProvider,
+		c.VeleroCfg.CloudCredentialsFile,
+		c.VeleroCfg.BSLBucket,
+		c.VeleroCfg.BSLPrefix,
+		c.VeleroCfg.BSLConfig,
+		c.BackupName,
+		BackupObjectsPrefix+"/"+c.BackupName,
+	)
+
+	Expect(err).ShouldNot(HaveOccurred(), fmt.Sprintf("Fail to get VolumeInfo metadata in the Backup Repository."))
+
+	fmt.Printf("The VolumeInfo metadata content: %+v\n", *volumeInfo[0])
+	Expect(len(volumeInfo) > 0).To(BeIdenticalTo(true))
+	Expect(volumeInfo[0].SnapshotDataMovementInfo).NotTo(BeNil())
+
+	// Clean SC and VSC
+	return c.cleanResource()
+}

test/e2e/basic/backup-volume-info/csi_snapshot.go

@@ -0,0 +1,65 @@
+/*
+Copyright the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package basic
+
+import (
+	"fmt"
+
+	. "github.com/onsi/gomega"
+
+	. "github.com/vmware-tanzu/velero/test/e2e/test"
+	. "github.com/vmware-tanzu/velero/test/util/providers"
+	. "github.com/vmware-tanzu/velero/test/util/velero"
+)
+
+var CSISnapshotVolumeInfoTest func() = TestFunc(&CSISnapshotVolumeInfo{
+	BackupVolumeInfo{
+		SnapshotVolumes: true,
+		TestCase: TestCase{
+			CaseBaseName: "csi-snapshot-volumeinfo",
+			TestMsg: &TestMSG{
+				Desc: "Test backup's VolumeInfo metadata content for CSI snapshot case.",
+				Text: "The VolumeInfo should be generated, and the CSISnapshotInfo structure should not be nil.",
+			},
+		},
+	},
+})
+
+type CSISnapshotVolumeInfo struct {
+	BackupVolumeInfo
+}
+
+func (c *CSISnapshotVolumeInfo) Verify() error {
+	volumeInfo, err := GetVolumeInfo(
+		c.VeleroCfg.ObjectStoreProvider,
+		c.VeleroCfg.CloudCredentialsFile,
+		c.VeleroCfg.BSLBucket,
+		c.VeleroCfg.BSLPrefix,
+		c.VeleroCfg.BSLConfig,
+		c.BackupName,
+		BackupObjectsPrefix+"/"+c.BackupName,
+	)
+
+	Expect(err).ShouldNot(HaveOccurred(), fmt.Sprintf("Fail to get VolumeInfo metadata in the Backup Repository."))
+
+	fmt.Printf("The VolumeInfo metadata content: %+v\n", *volumeInfo[0])
+	Expect(len(volumeInfo) > 0).To(BeIdenticalTo(true))
+	Expect(volumeInfo[0].CSISnapshotInfo).NotTo(BeNil())
+
+	// Clean SC and VSC
+	return c.cleanResource()
+}

test/e2e/basic/backup-volume-info/filesystem_upload.go

@@ -0,0 +1,65 @@
+/*
+Copyright the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package basic
+
+import (
+	"fmt"
+
+	. "github.com/onsi/gomega"
+
+	. "github.com/vmware-tanzu/velero/test/e2e/test"
+	. "github.com/vmware-tanzu/velero/test/util/providers"
+	. "github.com/vmware-tanzu/velero/test/util/velero"
+)
+
+var FilesystemUploadVolumeInfoTest func() = TestFunc(&FilesystemUploadVolumeInfo{
+	BackupVolumeInfo{
+		DefaultVolumesToFSBackup: true,
+		TestCase: TestCase{
+			CaseBaseName: "fs-upload-volumeinfo",
+			TestMsg: &TestMSG{
+				Desc: "Test backup's VolumeInfo metadata content for filesystem upload case.",
+				Text: "The VolumeInfo should be generated, and the PVBInfo structure should not be nil.",
+			},
+		},
+	},
+})
+
+type FilesystemUploadVolumeInfo struct {
+	BackupVolumeInfo
+}
+
+func (f *FilesystemUploadVolumeInfo) Verify() error {
+	volumeInfo, err := GetVolumeInfo(
+		f.VeleroCfg.ObjectStoreProvider,
+		f.VeleroCfg.CloudCredentialsFile,
+		f.VeleroCfg.BSLBucket,
+		f.VeleroCfg.BSLPrefix,
+		f.VeleroCfg.BSLConfig,
+		f.BackupName,
+		BackupObjectsPrefix+"/"+f.BackupName,
+	)
+
+	Expect(err).ShouldNot(HaveOccurred(), fmt.Sprintf("Fail to get VolumeInfo metadata in the Backup Repository."))
+
+	fmt.Printf("The VolumeInfo metadata content: %+v\n", *volumeInfo[0])
+	Expect(len(volumeInfo) > 0).To(BeIdenticalTo(true))
+	Expect(volumeInfo[0].PVBInfo).NotTo(BeNil())
+
+	// Clean SC and VSC
+	return f.cleanResource()
+}

test/e2e/basic/backup-volume-info/native_snapshot.go

@@ -0,0 +1,65 @@
+/*
+Copyright the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package basic
+
+import (
+	"fmt"
+
+	. "github.com/onsi/gomega"
+
+	. "github.com/vmware-tanzu/velero/test/e2e/test"
+	. "github.com/vmware-tanzu/velero/test/util/providers"
+	. "github.com/vmware-tanzu/velero/test/util/velero"
+)
+
+var NativeSnapshotVolumeInfoTest func() = TestFunc(&NativeSnapshotVolumeInfo{
+	BackupVolumeInfo{
+		SnapshotVolumes: true,
+		TestCase: TestCase{
+			CaseBaseName: "native-snapshot-volumeinfo",
+			TestMsg: &TestMSG{
+				Desc: "Test backup's VolumeInfo metadata content for native snapshot case.",
+				Text: "The VolumeInfo should be generated, and the NativeSnapshotInfo structure should not be nil.",
+			},
+		},
+	},
+})
+
+type NativeSnapshotVolumeInfo struct {
+	BackupVolumeInfo
+}
+
+func (n *NativeSnapshotVolumeInfo) Verify() error {
+	volumeInfo, err := GetVolumeInfo(
+		n.VeleroCfg.ObjectStoreProvider,
+		n.VeleroCfg.CloudCredentialsFile,
+		n.VeleroCfg.BSLBucket,
+		n.VeleroCfg.BSLPrefix,
+		n.VeleroCfg.BSLConfig,
+		n.BackupName,
+		BackupObjectsPrefix+"/"+n.BackupName,
+	)
+
+	Expect(err).ShouldNot(HaveOccurred(), fmt.Sprintf("Fail to get VolumeInfo metadata in the Backup Repository."))
+
+	fmt.Printf("The VolumeInfo metadata content: %+v\n", *volumeInfo[0])
+	Expect(len(volumeInfo) > 0).To(BeIdenticalTo(true))
+	Expect(volumeInfo[0].NativeSnapshotInfo).NotTo(BeNil())
+
+	// Clean SC and VSC
+	return n.cleanResource()
+}

test/e2e/basic/backup-volume-info/skipped_volumes.go

@@ -0,0 +1,65 @@
+/*
+Copyright the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package basic
+
+import (
+	"fmt"
+
+	. "github.com/onsi/gomega"
+
+	. "github.com/vmware-tanzu/velero/test/e2e/test"
+	. "github.com/vmware-tanzu/velero/test/util/providers"
+	. "github.com/vmware-tanzu/velero/test/util/velero"
+)
+
+var SkippedVolumeInfoTest func() = TestFunc(&SkippedVolumeInfo{
+	BackupVolumeInfo{
+		SnapshotVolumes: false,
+		TestCase: TestCase{
+			CaseBaseName: "skipped-volumes-volumeinfo",
+			TestMsg: &TestMSG{
+				Desc: "Test backup's VolumeInfo metadata content for volume-skipped case.",
+				Text: "The VolumeInfo should be generated, and the Skipped parameter should be true.",
+			},
+		},
+	},
+})
+
+type SkippedVolumeInfo struct {
+	BackupVolumeInfo
+}
+
+func (s *SkippedVolumeInfo) Verify() error {
+	volumeInfo, err := GetVolumeInfo(
+		s.VeleroCfg.ObjectStoreProvider,
+		s.VeleroCfg.CloudCredentialsFile,
+		s.VeleroCfg.BSLBucket,
+		s.VeleroCfg.BSLPrefix,
+		s.VeleroCfg.BSLConfig,
+		s.BackupName,
+		BackupObjectsPrefix+"/"+s.BackupName,
+	)
+
+	Expect(err).ShouldNot(HaveOccurred(), fmt.Sprintf("Fail to get VolumeInfo metadata in the Backup Repository."))
+
+	fmt.Printf("The VolumeInfo metadata content: %+v\n", *volumeInfo[0])
+	Expect(len(volumeInfo) > 0).To(BeIdenticalTo(true))
+	Expect(volumeInfo[0].Skipped == true).To(BeIdenticalTo(true))
+
+	// Clean SC and VSC
+	return s.cleanResource()
+}

test/e2e/e2e_suite_test.go

@@ -34,6 +34,7 @@ import (
 	. "github.com/vmware-tanzu/velero/test/e2e/backups"
 	. "github.com/vmware-tanzu/velero/test/e2e/basic"
 	. "github.com/vmware-tanzu/velero/test/e2e/basic/api-group"
+	. "github.com/vmware-tanzu/velero/test/e2e/basic/backup-volume-info"
 	. "github.com/vmware-tanzu/velero/test/e2e/basic/resources-check"
 	. "github.com/vmware-tanzu/velero/test/e2e/bsl-mgmt"
 	. "github.com/vmware-tanzu/velero/test/e2e/migration"
@@ -136,6 +137,13 @@ var _ = Describe("[ResourceFiltering][IncludeResources][Restore] Velero test on
 var _ = Describe("[ResourceFiltering][LabelSelector] Velero test on backup include resources matching the label selector", BackupWithLabelSelector)
 var _ = Describe("[ResourceFiltering][ResourcePolicies][Restic] Velero test on skip backup of volume by resource policies", ResourcePoliciesTest)
 
+// backup VolumeInfo test
+var _ = Describe("[BackupVolumeInfo][SkippedVolume]", SkippedVolumeInfoTest)
+var _ = Describe("[BackupVolumeInfo][FilesystemUpload]", FilesystemUploadVolumeInfoTest)
+var _ = Describe("[BackupVolumeInfo][CSIDataMover]", CSIDataMoverVolumeInfoTest)
+var _ = Describe("[BackupVolumeInfo][CSISnapshot]", CSISnapshotVolumeInfoTest)
+var _ = Describe("[BackupVolumeInfo][NativeSnapshot]", NativeSnapshotVolumeInfoTest)
+
 var _ = Describe("[ResourceModifier][Restore] Velero test on resource modifiers from the cluster restore", ResourceModifiersTest)
 
 var _ = Describe("[Backups][Deletion][Restic] Velero tests of Restic backup deletion", BackupDeletionWithRestic)

test/testdata/storage-class/aws-csi.yaml

@@ -0,0 +1,9 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: e2e-csi-storage-class
+provisioner: ebs.csi.aws.com
+parameters:
+  type: gp2
+reclaimPolicy: Delete
+volumeBindingMode: WaitForFirstConsumer

test/testdata/storage-class/azure-csi.yaml

@@ -0,0 +1,11 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: e2e-csi-storage-class
+provisioner: disk.csi.azure.com
+parameters:
+  cachingmode: ReadOnly
+  kind: Managed
+  storageaccounttype: StandardSSD_LRS
+reclaimPolicy: Delete
+volumeBindingMode: WaitForFirstConsumer

test/testdata/storage-class/gcp-csi.yaml

@@ -0,0 +1,13 @@
+allowVolumeExpansion: true
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  labels:
+    addonmanager.kubernetes.io/mode: EnsureExists
+  name: e2e-csi-storage-class
+parameters:
+  type: pd-standard
+provisioner: pd.csi.storage.gke.io
+reclaimPolicy: Delete
+volumeBindingMode: WaitForFirstConsumer
+

test/testdata/storage-class/vsphere-csi.yaml

@@ -0,0 +1,11 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: e2e-csi-storage-class
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "false"
+parameters:
+  StoragePolicyName: "vSAN Default Storage Policy"
+provisioner: csi.vsphere.vmware.com
+reclaimPolicy: Delete
+volumeBindingMode: WaitForFirstConsumer

test/testdata/volume-snapshot-class/aws.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: snapshot.storage.k8s.io/v1
+deletionPolicy: Delete
+driver: ebs.csi.aws.com
+kind: VolumeSnapshotClass
+metadata:
+  labels:
+    velero.io/csi-volumesnapshot-class: "true"
+  name: e2e-volume-snapshot-class

test/testdata/volume-snapshot-class/azure.yaml

@@ -1,8 +1,9 @@
-apiVersion: snapshot.storage.k8s.io/v1
-kind: VolumeSnapshotClass
-metadata:
-  name: velero
-  labels:
-    velero.io/csi-volumesnapshot-class: "true"
-driver: disk.csi.azure.com
-deletionPolicy: Delete
+---
+apiVersion: snapshot.storage.k8s.io/v1
+deletionPolicy: Delete
+driver: disk.csi.azure.com
+kind: VolumeSnapshotClass
+metadata:
+  labels:
+    velero.io/csi-volumesnapshot-class: "true"
+  name: e2e-volume-snapshot-class

test/testdata/volume-snapshot-class/gcp.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: snapshot.storage.k8s.io/v1
+deletionPolicy: Delete
+driver: pd.csi.storage.gke.io
+kind: VolumeSnapshotClass
+metadata:
+  labels:
+    velero.io/csi-volumesnapshot-class: "true"
+  name: e2e-volume-snapshot-class

test/types.go

@@ -27,6 +27,7 @@ import (
 
 const StorageClassName = "e2e-storage-class"
 const StorageClassName2 = "e2e-storage-class-2"
+const CSIStorageClassName = "e2e-csi-storage-class"
 const FeatureCSI = "EnableCSI"
 
 var InstallVelero bool

test/util/k8s/common.go

@@ -204,6 +204,12 @@ func KubectlApplyByFile(ctx context.Context, file string) error {
 	return exec.CommandContext(ctx, "kubectl", args...).Run()
 }
 
+func KubectlDeleteByFile(ctx context.Context, file string) error {
+	args := []string{"delete", "-f", file, "--force=true"}
+	fmt.Println(args)
+	return exec.CommandContext(ctx, "kubectl", args...).Run()
+}
+
 func KubectlConfigUseContext(ctx context.Context, kubectlContext string) error {
 	cmd := exec.CommandContext(ctx, "kubectl",
 		"config", "use-context", kubectlContext)

test/util/providers/aws_utils.go

@@ -20,6 +20,7 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"fmt"
+	"io"
 	"net/http"
 	"net/url"
 	"os"
@@ -415,3 +416,56 @@ func (s AWSStorage) GetMinioBucketSize(cloudCredentialsFile, bslBucket, bslPrefi
 	}
 	return totalSize, nil
 }
+
+func (s AWSStorage) GetObject(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, objectKey string) (io.ReadCloser, error) {
+	config := flag.NewMap()
+	config.Set(bslConfig)
+	objectsInput := s3.ListObjectsV2Input{}
+	objectsInput.Bucket = aws.String(bslBucket)
+	objectsInput.Delimiter = aws.String("/")
+
+	if bslPrefix != "" {
+		objectsInput.Prefix = aws.String(bslPrefix)
+	}
+
+	var err error
+	var s3Config aws.Config
+	var s3Client *s3.Client
+	region := config.Data()["region"]
+	s3url := ""
+	if region == "" {
+		region, err = GetBucketRegion(bslBucket)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to get region for bucket %s", bslBucket)
+		}
+	}
+	if region == "minio" {
+		s3url = config.Data()["s3Url"]
+		s3Config, err = newAWSConfig(region, "", cloudCredentialsFile, true, "")
+		if err != nil {
+			return nil, errors.Wrapf(err, "Failed to create AWS config of region %s", region)
+		}
+		s3Client, err = newS3Client(s3Config, s3url, true)
+	} else {
+		s3Config, err = newAWSConfig(region, "", cloudCredentialsFile, false, "")
+		if err != nil {
+			return nil, errors.Wrapf(err, "Failed to create AWS config of region %s", region)
+		}
+		s3Client, err = newS3Client(s3Config, s3url, true)
+	}
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to create S3 client of region %s", region)
+	}
+
+	fullObjectKey := strings.Trim(bslPrefix, "/") + "/" + strings.Trim(objectKey, "/")
+
+	result, err := s3Client.GetObject(context.TODO(), &s3.GetObjectInput{
+		Bucket: aws.String(bslBucket),
+		Key:    aws.String(fullObjectKey),
+	})
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to get object %s", fullObjectKey)
+	}
+
+	return result.Body, nil
+}

test/util/providers/azure_utils.go

@@ -18,6 +18,7 @@ package providers
 
 import (
 	"fmt"
+	"io"
 	"log"
 	"net/url"
 	"os"
@@ -364,3 +365,32 @@ func (s AzureStorage) IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupN
 		return nil
 	}
 }
+
+func (s AzureStorage) GetObject(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, objectKey string) (io.ReadCloser, error) {
+	ctx := context.Background()
+	accountName, accountKey, err := getStorageCredential(cloudCredentialsFile, bslConfig)
+	if err != nil {
+		return nil, errors.Wrapf(err, "Fail to get storage account name and  key of bucket %s", bslBucket)
+	}
+
+	credential, err := azblob.NewSharedKeyCredential(accountName, accountKey)
+	if err != nil {
+		log.Fatal("Invalid credentials with error: " + err.Error())
+	}
+	p := azblob.NewPipeline(credential, azblob.PipelineOptions{})
+
+	URL, _ := url.Parse(
+		fmt.Sprintf("https://%s.blob.core.windows.net/%s", accountName, bslBucket))
+
+	containerURL := azblob.NewContainerURL(*URL, p)
+	_, err = containerURL.Create(ctx, azblob.Metadata{}, azblob.PublicAccessNone)
+	handleErrors(err)
+
+	blobURL := containerURL.NewBlockBlobURL(strings.Join([]string{bslPrefix, objectKey}, "/"))
+	downloadResponse, err := blobURL.Download(ctx, 0, 0, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{})
+	if err != nil {
+		handleErrors(err)
+	}
+
+	return downloadResponse.Body(azblob.RetryReaderOptions{}), nil
+}

test/util/providers/common.go

@@ -17,13 +17,17 @@ limitations under the License.
 package providers
 
 import (
+	"compress/gzip"
 	"context"
+	"encoding/json"
 	"fmt"
+	"io"
 	"strings"
 	"time"
 
 	"github.com/pkg/errors"
 
+	"github.com/vmware-tanzu/velero/internal/volume"
 	. "github.com/vmware-tanzu/velero/test"
 	velero "github.com/vmware-tanzu/velero/test/util/velero"
 )
@@ -32,6 +36,7 @@ type ObjectsInStorage interface {
 	IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) (bool, error)
 	DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) error
 	IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupName string, snapshotCheck SnapshotCheckPoint) error
+	GetObject(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, objectKey string) (io.ReadCloser, error)
 }
 
 func ObjectsShouldBeInBucket(objectStoreProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) error {
@@ -189,3 +194,59 @@ func IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig
 	}
 	return nil
 }
+
+func GetVolumeInfoMetadataContent(
+	objectStoreProvider,
+	cloudCredentialsFile,
+	bslBucket,
+	bslPrefix,
+	bslConfig,
+	backupName,
+	subPrefix string,
+) (io.Reader, error) {
+	bslPrefix = strings.Trim(getFullPrefix(bslPrefix, subPrefix), "/")
+	volumeFileName := backupName + "-volumeinfo.json.gz"
+	fmt.Printf("|| VERIFICATION || - Get backup %s volumeinfo file in storage %s\n", backupName, bslPrefix)
+	s, err := getProvider(objectStoreProvider)
+	if err != nil {
+		return nil, errors.Wrapf(err, fmt.Sprintf("Cloud provider %s is not valid", objectStoreProvider))
+	}
+
+	return s.GetObject(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, volumeFileName)
+}
+
+func GetVolumeInfo(
+	objectStoreProvider,
+	cloudCredentialsFile,
+	bslBucket,
+	bslPrefix,
+	bslConfig,
+	backupName,
+	subPrefix string,
+) ([]*volume.VolumeInfo, error) {
+	readCloser, err := GetVolumeInfoMetadataContent(objectStoreProvider,
+		cloudCredentialsFile,
+		bslBucket,
+		bslPrefix,
+		bslConfig,
+		backupName,
+		subPrefix,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	gzr, err := gzip.NewReader(readCloser)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+	defer gzr.Close()
+
+	volumeInfos := make([]*volume.VolumeInfo, 0)
+
+	if err := json.NewDecoder(gzr).Decode(&volumeInfos); err != nil {
+		return nil, errors.Wrap(err, "error decoding object data")
+	}
+
+	return volumeInfos, nil
+}

test/util/providers/gcloud_utils.go

@@ -19,6 +19,7 @@ package providers
 import (
 	"encoding/json"
 	"fmt"
+	"io"
 	"os"
 	"strings"
 
@@ -65,6 +66,7 @@ func (s GCSStorage) IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix
 		}
 	}
 }
+
 func (s GCSStorage) DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) error {
 	q := &storage.Query{
 		Prefix: bslPrefix,
@@ -141,3 +143,13 @@ func (s GCSStorage) IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupObj
 		return nil
 	}
 }
+
+func (s GCSStorage) GetObject(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, objectKey string) (io.ReadCloser, error) {
+	ctx := context.Background()
+	client, err := storage.NewClient(ctx, option.WithCredentialsFile(cloudCredentialsFile))
+	if err != nil {
+		return nil, errors.Wrapf(err, "fail to create GCloud client")
+	}
+
+	return client.Bucket(bslBucket).Object(strings.Join([]string{bslPrefix, objectKey}, "/")).NewReader(ctx)
+}

test/util/velero/install.go

@@ -280,9 +280,10 @@ func installVeleroServer(ctx context.Context, cli, cloudProvider string, options
 
 	if len(options.Features) > 0 {
 		args = append(args, "--features", options.Features)
-		if strings.EqualFold(cloudProvider, "azure") && strings.EqualFold(options.Features, FeatureCSI) && options.UseVolumeSnapshots {
+		if !strings.EqualFold(cloudProvider, "vsphere") && strings.EqualFold(options.Features, FeatureCSI) && options.UseVolumeSnapshots {
 			fmt.Println("Start to install Azure VolumeSnapshotClass ...")
-			if err := KubectlApplyByFile(ctx, "../util/csi/AzureVolumeSnapshotClass.yaml"); err != nil {
+			if err := KubectlApplyByFile(ctx, fmt.Sprintf("../testdata/volume-snapshot-class/%s.yaml", cloudProvider)); err != nil {
+				fmt.Println("Fail to install VolumeSnapshotClass when CSI feature is enabled: ", err)
 				return err
 			}
 		}
@@ -634,6 +635,7 @@ func VeleroUninstall(ctx context.Context, cli, namespace string) error {
 		return errors.Wrapf(err, "failed to uninstall velero, stdout=%s, stderr=%s", stdout, stderr)
 	}
 	fmt.Println("Velero uninstalled ⛵")
+
 	return nil
 }