From ef5c2ed8056655975224c2a46db93e2194748607 Mon Sep 17 00:00:00 2001 From: Xun Jiang Date: Wed, 21 Feb 2024 10:05:50 +0800 Subject: [PATCH] Modify according to comments. Signed-off-by: Xun Jiang --- test/e2e/basic/backup-volume-info/base.go | 16 ++++++++-------- .../basic/backup-volume-info/csi_data_mover.go | 8 +++++++- .../e2e/basic/backup-volume-info/csi_snapshot.go | 8 +++++++- .../backup-volume-info/filesystem_upload.go | 8 +++++++- .../basic/backup-volume-info/native_snapshot.go | 8 +++++++- .../basic/backup-volume-info/skipped_volumes.go | 8 +++++++- test/testdata/storage-class/vsphere-csi.yaml | 11 +++++++++++ test/util/csi/AzureVolumeSnapshotClass.yaml | 8 -------- test/util/providers/common.go | 10 +++++----- test/util/velero/install.go | 6 ++++-- 10 files changed, 63 insertions(+), 28 deletions(-) create mode 100644 test/testdata/storage-class/vsphere-csi.yaml delete mode 100644 test/util/csi/AzureVolumeSnapshotClass.yaml diff --git a/test/e2e/basic/backup-volume-info/base.go b/test/e2e/basic/backup-volume-info/base.go index 8b60e3b03..a10e75a05 100644 --- a/test/e2e/basic/backup-volume-info/base.go +++ b/test/e2e/basic/backup-volume-info/base.go @@ -45,15 +45,20 @@ func (v *BackupVolumeInfo) Init() error { v.TestCase.Init() BeforeEach(func() { + if v.VeleroCfg.CloudProvider == "vsphere" && (!strings.Contains(v.CaseBaseName, "fs-upload") && !strings.Contains(v.CaseBaseName, "skipped")) { + fmt.Printf("Skip snapshot case %s for vsphere environment.\n", v.CaseBaseName) + Skip("Skip snapshot case due to vsphere environment doesn't cover the CSI test, and it doesn't have a Velero native snapshot plugin.") + } + if strings.Contains(v.VeleroCfg.Features, "EnableCSI") { if strings.Contains(v.CaseBaseName, "native-snapshot") { fmt.Printf("Skip native snapshot case %s when the CSI feature is enabled.\n", v.CaseBaseName) - Skip("Skip due to vSphere CSI driver long time issue of Static provisioning") + Skip("Skip native snapshot case due to CSI feature is enabled.") } } else { if strings.Contains(v.CaseBaseName, "csi") { fmt.Printf("Skip CSI related case %s when the CSI feature is not enabled.\n", v.CaseBaseName) - Skip("Skip due to vSphere CSI driver long time issue of Static provisioning") + Skip("Skip CSI cases due to CSI feature is not enabled.") } } }) @@ -103,11 +108,9 @@ func (v *BackupVolumeInfo) CreateResources() error { return errors.Wrapf(err, "Failed to create namespace %s", createNSName) } + // Install StorageClass Expect(InstallTestStorageClasses(fmt.Sprintf("../testdata/storage-class/%s-csi.yaml", v.VeleroCfg.CloudProvider))).To(Succeed(), "Failed to install StorageClass") - // Install VolumeSnapshotClass - Expect(KubectlApplyByFile(v.Ctx, fmt.Sprintf("../testdata/volume-snapshot-class/%s.yaml", v.VeleroCfg.CloudProvider))).To(Succeed(), "Failed to install VolumeSnapshotClass") - pvc, err := CreatePVC(v.Client, createNSName, "volume-info", CSIStorageClassName, nil) Expect(err).To(Succeed()) vols := CreateVolumes(pvc.Name, []string{"volume-info"}) @@ -141,8 +144,5 @@ func (v *BackupVolumeInfo) cleanResource() error { return errors.Wrap(err, "fail to delete the StorageClass") } - if err := KubectlDeleteByFile(v.Ctx, fmt.Sprintf("../testdata/volume-snapshot-class/%s.yaml", v.VeleroCfg.CloudProvider)); err != nil { - return errors.Wrap(err, "fail to delete the VolumeSnapshotClass") - } return nil } diff --git a/test/e2e/basic/backup-volume-info/csi_data_mover.go b/test/e2e/basic/backup-volume-info/csi_data_mover.go index b18412ef7..a93f4a60c 100644 --- a/test/e2e/basic/backup-volume-info/csi_data_mover.go +++ b/test/e2e/basic/backup-volume-info/csi_data_mover.go @@ -32,6 +32,10 @@ var CSIDataMoverVolumeInfoTest func() = TestFunc(&CSIDataMoverVolumeInfo{ SnapshotVolumes: true, TestCase: TestCase{ CaseBaseName: "csi-data-mover-volumeinfo", + TestMsg: &TestMSG{ + Desc: "Test backup's VolumeInfo metadata content for CSI data mover case.", + Text: "The VolumeInfo should be generated, and the SnapshotDataMovementInfo structure should not be nil.", + }, }, }, }) @@ -42,7 +46,7 @@ type CSIDataMoverVolumeInfo struct { func (c *CSIDataMoverVolumeInfo) Verify() error { volumeInfo, err := GetVolumeInfo( - c.VeleroCfg.CloudProvider, + c.VeleroCfg.ObjectStoreProvider, c.VeleroCfg.CloudCredentialsFile, c.VeleroCfg.BSLBucket, c.VeleroCfg.BSLPrefix, @@ -52,6 +56,8 @@ func (c *CSIDataMoverVolumeInfo) Verify() error { ) Expect(err).ShouldNot(HaveOccurred(), fmt.Sprintf("Fail to get VolumeInfo metadata in the Backup Repository.")) + + fmt.Printf("The VolumeInfo metadata content: %+v\n", *volumeInfo[0]) Expect(len(volumeInfo) > 0).To(BeIdenticalTo(true)) Expect(volumeInfo[0].SnapshotDataMovementInfo).NotTo(BeNil()) diff --git a/test/e2e/basic/backup-volume-info/csi_snapshot.go b/test/e2e/basic/backup-volume-info/csi_snapshot.go index 9fa8bb3af..a8d5669db 100644 --- a/test/e2e/basic/backup-volume-info/csi_snapshot.go +++ b/test/e2e/basic/backup-volume-info/csi_snapshot.go @@ -31,6 +31,10 @@ var CSISnapshotVolumeInfoTest func() = TestFunc(&CSISnapshotVolumeInfo{ SnapshotVolumes: true, TestCase: TestCase{ CaseBaseName: "csi-snapshot-volumeinfo", + TestMsg: &TestMSG{ + Desc: "Test backup's VolumeInfo metadata content for CSI snapshot case.", + Text: "The VolumeInfo should be generated, and the CSISnapshotInfo structure should not be nil.", + }, }, }, }) @@ -41,7 +45,7 @@ type CSISnapshotVolumeInfo struct { func (c *CSISnapshotVolumeInfo) Verify() error { volumeInfo, err := GetVolumeInfo( - c.VeleroCfg.CloudProvider, + c.VeleroCfg.ObjectStoreProvider, c.VeleroCfg.CloudCredentialsFile, c.VeleroCfg.BSLBucket, c.VeleroCfg.BSLPrefix, @@ -51,6 +55,8 @@ func (c *CSISnapshotVolumeInfo) Verify() error { ) Expect(err).ShouldNot(HaveOccurred(), fmt.Sprintf("Fail to get VolumeInfo metadata in the Backup Repository.")) + + fmt.Printf("The VolumeInfo metadata content: %+v\n", *volumeInfo[0]) Expect(len(volumeInfo) > 0).To(BeIdenticalTo(true)) Expect(volumeInfo[0].CSISnapshotInfo).NotTo(BeNil()) diff --git a/test/e2e/basic/backup-volume-info/filesystem_upload.go b/test/e2e/basic/backup-volume-info/filesystem_upload.go index a6ff9dc54..99323610f 100644 --- a/test/e2e/basic/backup-volume-info/filesystem_upload.go +++ b/test/e2e/basic/backup-volume-info/filesystem_upload.go @@ -31,6 +31,10 @@ var FilesystemUploadVolumeInfoTest func() = TestFunc(&FilesystemUploadVolumeInfo DefaultVolumesToFSBackup: true, TestCase: TestCase{ CaseBaseName: "fs-upload-volumeinfo", + TestMsg: &TestMSG{ + Desc: "Test backup's VolumeInfo metadata content for filesystem upload case.", + Text: "The VolumeInfo should be generated, and the PVBInfo structure should not be nil.", + }, }, }, }) @@ -41,7 +45,7 @@ type FilesystemUploadVolumeInfo struct { func (f *FilesystemUploadVolumeInfo) Verify() error { volumeInfo, err := GetVolumeInfo( - f.VeleroCfg.CloudProvider, + f.VeleroCfg.ObjectStoreProvider, f.VeleroCfg.CloudCredentialsFile, f.VeleroCfg.BSLBucket, f.VeleroCfg.BSLPrefix, @@ -51,6 +55,8 @@ func (f *FilesystemUploadVolumeInfo) Verify() error { ) Expect(err).ShouldNot(HaveOccurred(), fmt.Sprintf("Fail to get VolumeInfo metadata in the Backup Repository.")) + + fmt.Printf("The VolumeInfo metadata content: %+v\n", *volumeInfo[0]) Expect(len(volumeInfo) > 0).To(BeIdenticalTo(true)) Expect(volumeInfo[0].PVBInfo).NotTo(BeNil()) diff --git a/test/e2e/basic/backup-volume-info/native_snapshot.go b/test/e2e/basic/backup-volume-info/native_snapshot.go index ef3d37b1c..fb3a6f3d8 100644 --- a/test/e2e/basic/backup-volume-info/native_snapshot.go +++ b/test/e2e/basic/backup-volume-info/native_snapshot.go @@ -31,6 +31,10 @@ var NativeSnapshotVolumeInfoTest func() = TestFunc(&NativeSnapshotVolumeInfo{ SnapshotVolumes: true, TestCase: TestCase{ CaseBaseName: "native-snapshot-volumeinfo", + TestMsg: &TestMSG{ + Desc: "Test backup's VolumeInfo metadata content for native snapshot case.", + Text: "The VolumeInfo should be generated, and the NativeSnapshotInfo structure should not be nil.", + }, }, }, }) @@ -41,7 +45,7 @@ type NativeSnapshotVolumeInfo struct { func (n *NativeSnapshotVolumeInfo) Verify() error { volumeInfo, err := GetVolumeInfo( - n.VeleroCfg.CloudProvider, + n.VeleroCfg.ObjectStoreProvider, n.VeleroCfg.CloudCredentialsFile, n.VeleroCfg.BSLBucket, n.VeleroCfg.BSLPrefix, @@ -51,6 +55,8 @@ func (n *NativeSnapshotVolumeInfo) Verify() error { ) Expect(err).ShouldNot(HaveOccurred(), fmt.Sprintf("Fail to get VolumeInfo metadata in the Backup Repository.")) + + fmt.Printf("The VolumeInfo metadata content: %+v\n", *volumeInfo[0]) Expect(len(volumeInfo) > 0).To(BeIdenticalTo(true)) Expect(volumeInfo[0].NativeSnapshotInfo).NotTo(BeNil()) diff --git a/test/e2e/basic/backup-volume-info/skipped_volumes.go b/test/e2e/basic/backup-volume-info/skipped_volumes.go index 73a1c9be9..0382619a3 100644 --- a/test/e2e/basic/backup-volume-info/skipped_volumes.go +++ b/test/e2e/basic/backup-volume-info/skipped_volumes.go @@ -31,6 +31,10 @@ var SkippedVolumeInfoTest func() = TestFunc(&SkippedVolumeInfo{ SnapshotVolumes: false, TestCase: TestCase{ CaseBaseName: "skipped-volumes-volumeinfo", + TestMsg: &TestMSG{ + Desc: "Test backup's VolumeInfo metadata content for volume-skipped case.", + Text: "The VolumeInfo should be generated, and the Skipped parameter should be true.", + }, }, }, }) @@ -41,7 +45,7 @@ type SkippedVolumeInfo struct { func (s *SkippedVolumeInfo) Verify() error { volumeInfo, err := GetVolumeInfo( - s.VeleroCfg.CloudProvider, + s.VeleroCfg.ObjectStoreProvider, s.VeleroCfg.CloudCredentialsFile, s.VeleroCfg.BSLBucket, s.VeleroCfg.BSLPrefix, @@ -51,6 +55,8 @@ func (s *SkippedVolumeInfo) Verify() error { ) Expect(err).ShouldNot(HaveOccurred(), fmt.Sprintf("Fail to get VolumeInfo metadata in the Backup Repository.")) + + fmt.Printf("The VolumeInfo metadata content: %+v\n", *volumeInfo[0]) Expect(len(volumeInfo) > 0).To(BeIdenticalTo(true)) Expect(volumeInfo[0].Skipped == true).To(BeIdenticalTo(true)) diff --git a/test/testdata/storage-class/vsphere-csi.yaml b/test/testdata/storage-class/vsphere-csi.yaml new file mode 100644 index 000000000..c6d11bc7b --- /dev/null +++ b/test/testdata/storage-class/vsphere-csi.yaml @@ -0,0 +1,11 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: e2e-csi-storage-class + annotations: + storageclass.kubernetes.io/is-default-class: "false" +parameters: + StoragePolicyName: "vSAN Default Storage Policy" +provisioner: csi.vsphere.vmware.com +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer \ No newline at end of file diff --git a/test/util/csi/AzureVolumeSnapshotClass.yaml b/test/util/csi/AzureVolumeSnapshotClass.yaml deleted file mode 100644 index 4e3ca494f..000000000 --- a/test/util/csi/AzureVolumeSnapshotClass.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: snapshot.storage.k8s.io/v1 -kind: VolumeSnapshotClass -metadata: - name: velero - labels: - velero.io/csi-volumesnapshot-class: "true" -driver: disk.csi.azure.com -deletionPolicy: Delete \ No newline at end of file diff --git a/test/util/providers/common.go b/test/util/providers/common.go index 6bd37dcb3..d8ca01251 100644 --- a/test/util/providers/common.go +++ b/test/util/providers/common.go @@ -196,7 +196,7 @@ func IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig } func GetVolumeInfoMetadataContent( - cloudProvider, + objectStoreProvider, cloudCredentialsFile, bslBucket, bslPrefix, @@ -207,16 +207,16 @@ func GetVolumeInfoMetadataContent( bslPrefix = strings.Trim(getFullPrefix(bslPrefix, subPrefix), "/") volumeFileName := backupName + "-volumeinfo.json.gz" fmt.Printf("|| VERIFICATION || - Get backup %s volumeinfo file in storage %s\n", backupName, bslPrefix) - s, err := getProvider(cloudProvider) + s, err := getProvider(objectStoreProvider) if err != nil { - return nil, errors.Wrapf(err, fmt.Sprintf("Cloud provider %s is not valid", cloudProvider)) + return nil, errors.Wrapf(err, fmt.Sprintf("Cloud provider %s is not valid", objectStoreProvider)) } return s.GetObject(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, volumeFileName) } func GetVolumeInfo( - cloudProvider, + objectStoreProvider, cloudCredentialsFile, bslBucket, bslPrefix, @@ -224,7 +224,7 @@ func GetVolumeInfo( backupName, subPrefix string, ) ([]*volume.VolumeInfo, error) { - readCloser, err := GetVolumeInfoMetadataContent(cloudProvider, + readCloser, err := GetVolumeInfoMetadataContent(objectStoreProvider, cloudCredentialsFile, bslBucket, bslPrefix, diff --git a/test/util/velero/install.go b/test/util/velero/install.go index 633ef113f..0efcf3849 100644 --- a/test/util/velero/install.go +++ b/test/util/velero/install.go @@ -280,9 +280,10 @@ func installVeleroServer(ctx context.Context, cli, cloudProvider string, options if len(options.Features) > 0 { args = append(args, "--features", options.Features) - if strings.EqualFold(cloudProvider, "azure") && strings.EqualFold(options.Features, FeatureCSI) && options.UseVolumeSnapshots { + if !strings.EqualFold(cloudProvider, "vsphere") && strings.EqualFold(options.Features, FeatureCSI) && options.UseVolumeSnapshots { fmt.Println("Start to install Azure VolumeSnapshotClass ...") - if err := KubectlApplyByFile(ctx, "../util/csi/AzureVolumeSnapshotClass.yaml"); err != nil { + if err := KubectlApplyByFile(ctx, fmt.Sprintf("../testdata/volume-snapshot-class/%s.yaml", cloudProvider)); err != nil { + fmt.Println("Fail to install VolumeSnapshotClass when CSI feature is enabled: ", err) return err } } @@ -634,6 +635,7 @@ func VeleroUninstall(ctx context.Context, cli, namespace string) error { return errors.Wrapf(err, "failed to uninstall velero, stdout=%s, stderr=%s", stdout, stderr) } fmt.Println("Velero uninstalled ⛵") + return nil }