From ce00387ccf85f985181c1887efec82f49cba0b5d Mon Sep 17 00:00:00 2001 From: danfengl Date: Sat, 6 Jan 2024 07:58:50 +0000 Subject: [PATCH] Add sleep to avoid AWS snapshot throttle issue 1. Add sleep in backup deletion E2E test to avoid AWS snapshot throttle issue: https://docs.aws.amazon.com/AWSEC2/latest/APIReference/errors-overview.html#:~:text=SnapshotCreationPerVolumeRateExceeded; 2. Support CSI snapshot checkpoint for EKS pipeline. Signed-off-by: danfengl --- test/e2e/backups/deletion.go | 5 +++ test/util/csi/common.go | 12 +++--- test/util/k8s/common.go | 2 +- test/util/kibishii/kibishii_utils.go | 1 + test/util/providers/aws_utils.go | 60 ++++++++++++++++++---------- test/util/velero/velero_utils.go | 9 ++--- 6 files changed, 56 insertions(+), 33 deletions(-) diff --git a/test/e2e/backups/deletion.go b/test/e2e/backups/deletion.go index 34e502df0..c75d09cbf 100644 --- a/test/e2e/backups/deletion.go +++ b/test/e2e/backups/deletion.go @@ -178,6 +178,11 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam return errors.Wrap(err, "exceed waiting for snapshot created in cloud") } } + + // Hit issue: https://docs.aws.amazon.com/AWSEC2/latest/APIReference/errors-overview.html#:~:text=SnapshotCreationPerVolumeRateExceeded + // Sleep for more than 15 seconds to avoid this issue. + time.Sleep(1 * time.Minute) + backupName = "backup-1-" + UUIDgen.String() BackupCfg.BackupName = backupName diff --git a/test/util/csi/common.go b/test/util/csi/common.go index e96e865b0..4839b6049 100644 --- a/test/util/csi/common.go +++ b/test/util/csi/common.go @@ -21,14 +21,12 @@ import ( "fmt" "strings" - "github.com/pkg/errors" - snapshotterClientSet "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned" + "github.com/pkg/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - . "github.com/vmware-tanzu/velero/test/util/k8s" ) @@ -124,7 +122,7 @@ func GetCsiSnapshotHandleV1(client TestClient, backupName string) ([]string, err } if len(snapshotHandleList) == 0 { - fmt.Printf("No VolumeSnapshotContent from backup %s", backupName) + fmt.Printf("No VolumeSnapshotContent from backup %s\n", backupName) } return snapshotHandleList, nil } @@ -170,11 +168,11 @@ func CheckVolumeSnapshotCR(client TestClient, backupName string, expectedCount i var snapshotContentNameList []string if apiVersion == "v1beta1" { if snapshotContentNameList, err = GetCsiSnapshotHandle(client, backupName); err != nil { - return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content") + return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content for v1beta1") } } else if apiVersion == "v1" { if snapshotContentNameList, err = GetCsiSnapshotHandleV1(client, backupName); err != nil { - return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content") + return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content for v1") } } else { return nil, errors.New("API version is invalid") diff --git a/test/util/k8s/common.go b/test/util/k8s/common.go index ed579cb77..db323da4a 100644 --- a/test/util/k8s/common.go +++ b/test/util/k8s/common.go @@ -225,7 +225,7 @@ func GetAPIVersions(client *TestClient, name string) ([]string, error) { fmt.Println(group.Name) if group.Name == name { for _, v := range group.Versions { - fmt.Println(v.Version) + fmt.Printf("group: %s version:%s", group.Name, v.Version) version = append(version, v.Version) } return version, nil diff --git a/test/util/kibishii/kibishii_utils.go b/test/util/kibishii/kibishii_utils.go index f9d2b00d8..252e2018c 100644 --- a/test/util/kibishii/kibishii_utils.go +++ b/test/util/kibishii/kibishii_utils.go @@ -117,6 +117,7 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc } } snapshotCheckPoint, err := GetSnapshotCheckPoint(client, veleroCfg, 2, kibishiiNamespace, backupName, KibishiiPVCNameList) + if err != nil { return errors.Wrap(err, "Fail to get snapshot checkpoint") } diff --git a/test/util/providers/aws_utils.go b/test/util/providers/aws_utils.go index f9dd597ac..342acba1b 100644 --- a/test/util/providers/aws_utils.go +++ b/test/util/providers/aws_utils.go @@ -345,41 +345,61 @@ func (s AWSStorage) IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupObj fmt.Printf("Fail to create session with profile %s and config %s", cloudCredentialsFile, bslConfig) return errors.Wrapf(err, "Fail to create session with profile %s and config %s", cloudCredentialsFile, bslConfig) } + svc := ec2.New(sess) params := &ec2.DescribeSnapshotsInput{ OwnerIds: []*string{aws.String("self")}, - Filters: []*ec2.Filter{ - { - Name: aws.String("tag:velero.io/backup"), - Values: []*string{ - aws.String(backupObject), + } + + if !snapshotCheck.EnableCSI { + params = &ec2.DescribeSnapshotsInput{ + OwnerIds: []*string{aws.String("self")}, + Filters: []*ec2.Filter{ + { + Name: aws.String("tag:velero.io/backup"), + Values: []*string{ + aws.String(backupObject), + }, }, }, - }, + } } result, err := svc.DescribeSnapshots(params) if err != nil { fmt.Println(err) } - - for _, n := range result.Snapshots { - fmt.Println(n.SnapshotId) - if n.SnapshotId != nil { - fmt.Println(*n.SnapshotId) + var actualCount int + if snapshotCheck.EnableCSI { + for _, snapshotId := range snapshotCheck.SnapshotIDList { + for _, n := range result.Snapshots { + if n.SnapshotId != nil && (*n.SnapshotId == snapshotId) { + actualCount++ + fmt.Printf("SnapshotId: %v, Tags: %v \n", *n.SnapshotId, n.Tags) + if n.VolumeId != nil { + fmt.Printf("VolumeId: %v \n", *n.VolumeId) + } + } + } } - fmt.Println(n.Tags) - fmt.Println(n.VolumeId) - if n.VolumeId != nil { - fmt.Println(*n.VolumeId) - } - } - if len(result.Snapshots) != snapshotCheck.ExpectCount { - return errors.New(fmt.Sprintf("Snapshot count is not as expected %d", snapshotCheck.ExpectCount)) } else { - fmt.Printf("Snapshot count %d is as expected %d\n", len(result.Snapshots), snapshotCheck.ExpectCount) + for _, n := range result.Snapshots { + if n.SnapshotId != nil { + fmt.Printf("SnapshotId: %v, Tags: %v \n", *n.SnapshotId, n.Tags) + if n.VolumeId != nil { + fmt.Printf("VolumeId: %v \n", *n.VolumeId) + } + } + } + actualCount = len(result.Snapshots) + } + if actualCount != snapshotCheck.ExpectCount { + return errors.New(fmt.Sprintf("Snapshot count %d is not as expected %d", actualCount, snapshotCheck.ExpectCount)) + } else { + fmt.Printf("Snapshot count %d is as expected %d\n", actualCount, snapshotCheck.ExpectCount) return nil } + } func (s AWSStorage) GetMinioBucketSize(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig string) (int64, error) { diff --git a/test/util/velero/velero_utils.go b/test/util/velero/velero_utils.go index f256aa6f1..dba18f217 100644 --- a/test/util/velero/velero_utils.go +++ b/test/util/velero/velero_utils.go @@ -35,11 +35,9 @@ import ( "github.com/pkg/errors" "golang.org/x/exp/slices" - "k8s.io/apimachinery/pkg/util/wait" - - kbclient "sigs.k8s.io/controller-runtime/pkg/client" - ver "k8s.io/apimachinery/pkg/util/version" + "k8s.io/apimachinery/pkg/util/wait" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" cliinstall "github.com/vmware-tanzu/velero/pkg/cmd/cli/install" @@ -1201,7 +1199,8 @@ func GetSnapshotCheckPoint(client TestClient, VeleroCfg VeleroConfig, expectCoun snapshotCheckPoint.ExpectCount = expectCount snapshotCheckPoint.NamespaceBackedUp = namespaceBackedUp snapshotCheckPoint.PodName = KibishiiPVCNameList - if VeleroCfg.CloudProvider == "azure" && strings.EqualFold(VeleroCfg.Features, "EnableCSI") { + + if (VeleroCfg.CloudProvider == "azure" || VeleroCfg.CloudProvider == "aws") && strings.EqualFold(VeleroCfg.Features, "EnableCSI") { snapshotCheckPoint.EnableCSI = true resourceName := "snapshot.storage.k8s.io"