Merge pull request #7280 from danfengliu/fix-backup-deletion-snapshot-throttle-issue

Add sleep to avoid snapshot throttle issue

test/e2e/backups/deletion.go

@@ -178,6 +178,11 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam
 			return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
 		}
 	}
+
+	// Hit issue: https://docs.aws.amazon.com/AWSEC2/latest/APIReference/errors-overview.html#:~:text=SnapshotCreationPerVolumeRateExceeded
+	// Sleep for more than 15 seconds to avoid this issue.
+	time.Sleep(1 * time.Minute)
+
 	backupName = "backup-1-" + UUIDgen.String()
 	BackupCfg.BackupName = backupName
 

test/util/csi/common.go

@@ -21,14 +21,12 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/pkg/errors"
-
 	snapshotterClientSet "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned"
+	"github.com/pkg/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
 
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
 	. "github.com/vmware-tanzu/velero/test/util/k8s"
 )
 
@@ -124,7 +122,7 @@ func GetCsiSnapshotHandleV1(client TestClient, backupName string) ([]string, err
 	}
 
 	if len(snapshotHandleList) == 0 {
-		fmt.Printf("No VolumeSnapshotContent from backup %s", backupName)
+		fmt.Printf("No VolumeSnapshotContent from backup %s\n", backupName)
 	}
 	return snapshotHandleList, nil
 }
@@ -170,11 +168,11 @@ func CheckVolumeSnapshotCR(client TestClient, backupName string, expectedCount i
 	var snapshotContentNameList []string
 	if apiVersion == "v1beta1" {
 		if snapshotContentNameList, err = GetCsiSnapshotHandle(client, backupName); err != nil {
-			return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content")
+			return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content for v1beta1")
 		}
 	} else if apiVersion == "v1" {
 		if snapshotContentNameList, err = GetCsiSnapshotHandleV1(client, backupName); err != nil {
-			return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content")
+			return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content for v1")
 		}
 	} else {
 		return nil, errors.New("API version is invalid")

test/util/k8s/common.go

@@ -225,7 +225,7 @@ func GetAPIVersions(client *TestClient, name string) ([]string, error) {
 		fmt.Println(group.Name)
 		if group.Name == name {
 			for _, v := range group.Versions {
-				fmt.Println(v.Version)
+				fmt.Printf("group: %s version:%s", group.Name, v.Version)
 				version = append(version, v.Version)
 			}
 			return version, nil

test/util/kibishii/kibishii_utils.go

@@ -117,6 +117,7 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc
 			}
 		}
 		snapshotCheckPoint, err := GetSnapshotCheckPoint(client, veleroCfg, 2, kibishiiNamespace, backupName, KibishiiPVCNameList)
+
 		if err != nil {
 			return errors.Wrap(err, "Fail to get snapshot checkpoint")
 		}

test/util/providers/aws_utils.go

@@ -345,41 +345,61 @@ func (s AWSStorage) IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupObj
 		fmt.Printf("Fail to create session with profile %s and config %s", cloudCredentialsFile, bslConfig)
 		return errors.Wrapf(err, "Fail to create session with profile %s and config %s", cloudCredentialsFile, bslConfig)
 	}
+
 	svc := ec2.New(sess)
 	params := &ec2.DescribeSnapshotsInput{
 		OwnerIds: []*string{aws.String("self")},
-		Filters: []*ec2.Filter{
-			{
-				Name: aws.String("tag:velero.io/backup"),
-				Values: []*string{
-					aws.String(backupObject),
+	}
+
+	if !snapshotCheck.EnableCSI {
+		params = &ec2.DescribeSnapshotsInput{
+			OwnerIds: []*string{aws.String("self")},
+			Filters: []*ec2.Filter{
+				{
+					Name: aws.String("tag:velero.io/backup"),
+					Values: []*string{
+						aws.String(backupObject),
+					},
 				},
 			},
-		},
+		}
 	}
 
 	result, err := svc.DescribeSnapshots(params)
 	if err != nil {
 		fmt.Println(err)
 	}
-
-	for _, n := range result.Snapshots {
-		fmt.Println(n.SnapshotId)
-		if n.SnapshotId != nil {
-			fmt.Println(*n.SnapshotId)
+	var actualCount int
+	if snapshotCheck.EnableCSI {
+		for _, snapshotId := range snapshotCheck.SnapshotIDList {
+			for _, n := range result.Snapshots {
+				if n.SnapshotId != nil && (*n.SnapshotId == snapshotId) {
+					actualCount++
+					fmt.Printf("SnapshotId: %v, Tags: %v \n", *n.SnapshotId, n.Tags)
+					if n.VolumeId != nil {
+						fmt.Printf("VolumeId: %v \n", *n.VolumeId)
+					}
+				}
+			}
 		}
-		fmt.Println(n.Tags)
-		fmt.Println(n.VolumeId)
-		if n.VolumeId != nil {
-			fmt.Println(*n.VolumeId)
-		}
-	}
-	if len(result.Snapshots) != snapshotCheck.ExpectCount {
-		return errors.New(fmt.Sprintf("Snapshot count is not as expected %d", snapshotCheck.ExpectCount))
 	} else {
-		fmt.Printf("Snapshot count %d is as expected %d\n", len(result.Snapshots), snapshotCheck.ExpectCount)
+		for _, n := range result.Snapshots {
+			if n.SnapshotId != nil {
+				fmt.Printf("SnapshotId: %v, Tags: %v \n", *n.SnapshotId, n.Tags)
+				if n.VolumeId != nil {
+					fmt.Printf("VolumeId: %v \n", *n.VolumeId)
+				}
+			}
+		}
+		actualCount = len(result.Snapshots)
+	}
+	if actualCount != snapshotCheck.ExpectCount {
+		return errors.New(fmt.Sprintf("Snapshot count %d is not as expected %d", actualCount, snapshotCheck.ExpectCount))
+	} else {
+		fmt.Printf("Snapshot count %d is as expected %d\n", actualCount, snapshotCheck.ExpectCount)
 		return nil
 	}
+
 }
 
 func (s AWSStorage) GetMinioBucketSize(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig string) (int64, error) {

test/util/velero/velero_utils.go

@@ -35,11 +35,9 @@ import (
 
 	"github.com/pkg/errors"
 	"golang.org/x/exp/slices"
-	"k8s.io/apimachinery/pkg/util/wait"
-
-	kbclient "sigs.k8s.io/controller-runtime/pkg/client"
-
 	ver "k8s.io/apimachinery/pkg/util/version"
+	"k8s.io/apimachinery/pkg/util/wait"
+	kbclient "sigs.k8s.io/controller-runtime/pkg/client"
 
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	cliinstall "github.com/vmware-tanzu/velero/pkg/cmd/cli/install"
@@ -1201,7 +1199,8 @@ func GetSnapshotCheckPoint(client TestClient, VeleroCfg VeleroConfig, expectCoun
 	snapshotCheckPoint.ExpectCount = expectCount
 	snapshotCheckPoint.NamespaceBackedUp = namespaceBackedUp
 	snapshotCheckPoint.PodName = KibishiiPVCNameList
-	if VeleroCfg.CloudProvider == "azure" && strings.EqualFold(VeleroCfg.Features, "EnableCSI") {
+
+	if (VeleroCfg.CloudProvider == "azure" || VeleroCfg.CloudProvider == "aws") && strings.EqualFold(VeleroCfg.Features, "EnableCSI") {
 		snapshotCheckPoint.EnableCSI = true
 		resourceName := "snapshot.storage.k8s.io"