Merge pull request #4401 from danfengliu/add-backup-deletion-e2e-test-to-main

Add backup deletion e2e test

test/e2e/util/providers/aws_utils.go

@@ -0,0 +1,141 @@
+/*
+Copyright the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package providers
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+	"github.com/pkg/errors"
+
+	"github.com/vmware-tanzu/velero/pkg/cmd/util/flag"
+)
+
+type AWSStorage string
+
+func (s AWSStorage) ListItems(client *s3.S3, objectsV2Input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
+	res, err := client.ListObjectsV2(objectsV2Input)
+	if err != nil {
+		return nil, err
+	}
+
+	return res, nil
+}
+
+func (s AWSStorage) DeleteItem(client *s3.S3, deleteObjectV2Input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
+	res, err := client.DeleteObject(deleteObjectV2Input)
+	if err != nil {
+		return nil, err
+	}
+	fmt.Println(res)
+	return res, nil
+}
+func (s AWSStorage) IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) (bool, error) {
+	config := flag.NewMap()
+	config.Set(bslConfig)
+	region := config.Data()["region"]
+	objectsInput := s3.ListObjectsV2Input{}
+	objectsInput.Bucket = aws.String(bslBucket)
+	objectsInput.Delimiter = aws.String("/")
+	s3url := ""
+	if bslPrefix != "" {
+		objectsInput.Prefix = aws.String(bslPrefix)
+	}
+	s3Config := &aws.Config{
+		Region:      aws.String(region),
+		Credentials: credentials.NewSharedCredentials(cloudCredentialsFile, ""),
+	}
+	if region == "minio" {
+		s3url = config.Data()["s3Url"]
+		s3Config = &aws.Config{
+			Credentials:      credentials.NewSharedCredentials(cloudCredentialsFile, ""),
+			Endpoint:         aws.String(s3url),
+			Region:           aws.String(region),
+			DisableSSL:       aws.Bool(true),
+			S3ForcePathStyle: aws.Bool(true),
+		}
+	}
+
+	sess, err := session.NewSession(s3Config)
+
+	if err != nil {
+		return false, errors.Wrapf(err, "Failed to create AWS session")
+	}
+	svc := s3.New(sess)
+
+	bucketObjects, err := s.ListItems(svc, &objectsInput)
+	if err != nil {
+		return false, errors.Wrapf(err, "Couldn't retrieve bucket items")
+	}
+
+	for _, item := range bucketObjects.Contents {
+		fmt.Println(*item)
+	}
+	var backupNameInStorage string
+	for _, item := range bucketObjects.CommonPrefixes {
+		backupNameInStorage = strings.TrimPrefix(*item.Prefix, strings.Trim(bslPrefix, "/")+"/")
+		fmt.Println(backupNameInStorage)
+		if strings.Contains(backupNameInStorage, backupObject) {
+			fmt.Printf("Backup %s was found under prefix %s \n", backupObject, bslPrefix)
+			return true, nil
+		}
+	}
+	fmt.Printf("Backup %s was not found under prefix %s \n", backupObject, bslPrefix)
+	return false, nil
+}
+
+func (s AWSStorage) DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) error {
+	config := flag.NewMap()
+	config.Set(bslConfig)
+	region := config.Data()["region"]
+	s3url := ""
+	s3Config := &aws.Config{
+		Region:      aws.String(region),
+		Credentials: credentials.NewSharedCredentials(cloudCredentialsFile, ""),
+	}
+	if region == "minio" {
+		s3url = config.Data()["s3Url"]
+		s3Config = &aws.Config{
+			Credentials:      credentials.NewSharedCredentials(cloudCredentialsFile, ""),
+			Endpoint:         aws.String(s3url),
+			Region:           aws.String(region),
+			DisableSSL:       aws.Bool(true),
+			S3ForcePathStyle: aws.Bool(true),
+		}
+	}
+	sess, err := session.NewSession(s3Config)
+	if err != nil {
+		return errors.Wrapf(err, "Error waiting for uploads to complete")
+	}
+	svc := s3.New(sess)
+	fullPrefix := strings.Trim(bslPrefix, "/") + "/" + strings.Trim(backupObject, "/") + "/"
+	iter := s3manager.NewDeleteListIterator(svc, &s3.ListObjectsInput{
+		Bucket: aws.String(bslBucket),
+		Prefix: aws.String(fullPrefix),
+	})
+
+	if err := s3manager.NewBatchDeleteWithClient(svc).Delete(aws.BackgroundContext(), iter); err != nil {
+		return errors.Wrapf(err, "Error waiting for uploads to complete")
+	}
+	fmt.Printf("Deleted object(s) from bucket: %s %s \n", bslBucket, fullPrefix)
+	return nil
+}

test/e2e/util/providers/azure_utils.go

@@ -0,0 +1,258 @@
+/*
+Copyright the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package providers
+
+import (
+	"fmt"
+	"log"
+	"net/url"
+	"os"
+	"strings"
+
+	"github.com/Azure/azure-pipeline-go/pipeline"
+	storagemgmt "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-06-01/storage"
+	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/Azure/go-autorest/autorest/azure"
+	"github.com/Azure/go-autorest/autorest/azure/auth"
+	"github.com/joho/godotenv"
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+
+	"github.com/vmware-tanzu/velero/pkg/cmd/util/flag"
+)
+
+type AzureStorage string
+
+const (
+	subscriptionIDEnvVar = "AZURE_SUBSCRIPTION_ID"
+	cloudNameEnvVar      = "AZURE_CLOUD_NAME"
+	resourceGroupEnvVar  = "AZURE_RESOURCE_GROUP"
+	storageAccountKey    = "AZURE_STORAGE_ACCOUNT_ACCESS_KEY"
+	storageAccount       = "storageAccount"
+	subscriptionID       = "subscriptionId"
+	resourceGroup        = "resourceGroup"
+)
+
+func getStorageCredential(cloudCredentialsFile, bslConfig string) (string, string, error) {
+	config := flag.NewMap()
+	config.Set(bslConfig)
+	accountName := config.Data()[storageAccount]
+	// Account name must be provided in config
+	if len(accountName) == 0 {
+		return "", "", errors.New("Please provide bucket as Azure account name ")
+	}
+	subscriptionID := config.Data()[subscriptionID]
+	resourceGroupCfg := config.Data()[resourceGroup]
+	accountKey, err := getStorageAccountKey(cloudCredentialsFile, accountName, subscriptionID, resourceGroupCfg)
+	if err != nil {
+		return "", "", errors.Wrapf(err, "Fail to get storage key of bucket %s", accountName)
+	}
+	return accountName, accountKey, nil
+}
+func loadCredentialsIntoEnv(credentialsFile string) error {
+	if credentialsFile == "" {
+		return nil
+	}
+
+	if err := godotenv.Overload(credentialsFile); err != nil {
+		return errors.Wrapf(err, "error loading environment from credentials file (%s)", credentialsFile)
+	}
+	return nil
+}
+func parseAzureEnvironment(cloudName string) (*azure.Environment, error) {
+	if cloudName == "" {
+		fmt.Println("cloudName is empty")
+		return &azure.PublicCloud, nil
+	}
+
+	env, err := azure.EnvironmentFromName(cloudName)
+	return &env, errors.WithStack(err)
+}
+func getStorageAccountKey(credentialsFile, accountName, subscriptionID, resourceGroupCfg string) (string, error) {
+	if err := loadCredentialsIntoEnv(credentialsFile); err != nil {
+		return "", err
+	}
+	storageKey := os.Getenv(storageAccountKey)
+	if storageKey != "" {
+		return storageKey, nil
+	}
+	if os.Getenv(cloudNameEnvVar) == "" {
+		return "", errors.New("Credential file should contain AZURE_CLOUD_NAME")
+	}
+	var resourceGroup string
+	if os.Getenv(resourceGroupEnvVar) == "" {
+		if resourceGroupCfg == "" {
+			return "", errors.New("Credential file should contain AZURE_RESOURCE_GROUP or AZURE_STORAGE_ACCOUNT_ACCESS_KEY")
+		} else {
+			resourceGroup = resourceGroupCfg
+		}
+	} else {
+		resourceGroup = os.Getenv(resourceGroupEnvVar)
+	}
+	// get Azure cloud from AZURE_CLOUD_NAME, if it exists. If the env var does not
+	// exist, parseAzureEnvironment will return azure.PublicCloud.
+	env, err := parseAzureEnvironment(os.Getenv(cloudNameEnvVar))
+	if err != nil {
+		return "", errors.Wrap(err, "unable to parse azure cloud name environment variable")
+	}
+
+	// get subscription ID from object store config or AZURE_SUBSCRIPTION_ID environment variable
+	if subscriptionID == "" {
+		return "", errors.New("azure subscription ID not found in object store's config or in environment variable")
+	}
+
+	authorizer, err := auth.NewAuthorizerFromEnvironment()
+	if err != nil {
+		return "", errors.Wrap(err, "error getting authorizer from environment")
+	}
+
+	// get storageAccountsClient
+	storageAccountsClient := storagemgmt.NewAccountsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID)
+	storageAccountsClient.Authorizer = authorizer
+
+	// get storage key
+	res, err := storageAccountsClient.ListKeys(context.TODO(), resourceGroup, accountName, storagemgmt.Kerb)
+	if err != nil {
+		return "", errors.WithStack(err)
+	}
+	if res.Keys == nil || len(*res.Keys) == 0 {
+		return "", errors.New("No storage keys found")
+	}
+
+	for _, key := range *res.Keys {
+		// uppercase both strings for comparison because the ListKeys call returns e.g. "FULL" but
+		// the storagemgmt.Full constant in the SDK is defined as "Full".
+		if strings.EqualFold(string(key.Permissions), string(storagemgmt.Full)) {
+			storageKey = *key.Value
+			break
+		}
+	}
+
+	if storageKey == "" {
+		return "", errors.New("No storage key with Full permissions found")
+	}
+
+	return storageKey, nil
+}
+func handleErrors(err error) {
+	if err != nil {
+		if serr, ok := err.(azblob.StorageError); ok { // This error is a Service-specific
+			switch serr.ServiceCode() { // Compare serviceCode to ServiceCodeXxx constants
+			case azblob.ServiceCodeContainerAlreadyExists:
+				return
+			}
+		}
+		log.Fatal(err)
+	}
+}
+
+func deleteBlob(p pipeline.Pipeline, accountName, containerName, blobName string) error {
+	ctx := context.Background()
+
+	URL_BLOB, err := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net/%s/%s", accountName, containerName, blobName))
+	if err != nil {
+		return errors.Wrapf(err, "Fail to url.Parse")
+	}
+	blobURL := azblob.NewBlobURL(*URL_BLOB, p)
+	_, err = blobURL.Delete(ctx, azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{})
+	return err
+}
+func (s AzureStorage) IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) (bool, error) {
+	accountName, accountKey, err := getStorageCredential(cloudCredentialsFile, bslConfig)
+	if err != nil {
+		log.Fatal("Fail to get : accountName and accountKey, " + err.Error())
+	}
+
+	credential, err := azblob.NewSharedKeyCredential(accountName, accountKey)
+	if err != nil {
+		log.Fatal("Invalid credentials with error: " + err.Error())
+	}
+	p := azblob.NewPipeline(credential, azblob.PipelineOptions{})
+
+	containerName := bslBucket
+
+	URL, _ := url.Parse(
+		fmt.Sprintf("https://%s.blob.core.windows.net/%s", accountName, containerName))
+
+	containerURL := azblob.NewContainerURL(*URL, p)
+
+	// Create the container, if container is already exist, then do nothing
+	ctx := context.Background()
+	_, err = containerURL.Create(ctx, azblob.Metadata{}, azblob.PublicAccessNone)
+	handleErrors(err)
+
+	fmt.Printf("Finding backup %s blobs in Azure container/bucket %s\n", backupObject, containerName)
+	for marker := (azblob.Marker{}); marker.NotDone(); {
+		listBlob, err := containerURL.ListBlobsFlatSegment(ctx, marker, azblob.ListBlobsSegmentOptions{})
+		if err != nil {
+			return false, errors.Wrapf(err, "Fail to create gcloud client")
+		}
+		marker = listBlob.NextMarker
+
+		for _, blobInfo := range listBlob.Segment.BlobItems {
+			if strings.Contains(blobInfo.Name, backupObject) {
+				fmt.Printf("Blob name: %s exist in %s\n", backupObject, blobInfo.Name)
+				return true, nil
+			}
+		}
+	}
+	return false, nil
+}
+
+func (s AzureStorage) DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) error {
+	ctx := context.Background()
+	accountName, accountKey, err := getStorageCredential(cloudCredentialsFile, bslConfig)
+	if err != nil {
+		return errors.Wrapf(err, "Fail to get storage account name and  key of bucket %s", bslBucket)
+	}
+
+	credential, err := azblob.NewSharedKeyCredential(accountName, accountKey)
+	if err != nil {
+		log.Fatal("Invalid credentials with error: " + err.Error())
+	}
+	p := azblob.NewPipeline(credential, azblob.PipelineOptions{})
+
+	containerName := bslBucket
+
+	URL, _ := url.Parse(
+		fmt.Sprintf("https://%s.blob.core.windows.net/%s", accountName, containerName))
+
+	containerURL := azblob.NewContainerURL(*URL, p)
+	_, err = containerURL.Create(ctx, azblob.Metadata{}, azblob.PublicAccessNone)
+	handleErrors(err)
+
+	fmt.Println("Listing the blobs in the container:")
+	for marker := (azblob.Marker{}); marker.NotDone(); {
+		listBlob, err := containerURL.ListBlobsFlatSegment(ctx, marker, azblob.ListBlobsSegmentOptions{})
+		if err != nil {
+			return errors.Wrapf(err, "Fail to create gcloud client")
+		}
+
+		marker = listBlob.NextMarker
+		for _, blobInfo := range listBlob.Segment.BlobItems {
+
+			if strings.Contains(blobInfo.Name, bslPrefix+backupObject+"/") {
+				deleteBlob(p, accountName, containerName, blobInfo.Name)
+				if err != nil {
+					log.Fatal("Invalid credentials with error: " + err.Error())
+				}
+				fmt.Printf("Deleted blob: %s according to backup resource %s\n", blobInfo.Name, bslPrefix+backupObject+"/")
+			}
+		}
+	}
+	return nil
+}

test/e2e/util/providers/common.go

@@ -0,0 +1,105 @@
+/*
+Copyright the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package providers
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+)
+
+type ObjectsInStorage interface {
+	IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) (bool, error)
+	DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) error
+}
+
+func ObjectsShouldBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) error {
+	fmt.Printf("|| VERIFICATION || - Backup %s should exist in storage %s", backupName, bslPrefix)
+	exist, _ := IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix)
+	if !exist {
+		return errors.New(fmt.Sprintf("|| UNEXPECTED ||Backup object %s is not exist in object store after backup as expected", backupName))
+	}
+	fmt.Printf("|| EXPECTED || - Backup %s exist in object storage bucket %s\n", backupName, bslBucket)
+	return nil
+}
+func ObjectsShouldNotBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string, retryTimes int) error {
+	var err error
+	var exist bool
+	fmt.Printf("|| VERIFICATION || - Backup %s should not exist in storage %s", backupName, bslPrefix)
+	for i := 0; i < retryTimes; i++ {
+		exist, err = IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix)
+		if err != nil {
+			return errors.Wrapf(err, "|| UNEXPECTED || - Failed to get backup %s in object store", backupName)
+		}
+		if !exist {
+			fmt.Printf("|| EXPECTED || - Backup %s is not in object store\n", backupName)
+			return nil
+		}
+		time.Sleep(1 * time.Minute)
+	}
+	return errors.New(fmt.Sprintf("|| UNEXPECTED ||Backup object %s still exist in object store after backup deletion", backupName))
+}
+func getProvider(cloudProvider string) (ObjectsInStorage, error) {
+	var s ObjectsInStorage
+	switch cloudProvider {
+	case "aws", "vsphere":
+		aws := AWSStorage("")
+		s = &aws
+	case "gcp":
+		gcs := GCSStorage("")
+		s = &gcs
+	case "azure":
+		az := AzureStorage("")
+		s = &az
+	default:
+		return nil, errors.New(fmt.Sprintf("Cloud provider %s is not valid", cloudProvider))
+	}
+	return s, nil
+}
+func getFullPrefix(bslPrefix, subPrefix string) string {
+	if bslPrefix == "" {
+		bslPrefix = subPrefix + "/"
+	} else {
+		//subPrefix must have surfix "/", so that objects under it can be listed
+		bslPrefix = strings.Trim(bslPrefix, "/") + "/" + strings.Trim(subPrefix, "/") + "/"
+	}
+	return bslPrefix
+}
+func IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) (bool, error) {
+	bslPrefix = getFullPrefix(bslPrefix, subPrefix)
+	s, err := getProvider(cloudProvider)
+	if err != nil {
+		return false, errors.Wrapf(err, fmt.Sprintf("Cloud provider %s is not valid", cloudProvider))
+	}
+	return s.IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName)
+}
+
+func DeleteObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) error {
+	bslPrefix = getFullPrefix(bslPrefix, subPrefix)
+	fmt.Printf("|| VERIFICATION || - Delete backup %s in storage %s", backupName, bslPrefix)
+	s, err := getProvider(cloudProvider)
+	if err != nil {
+		return errors.Wrapf(err, fmt.Sprintf("Cloud provider %s is not valid", cloudProvider))
+	}
+	err = s.DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName)
+	if err != nil {
+		return errors.Wrapf(err, fmt.Sprintf("Fail to delete %s", bslPrefix))
+	}
+	return nil
+}

test/e2e/util/providers/gcloud_utils.go

@@ -0,0 +1,99 @@
+/*
+Copyright the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package providers
+
+import (
+	"fmt"
+	"strings"
+
+	"cloud.google.com/go/storage"
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+	"google.golang.org/api/iterator"
+	"google.golang.org/api/option"
+)
+
+type GCSStorage string
+
+func (s GCSStorage) IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) (bool, error) {
+	q := &storage.Query{
+		Prefix: bslPrefix,
+	}
+
+	ctx := context.Background()
+	client, err := storage.NewClient(ctx, option.WithCredentialsFile(cloudCredentialsFile))
+	if err != nil {
+		return false, errors.Wrapf(err, "Fail to create gcloud client")
+	}
+	iter := client.Bucket(bslBucket).Objects(context.Background(), q)
+	for {
+		obj, err := iter.Next()
+		if err == iterator.Done {
+			return false, errors.Wrapf(err, fmt.Sprintf("Backup %s was not found under prefix %s \n", backupObject, bslPrefix))
+		}
+		if err != nil {
+			return false, errors.WithStack(err)
+		}
+		if obj.Name == bslPrefix {
+			fmt.Println("Ignore GCS prefix itself")
+			continue
+		}
+		if strings.Contains(obj.Name, bslPrefix+backupObject+"/") {
+			fmt.Printf("Found delete-object %s of %s in bucket %s \n", backupObject, obj.Name, bslBucket)
+			return true, nil
+		}
+	}
+}
+func (s GCSStorage) DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) error {
+	q := &storage.Query{
+		Prefix: bslPrefix,
+	}
+
+	ctx := context.Background()
+	client, err := storage.NewClient(ctx, option.WithCredentialsFile(cloudCredentialsFile))
+	if err != nil {
+		return errors.Wrapf(err, "Fail to create gcloud client")
+	}
+	bucket := client.Bucket(bslBucket)
+	iter := bucket.Objects(context.Background(), q)
+	deleted := false
+	for {
+		obj, err := iter.Next()
+		if err == iterator.Done {
+			fmt.Println(err)
+			if !deleted {
+				return errors.New("|| UNEXPECTED ||Backup object is not exist and was not deleted in object store")
+			}
+			return nil
+		}
+		if err != nil {
+			return errors.WithStack(err)
+		}
+		if obj.Name == bslPrefix {
+			fmt.Println("Ignore GCS prefix itself")
+			continue
+		}
+		// Only delete folder named as backupObject under prefix
+		if strings.Contains(obj.Name, bslPrefix+backupObject+"/") {
+			if err = bucket.Object(obj.Name).Delete(ctx); err != nil {
+				return errors.Wrapf(err, fmt.Sprintf("Fail to delete object %s in bucket %s", obj.Name, bslBucket))
+			}
+			fmt.Printf("Delete item: %s\n", obj.Name)
+			deleted = true
+		}
+	}
+}

test/e2e/util/velero/velero_utils.go

@@ -41,6 +41,10 @@ import (
 	veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec"
 )
 
+const (
+	BackupObjectsPrefix = "backups"
+)
+
 var pluginsMatrix = map[string]map[string][]string{
 	"v1.4": {
 		"aws":     {"velero/velero-plugin-for-aws:v1.1.0"},
@@ -388,6 +392,9 @@ func getProviderPlugins(ctx context.Context, veleroCLI, objectStoreProvider, pro
 // installs them in the current Velero installation, skipping over those that are already installed.
 func VeleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string, addPlugins string) error {
 	plugins, err := getProviderPlugins(ctx, veleroCLI, provider, addPlugins)
+	fmt.Printf("addPlugins cmd =%v\n", addPlugins)
+	fmt.Printf("provider cmd = %v\n", provider)
+	fmt.Printf("plugins cmd = %v\n", plugins)
 	if err != nil {
 		return errors.WithMessage(err, "Failed to get plugins")
 	}
@@ -396,6 +403,7 @@ func VeleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNa
 		stderrBuf := new(bytes.Buffer)
 
 		installPluginCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "plugin", "add", plugin)
+		fmt.Printf("installPluginCmd cmd =%v\n", installPluginCmd)
 		installPluginCmd.Stdout = stdoutBuf
 		installPluginCmd.Stderr = stderrBuf
 
@@ -557,3 +565,34 @@ func getVeleroCliTarball(cliTarballUrl string) (*os.File, error) {
 
 	return tmpfile, nil
 }
+func DeleteBackupResource(ctx context.Context, veleroCLI string, backupName string) error {
+	args := []string{"backup", "delete", backupName, "--confirm"}
+
+	cmd := exec.CommandContext(ctx, veleroCLI, args...)
+	fmt.Println("Delete backup Command:" + cmd.String())
+	stdout, stderr, err := veleroexec.RunCommand(cmd)
+	if err != nil {
+		return errors.Wrapf(err, "Fail to get delete backup, stdout=%s, stderr=%s", stdout, stderr)
+	}
+
+	output := strings.Replace(stdout, "\n", " ", -1)
+	fmt.Println("Backup delete command output:" + output)
+
+	args = []string{"backup", "get", backupName}
+
+	retryTimes := 5
+	for i := 1; i < retryTimes+1; i++ {
+		cmd = exec.CommandContext(ctx, veleroCLI, args...)
+		fmt.Printf("Try %d times to delete backup %s \n", i, cmd.String())
+		stdout, stderr, err = veleroexec.RunCommand(cmd)
+		if err != nil {
+			if strings.Contains(stderr, "not found") {
+				fmt.Printf("|| EXPECTED || - Backup %s was deleted successfully according to message %s\n", backupName, stderr)
+				return nil
+			}
+			return errors.Wrapf(err, "Fail to get delete backup, stdout=%s, stderr=%s", stdout, stderr)
+		}
+		time.Sleep(1 * time.Minute)
+	}
+	return nil
+}