Merge branch 'main' into 9097

pkg/controller/backup_storage_location_controller.go

@@ -18,6 +18,7 @@ package controller
 
 import (
 	"context"
+	"regexp"
 	"strings"
 	"time"
 
@@ -46,6 +47,104 @@ const (
 	bslValidationEnqueuePeriod = 10 * time.Second
 )
 
+// sanitizeStorageError cleans up verbose HTTP responses from cloud provider errors,
+// particularly Azure which includes full HTTP response details and XML in error messages.
+// It extracts the error code and message while removing HTTP headers and response bodies.
+// It also scrubs sensitive information like SAS tokens from URLs.
+func sanitizeStorageError(err error) string {
+	if err == nil {
+		return ""
+	}
+
+	errMsg := err.Error()
+
+	// Scrub sensitive information from URLs (SAS tokens, credentials, etc.)
+	// Azure SAS token parameters: sig, se, st, sp, spr, sv, sr, sip, srt, ss
+	// These appear as query parameters in URLs like: ?sig=value&se=value
+	sasParamsRegex := regexp.MustCompile(`([?&])(sig|se|st|sp|spr|sv|sr|sip|srt|ss)=([^&\s<>\n]+)`)
+	errMsg = sasParamsRegex.ReplaceAllString(errMsg, `${1}${2}=***REDACTED***`)
+
+	// Check if this looks like an Azure HTTP response error
+	// Azure errors contain patterns like "RESPONSE 404:" and "ERROR CODE:"
+	if !strings.Contains(errMsg, "RESPONSE") || !strings.Contains(errMsg, "ERROR CODE:") {
+		// Not an Azure-style error, return as-is
+		return errMsg
+	}
+
+	// Extract the error code (e.g., "ContainerNotFound", "BlobNotFound")
+	errorCodeRegex := regexp.MustCompile(`ERROR CODE:\s*(\w+)`)
+	errorCodeMatch := errorCodeRegex.FindStringSubmatch(errMsg)
+	var errorCode string
+	if len(errorCodeMatch) > 1 {
+		errorCode = errorCodeMatch[1]
+	}
+
+	// Extract the error message from the XML or plain text
+	// Look for message between <Message> tags or after "RESPONSE XXX:"
+	var errorMessage string
+
+	// Try to extract from XML first
+	messageRegex := regexp.MustCompile(`<Message>(.*?)</Message>`)
+	messageMatch := messageRegex.FindStringSubmatch(errMsg)
+	if len(messageMatch) > 1 {
+		errorMessage = messageMatch[1]
+		// Remove RequestId and Time from the message
+		if idx := strings.Index(errorMessage, "\nRequestId:"); idx != -1 {
+			errorMessage = errorMessage[:idx]
+		}
+	} else {
+		// Try to extract from plain text response (e.g., "RESPONSE 404: 404 The specified container does not exist.")
+		responseRegex := regexp.MustCompile(`RESPONSE\s+\d+:\s+\d+\s+([^\n]+)`)
+		responseMatch := responseRegex.FindStringSubmatch(errMsg)
+		if len(responseMatch) > 1 {
+			errorMessage = strings.TrimSpace(responseMatch[1])
+		}
+	}
+
+	// Build a clean error message
+	var cleanMsg string
+	if errorCode != "" && errorMessage != "" {
+		cleanMsg = errorCode + ": " + errorMessage
+	} else if errorCode != "" {
+		cleanMsg = errorCode
+	} else if errorMessage != "" {
+		cleanMsg = errorMessage
+	} else {
+		// Fallback: try to extract the desc part from gRPC error
+		descRegex := regexp.MustCompile(`desc\s*=\s*(.+)`)
+		descMatch := descRegex.FindStringSubmatch(errMsg)
+		if len(descMatch) > 1 {
+			// Take everything up to the first newline or "RESPONSE" marker
+			desc := descMatch[1]
+			if idx := strings.Index(desc, "\n"); idx != -1 {
+				desc = desc[:idx]
+			}
+			if idx := strings.Index(desc, "RESPONSE"); idx != -1 {
+				desc = strings.TrimSpace(desc[:idx])
+			}
+			cleanMsg = desc
+		} else {
+			// Last resort: return first line
+			if idx := strings.Index(errMsg, "\n"); idx != -1 {
+				cleanMsg = errMsg[:idx]
+			} else {
+				cleanMsg = errMsg
+			}
+		}
+	}
+
+	// Preserve the prefix part of the error (e.g., "rpc error: code = Unknown desc = ")
+	// but replace the verbose description with our clean message
+	if strings.Contains(errMsg, "desc = ") {
+		parts := strings.SplitN(errMsg, "desc = ", 2)
+		if len(parts) == 2 {
+			return parts[0] + "desc = " + cleanMsg
+		}
+	}
+
+	return cleanMsg
+}
+
 // BackupStorageLocationReconciler reconciles a BackupStorageLocation object
 type backupStorageLocationReconciler struct {
 	ctx                       context.Context
@@ -125,9 +224,9 @@ func (r *backupStorageLocationReconciler) Reconcile(ctx context.Context, req ctr
 			if err != nil {
 				log.Info("BackupStorageLocation is invalid, marking as unavailable")
 				err = errors.Wrapf(err, "BackupStorageLocation %q is unavailable", location.Name)
-				unavailableErrors = append(unavailableErrors, err.Error())
+				unavailableErrors = append(unavailableErrors, sanitizeStorageError(err))
 				location.Status.Phase = velerov1api.BackupStorageLocationPhaseUnavailable
-				location.Status.Message = err.Error()
+				location.Status.Message = sanitizeStorageError(err)
 			} else {
 				log.Info("BackupStorageLocations is valid, marking as available")
 				location.Status.Phase = velerov1api.BackupStorageLocationPhaseAvailable

pkg/controller/backup_storage_location_controller_test.go

@@ -303,3 +303,115 @@ func TestBSLReconcile(t *testing.T) {
 		})
 	}
 }
+
+func TestSanitizeStorageError(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    error
+		expected string
+	}{
+		{
+			name:     "Nil error",
+			input:    nil,
+			expected: "",
+		},
+		{
+			name:     "Simple error without Azure formatting",
+			input:    errors.New("simple error message"),
+			expected: "simple error message",
+		},
+		{
+			name:     "AWS style error",
+			input:    errors.New("NoSuchBucket: The specified bucket does not exist"),
+			expected: "NoSuchBucket: The specified bucket does not exist",
+		},
+		{
+			name: "Azure container not found error with full HTTP response",
+			input: errors.New(`rpc error: code = Unknown desc = GET https://oadp100711zl59k.blob.core.windows.net/oadp100711zl59k1
+--------------------------------------------------------------------------------
+RESPONSE 404: 404 The specified container does not exist.
+ERROR CODE: ContainerNotFound
+--------------------------------------------------------------------------------
+<?xml version="1.0" encoding="utf-8"?><Error><Code>ContainerNotFound</Code><Message>The specified container does not exist.
+RequestId:63cf34d8-801e-0078-09b4-2e4682000000
+Time:2024-11-04T12:23:04.5623627Z</Message></Error>
+--------------------------------------------------------------------------------
+`),
+			expected: "rpc error: code = Unknown desc = ContainerNotFound: The specified container does not exist.",
+		},
+		{
+			name: "Azure blob not found error",
+			input: errors.New(`rpc error: code = Unknown desc = GET https://storage.blob.core.windows.net/container/blob
+--------------------------------------------------------------------------------
+RESPONSE 404: 404 The specified blob does not exist.
+ERROR CODE: BlobNotFound
+--------------------------------------------------------------------------------
+<?xml version="1.0" encoding="utf-8"?><Error><Code>BlobNotFound</Code><Message>The specified blob does not exist.
+RequestId:12345678-1234-1234-1234-123456789012
+Time:2024-11-04T12:23:04.5623627Z</Message></Error>
+--------------------------------------------------------------------------------
+`),
+			expected: "rpc error: code = Unknown desc = BlobNotFound: The specified blob does not exist.",
+		},
+		{
+			name: "Azure error with plain text response (no XML)",
+			input: errors.New(`rpc error: code = Unknown desc = GET https://storage.blob.core.windows.net/container
+--------------------------------------------------------------------------------
+RESPONSE 404: 404 The specified container does not exist.
+ERROR CODE: ContainerNotFound
+--------------------------------------------------------------------------------
+`),
+			expected: "rpc error: code = Unknown desc = ContainerNotFound: The specified container does not exist.",
+		},
+		{
+			name: "Azure error without XML message but with error code",
+			input: errors.New(`rpc error: code = Unknown desc = operation failed
+RESPONSE 403: 403 Forbidden
+ERROR CODE: AuthorizationFailure
+--------------------------------------------------------------------------------
+`),
+			expected: "rpc error: code = Unknown desc = AuthorizationFailure: Forbidden",
+		},
+		{
+			name: "Error with Azure SAS token in URL",
+			input: errors.New(`rpc error: code = Unknown desc = GET https://storage.blob.core.windows.net/backup?sv=2020-08-04&sig=abc123secrettoken&se=2024-12-31T23:59:59Z&sp=rwdl
+--------------------------------------------------------------------------------
+RESPONSE 404: 404 The specified container does not exist.
+ERROR CODE: ContainerNotFound
+--------------------------------------------------------------------------------
+`),
+			expected: "rpc error: code = Unknown desc = ContainerNotFound: The specified container does not exist.",
+		},
+		{
+			name:     "Error with multiple SAS parameters",
+			input:    errors.New(`GET https://mystorageaccount.blob.core.windows.net/container?sv=2020-08-04&ss=b&srt=sco&sp=rwdlac&se=2024-12-31&st=2024-01-01&sip=168.1.5.60&spr=https&sig=SIGNATURE_HASH`),
+			expected: "GET https://mystorageaccount.blob.core.windows.net/container?sv=***REDACTED***&ss=***REDACTED***&srt=***REDACTED***&sp=***REDACTED***&se=***REDACTED***&st=***REDACTED***&sip=***REDACTED***&spr=***REDACTED***&sig=***REDACTED***",
+		},
+		{
+			name:     "Simple URL without SAS tokens unchanged",
+			input:    errors.New("GET https://storage.blob.core.windows.net/container/blob"),
+			expected: "GET https://storage.blob.core.windows.net/container/blob",
+		},
+		{
+			name: "Azure error with SAS token in full HTTP response",
+			input: errors.New(`rpc error: code = Unknown desc = GET https://oadp100711zl59k.blob.core.windows.net/backup?sig=secretsignature123&se=2024-12-31
+--------------------------------------------------------------------------------
+RESPONSE 404: 404 The specified container does not exist.
+ERROR CODE: ContainerNotFound
+--------------------------------------------------------------------------------
+<?xml version="1.0" encoding="utf-8"?><Error><Code>ContainerNotFound</Code><Message>The specified container does not exist.
+RequestId:63cf34d8-801e-0078-09b4-2e4682000000
+Time:2024-11-04T12:23:04.5623627Z</Message></Error>
+--------------------------------------------------------------------------------
+`),
+			expected: "rpc error: code = Unknown desc = ContainerNotFound: The specified container does not exist.",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			actual := sanitizeStorageError(test.input)
+			assert.Equal(t, test.expected, actual)
+		})
+	}
+}

pkg/controller/backup_sync_controller.go

@@ -21,7 +21,6 @@ import (
 	"fmt"
 	"time"
 
-	snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -36,7 +35,6 @@ import (
 
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/constant"
-	"github.com/vmware-tanzu/velero/pkg/features"
 	"github.com/vmware-tanzu/velero/pkg/label"
 	"github.com/vmware-tanzu/velero/pkg/persistence"
 	"github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt"
@@ -243,31 +241,6 @@ func (b *backupSyncReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 				log.Debug("Synced pod volume backup into cluster")
 			}
 		}
-
-		if features.IsEnabled(velerov1api.CSIFeatureFlag) {
-			// we are syncing these objects only to ensure that the storage snapshots are cleaned up
-			// on backup deletion or expiry.
-			log.Info("Syncing CSI VolumeSnapshotClasses in backup")
-			vsClasses, err := backupStore.GetCSIVolumeSnapshotClasses(backupName)
-			if err != nil {
-				log.WithError(errors.WithStack(err)).Error("Error getting CSI VolumeSnapClasses for this backup from backup store")
-				continue
-			}
-			for _, vsClass := range vsClasses {
-				vsClass.ResourceVersion = ""
-				err := b.client.Create(ctx, vsClass, &client.CreateOptions{})
-				switch {
-				case err != nil && apierrors.IsAlreadyExists(err):
-					log.Debugf("VolumeSnapshotClass %s already exists in cluster", vsClass.Name)
-					continue
-				case err != nil && !apierrors.IsAlreadyExists(err):
-					log.WithError(errors.WithStack(err)).Errorf("Error syncing VolumeSnapshotClass %s into cluster", vsClass.Name)
-					continue
-				default:
-					log.Infof("Created CSI VolumeSnapshotClass %s", vsClass.Name)
-				}
-			}
-		}
 	}
 
 	b.deleteOrphanedBackups(ctx, location.Name, backupStoreBackups, log)
@@ -364,40 +337,10 @@ func (b *backupSyncReconciler) deleteOrphanedBackups(ctx context.Context, locati
 
 		if err := b.client.Delete(ctx, &backupList.Items[i], &client.DeleteOptions{}); err != nil {
 			log.WithError(errors.WithStack(err)).Error("Error deleting orphaned backup from cluster")
-		} else {
-			log.Debug("Deleted orphaned backup from cluster")
-			b.deleteCSISnapshotsByBackup(ctx, backup.Name, log)
 		}
 	}
 }
 
-func (b *backupSyncReconciler) deleteCSISnapshotsByBackup(ctx context.Context, backupName string, log logrus.FieldLogger) {
-	if !features.IsEnabled(velerov1api.CSIFeatureFlag) {
-		return
-	}
-	m := client.MatchingLabels{velerov1api.BackupNameLabel: label.GetValidName(backupName)}
-	var vsList snapshotv1api.VolumeSnapshotList
-	listOptions := &client.ListOptions{
-		LabelSelector: label.NewSelectorForBackup(label.GetValidName(backupName)),
-	}
-	if err := b.client.List(ctx, &vsList, listOptions); err != nil {
-		log.WithError(err).Warnf("Failed to list volumesnapshots for backup: %s, the deletion will be skipped", backupName)
-	} else {
-		for i, vs := range vsList.Items {
-			name := kube.NamespaceAndName(vs.GetObjectMeta())
-			log.Debugf("Deleting volumesnapshot %s", name)
-			if err := b.client.Delete(context.TODO(), &vsList.Items[i]); err != nil {
-				log.WithError(err).Warnf("Failed to delete volumesnapshot %s", name)
-			}
-		}
-	}
-	vsc := &snapshotv1api.VolumeSnapshotContent{}
-	log.Debugf("Deleting volumesnapshotcontents for backup: %s", backupName)
-	if err := b.client.DeleteAllOf(context.TODO(), vsc, m); err != nil {
-		log.WithError(err).Warnf("Failed to delete volumesnapshotcontents for backup: %s", backupName)
-	}
-}
-
 // backupSyncSourceOrderFunc returns a new slice with the default backup location first (if it exists),
 // followed by the rest of the locations in no particular order.
 func backupSyncSourceOrderFunc(objList client.ObjectList) client.ObjectList {

pkg/controller/backup_sync_controller_test.go

@@ -24,7 +24,6 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
-	snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1"
 	"github.com/sirupsen/logrus"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
@@ -451,8 +450,6 @@ var _ = Describe("Backup Sync Reconciler", func() {
 					backupStore.On("GetBackupMetadata", backup.backup.Name).Return(backup.backup, nil)
 					backupStore.On("GetPodVolumeBackups", backup.backup.Name).Return(backup.podVolumeBackups, nil)
 					backupStore.On("BackupExists", "bucket-1", backup.backup.Name).Return(true, nil)
-					backupStore.On("GetCSIVolumeSnapshotClasses", backup.backup.Name).Return([]*snapshotv1api.VolumeSnapshotClass{}, nil)
-					backupStore.On("GetCSIVolumeSnapshotContents", backup.backup.Name).Return([]*snapshotv1api.VolumeSnapshotContent{}, nil)
 				}
 				backupStore.On("ListBackups").Return(backupNames, nil)
 			}