Sanitize Azure HTTP responses in BSL status messages

Azure storage errors include verbose HTTP response details and XML
in error messages, making the BSL status.message field cluttered
and hard to read. This change adds sanitization to extract only
the error code and meaningful message.

Before:
  BackupStorageLocation "test" is unavailable: rpc error: code = Unknown
  desc = GET https://...
  RESPONSE 404: 404 The specified container does not exist.
  ERROR CODE: ContainerNotFound
  <?xml version="1.0"...>

After:
  BackupStorageLocation "test" is unavailable: rpc error: code = Unknown
  desc = ContainerNotFound: The specified container does not exist.

AWS and GCP error messages are preserved as-is since they don't
contain verbose HTTP responses.

Fixes #8368

Signed-off-by: Shubham Pampattiwar <spampatt@redhat.com>
This commit is contained in:
Shubham Pampattiwar
2025-10-10 12:21:29 -07:00
parent 975f647323
commit a5d32f29da
2 changed files with 171 additions and 1 deletions

View File

@@ -18,6 +18,7 @@ package controller
import (
"context"
"regexp"
"strings"
"time"
@@ -46,6 +47,97 @@ const (
bslValidationEnqueuePeriod = 10 * time.Second
)
// sanitizeStorageError cleans up verbose HTTP responses from cloud provider errors,
// particularly Azure which includes full HTTP response details and XML in error messages.
// It extracts the error code and message while removing HTTP headers and response bodies.
func sanitizeStorageError(err error) string {
if err == nil {
return ""
}
errMsg := err.Error()
// Check if this looks like an Azure HTTP response error
// Azure errors contain patterns like "RESPONSE 404:" and "ERROR CODE:"
if !strings.Contains(errMsg, "RESPONSE") || !strings.Contains(errMsg, "ERROR CODE:") {
// Not an Azure-style error, return as-is
return errMsg
}
// Extract the error code (e.g., "ContainerNotFound", "BlobNotFound")
errorCodeRegex := regexp.MustCompile(`ERROR CODE:\s*(\w+)`)
errorCodeMatch := errorCodeRegex.FindStringSubmatch(errMsg)
var errorCode string
if len(errorCodeMatch) > 1 {
errorCode = errorCodeMatch[1]
}
// Extract the error message from the XML or plain text
// Look for message between <Message> tags or after "RESPONSE XXX:"
var errorMessage string
// Try to extract from XML first
messageRegex := regexp.MustCompile(`<Message>(.*?)</Message>`)
messageMatch := messageRegex.FindStringSubmatch(errMsg)
if len(messageMatch) > 1 {
errorMessage = messageMatch[1]
// Remove RequestId and Time from the message
if idx := strings.Index(errorMessage, "\nRequestId:"); idx != -1 {
errorMessage = errorMessage[:idx]
}
} else {
// Try to extract from plain text response (e.g., "RESPONSE 404: 404 The specified container does not exist.")
responseRegex := regexp.MustCompile(`RESPONSE\s+\d+:\s+\d+\s+([^\n]+)`)
responseMatch := responseRegex.FindStringSubmatch(errMsg)
if len(responseMatch) > 1 {
errorMessage = strings.TrimSpace(responseMatch[1])
}
}
// Build a clean error message
var cleanMsg string
if errorCode != "" && errorMessage != "" {
cleanMsg = errorCode + ": " + errorMessage
} else if errorCode != "" {
cleanMsg = errorCode
} else if errorMessage != "" {
cleanMsg = errorMessage
} else {
// Fallback: try to extract the desc part from gRPC error
descRegex := regexp.MustCompile(`desc\s*=\s*(.+)`)
descMatch := descRegex.FindStringSubmatch(errMsg)
if len(descMatch) > 1 {
// Take everything up to the first newline or "RESPONSE" marker
desc := descMatch[1]
if idx := strings.Index(desc, "\n"); idx != -1 {
desc = desc[:idx]
}
if idx := strings.Index(desc, "RESPONSE"); idx != -1 {
desc = strings.TrimSpace(desc[:idx])
}
cleanMsg = desc
} else {
// Last resort: return first line
if idx := strings.Index(errMsg, "\n"); idx != -1 {
cleanMsg = errMsg[:idx]
} else {
cleanMsg = errMsg
}
}
}
// Preserve the prefix part of the error (e.g., "rpc error: code = Unknown desc = ")
// but replace the verbose description with our clean message
if strings.Contains(errMsg, "desc = ") {
parts := strings.SplitN(errMsg, "desc = ", 2)
if len(parts) == 2 {
return parts[0] + "desc = " + cleanMsg
}
}
return cleanMsg
}
// BackupStorageLocationReconciler reconciles a BackupStorageLocation object
type backupStorageLocationReconciler struct {
ctx context.Context
@@ -127,7 +219,7 @@ func (r *backupStorageLocationReconciler) Reconcile(ctx context.Context, req ctr
err = errors.Wrapf(err, "BackupStorageLocation %q is unavailable", location.Name)
unavailableErrors = append(unavailableErrors, err.Error())
location.Status.Phase = velerov1api.BackupStorageLocationPhaseUnavailable
location.Status.Message = err.Error()
location.Status.Message = sanitizeStorageError(err)
} else {
log.Info("BackupStorageLocations is valid, marking as available")
location.Status.Phase = velerov1api.BackupStorageLocationPhaseAvailable

View File

@@ -303,3 +303,81 @@ func TestBSLReconcile(t *testing.T) {
})
}
}
func TestSanitizeStorageError(t *testing.T) {
tests := []struct {
name string
input error
expected string
}{
{
name: "Nil error",
input: nil,
expected: "",
},
{
name: "Simple error without Azure formatting",
input: errors.New("simple error message"),
expected: "simple error message",
},
{
name: "AWS style error",
input: errors.New("NoSuchBucket: The specified bucket does not exist"),
expected: "NoSuchBucket: The specified bucket does not exist",
},
{
name: "Azure container not found error with full HTTP response",
input: errors.New(`rpc error: code = Unknown desc = GET https://oadp100711zl59k.blob.core.windows.net/oadp100711zl59k1
--------------------------------------------------------------------------------
RESPONSE 404: 404 The specified container does not exist.
ERROR CODE: ContainerNotFound
--------------------------------------------------------------------------------
<?xml version="1.0" encoding="utf-8"?><Error><Code>ContainerNotFound</Code><Message>The specified container does not exist.
RequestId:63cf34d8-801e-0078-09b4-2e4682000000
Time:2024-11-04T12:23:04.5623627Z</Message></Error>
--------------------------------------------------------------------------------
`),
expected: "rpc error: code = Unknown desc = ContainerNotFound: The specified container does not exist.",
},
{
name: "Azure blob not found error",
input: errors.New(`rpc error: code = Unknown desc = GET https://storage.blob.core.windows.net/container/blob
--------------------------------------------------------------------------------
RESPONSE 404: 404 The specified blob does not exist.
ERROR CODE: BlobNotFound
--------------------------------------------------------------------------------
<?xml version="1.0" encoding="utf-8"?><Error><Code>BlobNotFound</Code><Message>The specified blob does not exist.
RequestId:12345678-1234-1234-1234-123456789012
Time:2024-11-04T12:23:04.5623627Z</Message></Error>
--------------------------------------------------------------------------------
`),
expected: "rpc error: code = Unknown desc = BlobNotFound: The specified blob does not exist.",
},
{
name: "Azure error with plain text response (no XML)",
input: errors.New(`rpc error: code = Unknown desc = GET https://storage.blob.core.windows.net/container
--------------------------------------------------------------------------------
RESPONSE 404: 404 The specified container does not exist.
ERROR CODE: ContainerNotFound
--------------------------------------------------------------------------------
`),
expected: "rpc error: code = Unknown desc = ContainerNotFound: The specified container does not exist.",
},
{
name: "Azure error without XML message but with error code",
input: errors.New(`rpc error: code = Unknown desc = operation failed
RESPONSE 403: 403 Forbidden
ERROR CODE: AuthorizationFailure
--------------------------------------------------------------------------------
`),
expected: "rpc error: code = Unknown desc = AuthorizationFailure: Forbidden",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
actual := sanitizeStorageError(test.input)
assert.Equal(t, test.expected, actual)
})
}
}