From f4233c0f9f49b64b553f5ddaff4ab88627a5e4f8 Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Wed, 25 Dec 2024 12:00:44 +0700 Subject: [PATCH] CLI automatically discovers and uses cacert from BSL for download requests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tiger Kaovilai feat: Add CA cert fallback when caCertFile fails in download requests - Fallback to BSL cert when caCertFile cannot be opened - Combine certificate handling blocks to reuse CA pool initialization - Add comprehensive unit tests for fallback behavior This improves robustness by allowing downloads to proceed with BSL CA cert when the provided CA cert file is unavailable or unreadable. 🤖 Generated with [Claude Code](https://claude.ai/code) Signed-off-by: Tiger Kaovilai Co-Authored-By: Claude --- changelogs/unreleased/8557-kaovilai | 1 + pkg/cmd/cli/backup/describe.go | 10 +- pkg/cmd/cli/backup/download.go | 21 +- pkg/cmd/cli/backup/logs.go | 13 +- pkg/cmd/cli/backup/logs_test.go | 210 +++ pkg/cmd/cli/restore/logs.go | 15 +- pkg/cmd/cli/restore/logs_test.go | 126 +- pkg/cmd/util/cacert/bsl_cacert.go | 79 ++ pkg/cmd/util/cacert/bsl_cacert_test.go | 380 +++++ .../util/downloadrequest/downloadrequest.go | 50 +- .../downloadrequest/downloadrequest_test.go | 1217 +++++++++++++++++ pkg/cmd/util/output/backup_describer.go | 55 +- .../output/backup_structured_describer.go | 37 +- pkg/cmd/util/output/restore_describer.go | 43 +- .../main/api-types/backupstoragelocation.md | 21 + .../docs/main/self-signed-certificates.md | 35 +- 16 files changed, 2255 insertions(+), 58 deletions(-) create mode 100644 changelogs/unreleased/8557-kaovilai create mode 100644 pkg/cmd/util/cacert/bsl_cacert.go create mode 100644 pkg/cmd/util/cacert/bsl_cacert_test.go create mode 100644 pkg/cmd/util/downloadrequest/downloadrequest_test.go diff --git a/changelogs/unreleased/8557-kaovilai b/changelogs/unreleased/8557-kaovilai new file mode 100644 index 000000000..7abdbe9bd --- /dev/null +++ b/changelogs/unreleased/8557-kaovilai @@ -0,0 +1 @@ +CLI automatically discovers and uses cacert from BSL for download requests diff --git a/pkg/cmd/cli/backup/describe.go b/pkg/cmd/cli/backup/describe.go index 6466110a3..b0ef4a93e 100644 --- a/pkg/cmd/cli/backup/describe.go +++ b/pkg/cmd/cli/backup/describe.go @@ -62,21 +62,21 @@ func NewDescribeCommand(f client.Factory, use string) *cobra.Command { if len(args) > 0 { for _, name := range args { backup := new(velerov1api.Backup) - err := kbClient.Get(context.TODO(), controllerclient.ObjectKey{Namespace: f.Namespace(), Name: name}, backup) + err := kbClient.Get(context.Background(), controllerclient.ObjectKey{Namespace: f.Namespace(), Name: name}, backup) cmd.CheckError(err) backups.Items = append(backups.Items, *backup) } } else { parsedSelector, err := labels.Parse(listOptions.LabelSelector) cmd.CheckError(err) - err = kbClient.List(context.TODO(), backups, &controllerclient.ListOptions{LabelSelector: parsedSelector, Namespace: f.Namespace()}) + err = kbClient.List(context.Background(), backups, &controllerclient.ListOptions{LabelSelector: parsedSelector, Namespace: f.Namespace()}) cmd.CheckError(err) } first := true for i, backup := range backups.Items { deleteRequestList := new(velerov1api.DeleteBackupRequestList) - err := kbClient.List(context.TODO(), deleteRequestList, &controllerclient.ListOptions{ + err := kbClient.List(context.Background(), deleteRequestList, &controllerclient.ListOptions{ Namespace: f.Namespace(), LabelSelector: labels.SelectorFromSet(map[string]string{velerov1api.BackupNameLabel: label.GetValidName(backup.Name), velerov1api.BackupUIDLabel: string(backup.UID)}), }) @@ -85,7 +85,7 @@ func NewDescribeCommand(f client.Factory, use string) *cobra.Command { } podVolumeBackupList := new(velerov1api.PodVolumeBackupList) - err = kbClient.List(context.TODO(), podVolumeBackupList, &controllerclient.ListOptions{ + err = kbClient.List(context.Background(), podVolumeBackupList, &controllerclient.ListOptions{ Namespace: f.Namespace(), LabelSelector: labels.SelectorFromSet(map[string]string{velerov1api.BackupNameLabel: label.GetValidName(backup.Name)}), }) @@ -115,7 +115,7 @@ func NewDescribeCommand(f client.Factory, use string) *cobra.Command { c.Flags().StringVarP(&listOptions.LabelSelector, "selector", "l", listOptions.LabelSelector, "Only show items matching this label selector.") c.Flags().BoolVar(&details, "details", details, "Display additional detail in the command output.") c.Flags().BoolVar(&insecureSkipTLSVerify, "insecure-skip-tls-verify", insecureSkipTLSVerify, "If true, the object store's TLS certificate will not be checked for validity. This is insecure and susceptible to man-in-the-middle attacks. Not recommended for production.") - c.Flags().StringVar(&caCertFile, "cacert", caCertFile, "Path to a certificate bundle to use when verifying TLS connections.") + c.Flags().StringVar(&caCertFile, "cacert", caCertFile, "Path to a certificate bundle to use when verifying TLS connections. If not specified, the CA certificate from the BackupStorageLocation will be used if available.") c.Flags().StringVarP(&outputFormat, "output", "o", outputFormat, "Output display format. Valid formats are 'plaintext, json'. 'json' only applies to a single backup") return c diff --git a/pkg/cmd/cli/backup/download.go b/pkg/cmd/cli/backup/download.go index 5d2fec546..8bb973ff0 100644 --- a/pkg/cmd/cli/backup/download.go +++ b/pkg/cmd/cli/backup/download.go @@ -31,6 +31,7 @@ import ( velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" + "github.com/vmware-tanzu/velero/pkg/cmd/util/cacert" "github.com/vmware-tanzu/velero/pkg/cmd/util/downloadrequest" ) @@ -80,7 +81,7 @@ func (o *DownloadOptions) BindFlags(flags *pflag.FlagSet) { flags.BoolVar(&o.Force, "force", o.Force, "Forces the download and will overwrite file if it exists already.") flags.DurationVar(&o.Timeout, "timeout", o.Timeout, "Maximum time to wait to process download request.") flags.BoolVar(&o.InsecureSkipTLSVerify, "insecure-skip-tls-verify", o.InsecureSkipTLSVerify, "If true, the object store's TLS certificate will not be checked for validity. This is insecure and susceptible to man-in-the-middle attacks. Not recommended for production.") - flags.StringVar(&o.caCertFile, "cacert", o.caCertFile, "Path to a certificate bundle to use when verifying TLS connections.") + flags.StringVar(&o.caCertFile, "cacert", o.caCertFile, "Path to a certificate bundle to use when verifying TLS connections. If not specified, the CA certificate from the BackupStorageLocation will be used if available.") } func (o *DownloadOptions) Validate(c *cobra.Command, args []string, f client.Factory) error { @@ -88,7 +89,7 @@ func (o *DownloadOptions) Validate(c *cobra.Command, args []string, f client.Fac cmd.CheckError(err) backup := new(velerov1api.Backup) - if err := kbClient.Get(context.TODO(), controllerclient.ObjectKey{Namespace: f.Namespace(), Name: o.Name}, backup); err != nil { + if err := kbClient.Get(context.Background(), controllerclient.ObjectKey{Namespace: f.Namespace(), Name: o.Name}, backup); err != nil { return err } @@ -118,13 +119,27 @@ func (o *DownloadOptions) Run(c *cobra.Command, f client.Factory) error { kbClient, err := f.KubebuilderClient() cmd.CheckError(err) + // Get the backup to fetch BSL cacert + backup := new(velerov1api.Backup) + if err := kbClient.Get(context.Background(), controllerclient.ObjectKey{Namespace: f.Namespace(), Name: o.Name}, backup); err != nil { + return err + } + + // Get BSL cacert if available + bslCACert, err := cacert.GetCACertFromBackup(context.Background(), kbClient, f.Namespace(), backup) + if err != nil { + // Log the error but don't fail - we can still try to download without the BSL cacert + fmt.Fprintf(os.Stderr, "WARNING: Error getting cacert from BSL: %v\n", err) + bslCACert = "" + } + backupDest, err := os.OpenFile(o.Output, o.writeOptions, 0600) if err != nil { return err } defer backupDest.Close() - err = downloadrequest.Stream(context.Background(), kbClient, f.Namespace(), o.Name, velerov1api.DownloadTargetKindBackupContents, backupDest, o.Timeout, o.InsecureSkipTLSVerify, o.caCertFile) + err = downloadrequest.StreamWithBSLCACert(context.Background(), kbClient, f.Namespace(), o.Name, velerov1api.DownloadTargetKindBackupContents, backupDest, o.Timeout, o.InsecureSkipTLSVerify, o.caCertFile, bslCACert) if err != nil { os.Remove(o.Output) cmd.CheckError(err) diff --git a/pkg/cmd/cli/backup/logs.go b/pkg/cmd/cli/backup/logs.go index 0655eaa15..a0149acf1 100644 --- a/pkg/cmd/cli/backup/logs.go +++ b/pkg/cmd/cli/backup/logs.go @@ -30,6 +30,7 @@ import ( velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" + "github.com/vmware-tanzu/velero/pkg/cmd/util/cacert" "github.com/vmware-tanzu/velero/pkg/cmd/util/downloadrequest" ) @@ -62,7 +63,7 @@ func (l *LogsOptions) BindFlags(flags *pflag.FlagSet) { func (l *LogsOptions) Run(c *cobra.Command, f client.Factory) error { backup := new(velerov1api.Backup) - err := l.Client.Get(context.TODO(), kbclient.ObjectKey{Namespace: f.Namespace(), Name: l.BackupName}, backup) + err := l.Client.Get(context.Background(), kbclient.ObjectKey{Namespace: f.Namespace(), Name: l.BackupName}, backup) if apierrors.IsNotFound(err) { return fmt.Errorf("backup %q does not exist", l.BackupName) } else if err != nil { @@ -77,7 +78,15 @@ func (l *LogsOptions) Run(c *cobra.Command, f client.Factory) error { "until the backup has a phase of Completed or Failed and try again", l.BackupName) } - err = downloadrequest.Stream(context.Background(), l.Client, f.Namespace(), l.BackupName, velerov1api.DownloadTargetKindBackupLog, os.Stdout, l.Timeout, l.InsecureSkipTLSVerify, l.CaCertFile) + // Get BSL cacert if available + bslCACert, err := cacert.GetCACertFromBackup(context.Background(), l.Client, f.Namespace(), backup) + if err != nil { + // Log the error but don't fail - we can still try to download without the BSL cacert + fmt.Fprintf(os.Stderr, "WARNING: Error getting cacert from BSL: %v\n", err) + bslCACert = "" + } + + err = downloadrequest.StreamWithBSLCACert(context.Background(), l.Client, f.Namespace(), l.BackupName, velerov1api.DownloadTargetKindBackupLog, os.Stdout, l.Timeout, l.InsecureSkipTLSVerify, l.CaCertFile, bslCACert) return err } diff --git a/pkg/cmd/cli/backup/logs_test.go b/pkg/cmd/cli/backup/logs_test.go index 264068bc9..87e100732 100644 --- a/pkg/cmd/cli/backup/logs_test.go +++ b/pkg/cmd/cli/backup/logs_test.go @@ -17,7 +17,13 @@ limitations under the License. package backup import ( + "bytes" + "compress/gzip" "fmt" + "io" + "net/http" + "net/http/httptest" + "os" "strconv" "testing" "time" @@ -31,6 +37,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/builder" factorymocks "github.com/vmware-tanzu/velero/pkg/client/mocks" cmdtest "github.com/vmware-tanzu/velero/pkg/cmd/test" + "github.com/vmware-tanzu/velero/pkg/cmd/util/cacert" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) @@ -164,4 +171,207 @@ func TestNewLogsCommand(t *testing.T) { err := l.Complete([]string{""}, f) require.Equal(t, "test error", err.Error()) }) + + t.Run("Backup with BSL cacert test", func(t *testing.T) { + backupName := "bk-logs-with-cacert" + bslName := "test-bsl" + expectedCACert := "test-cacert-content" + expectedLogContent := "test backup log content" + + // create a factory + f := &factorymocks.Factory{} + + kbClient := velerotest.NewFakeControllerRuntimeClient(t) + + // Create BSL with cacert + bsl := builder.ForBackupStorageLocation(cmdtest.VeleroNameSpace, bslName). + Provider("aws"). + Bucket("test-bucket"). + CACert([]byte(expectedCACert)). + Result() + err := kbClient.Create(t.Context(), bsl, &kbclient.CreateOptions{}) + require.NoError(t, err) + + // Create backup referencing the BSL + backup := builder.ForBackup(cmdtest.VeleroNameSpace, backupName). + Phase(velerov1api.BackupPhaseCompleted). + StorageLocation(bslName). + Result() + err = kbClient.Create(t.Context(), backup, &kbclient.CreateOptions{}) + require.NoError(t, err) + + f.On("Namespace").Return(cmdtest.VeleroNameSpace) + f.On("KubebuilderClient").Return(kbClient, nil) + + c := NewLogsCommand(f) + assert.Equal(t, "Get backup logs", c.Short) + + l := NewLogsOptions() + flags := new(flag.FlagSet) + l.BindFlags(flags) + err = l.Complete([]string{backupName}, f) + require.NoError(t, err) + + // Verify that the BSL cacert can be fetched correctly before running the command + fetchedBackup := &velerov1api.Backup{} + err = kbClient.Get(t.Context(), kbclient.ObjectKey{Namespace: cmdtest.VeleroNameSpace, Name: backupName}, fetchedBackup) + require.NoError(t, err) + + // Test the cacert fetching logic directly + cacertValue, err := cacert.GetCACertFromBackup(t.Context(), kbClient, cmdtest.VeleroNameSpace, fetchedBackup) + require.NoError(t, err) + assert.Equal(t, expectedCACert, cacertValue, "BSL cacert should be retrieved correctly") + + // Create a mock HTTP server to serve the log content + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // For logs, we need to gzip the content + gzipWriter := gzip.NewWriter(w) + defer gzipWriter.Close() + gzipWriter.Write([]byte(expectedLogContent)) + })) + defer mockServer.Close() + + // Mock the download request controller by updating DownloadRequests + go func() { + time.Sleep(50 * time.Millisecond) // Wait a bit for the request to be created + + // List all DownloadRequests + downloadRequestList := &velerov1api.DownloadRequestList{} + if err := kbClient.List(t.Context(), downloadRequestList, &kbclient.ListOptions{ + Namespace: cmdtest.VeleroNameSpace, + }); err == nil { + // Update each download request with the mock server URL + for _, dr := range downloadRequestList.Items { + if dr.Spec.Target.Kind == velerov1api.DownloadTargetKindBackupLog && + dr.Spec.Target.Name == backupName { + dr.Status.DownloadURL = mockServer.URL + dr.Status.Phase = velerov1api.DownloadRequestPhaseProcessed + kbClient.Update(t.Context(), &dr) + } + } + } + }() + + // Capture the output + var logOutput bytes.Buffer + // Temporarily redirect stdout + oldStdout := os.Stdout + r, w, _ := os.Pipe() + os.Stdout = w + + // Run the logs command - it should now succeed + l.Timeout = 5 * time.Second + err = l.Run(c, f) + + // Restore stdout and read the output + w.Close() + os.Stdout = oldStdout + io.Copy(&logOutput, r) + + // Verify the command succeeded and output is correct + require.NoError(t, err) + assert.Equal(t, expectedLogContent, logOutput.String()) + }) +} + +func TestBSLCACertBehavior(t *testing.T) { + t.Run("Backup with BSL without cacert test", func(t *testing.T) { + backupName := "bk-logs-without-cacert" + bslName := "test-bsl-no-cacert" + + // create a factory + f := &factorymocks.Factory{} + + kbClient := velerotest.NewFakeControllerRuntimeClient(t) + + // Create BSL without cacert + bsl := builder.ForBackupStorageLocation(cmdtest.VeleroNameSpace, bslName). + Provider("aws"). + Bucket("test-bucket"). + // No CACert() call - BSL will have no cacert + Result() + err := kbClient.Create(t.Context(), bsl, &kbclient.CreateOptions{}) + require.NoError(t, err) + + // Create backup referencing the BSL + backup := builder.ForBackup(cmdtest.VeleroNameSpace, backupName). + Phase(velerov1api.BackupPhaseCompleted). + StorageLocation(bslName). + Result() + err = kbClient.Create(t.Context(), backup, &kbclient.CreateOptions{}) + require.NoError(t, err) + + f.On("Namespace").Return(cmdtest.VeleroNameSpace) + f.On("KubebuilderClient").Return(kbClient, nil) + + c := NewLogsCommand(f) + assert.Equal(t, "Get backup logs", c.Short) + + l := NewLogsOptions() + flags := new(flag.FlagSet) + l.BindFlags(flags) + err = l.Complete([]string{backupName}, f) + require.NoError(t, err) + + // Verify that the BSL cacert returns empty string when not present + fetchedBackup := &velerov1api.Backup{} + err = kbClient.Get(t.Context(), kbclient.ObjectKey{Namespace: cmdtest.VeleroNameSpace, Name: backupName}, fetchedBackup) + require.NoError(t, err) + + // Test the cacert fetching logic directly + cacertValue, err := cacert.GetCACertFromBackup(t.Context(), kbClient, cmdtest.VeleroNameSpace, fetchedBackup) + require.NoError(t, err) + assert.Empty(t, cacertValue, "BSL cacert should be empty when not configured") + + // The command should still work without cacert + l.Timeout = 100 * time.Millisecond + err = l.Run(c, f) + require.Error(t, err) + // The error should be about download request timeout, not about cacert fetching + assert.Contains(t, err.Error(), "download") + }) + + t.Run("Backup with nonexistent BSL test", func(t *testing.T) { + backupName := "bk-logs-with-missing-bsl" + bslName := "nonexistent-bsl" + + // create a factory + f := &factorymocks.Factory{} + + kbClient := velerotest.NewFakeControllerRuntimeClient(t) + + // Create backup referencing a BSL that doesn't exist + backup := builder.ForBackup(cmdtest.VeleroNameSpace, backupName). + Phase(velerov1api.BackupPhaseCompleted). + StorageLocation(bslName). + Result() + err := kbClient.Create(t.Context(), backup, &kbclient.CreateOptions{}) + require.NoError(t, err) + + f.On("Namespace").Return(cmdtest.VeleroNameSpace) + f.On("KubebuilderClient").Return(kbClient, nil) + + c := NewLogsCommand(f) + l := NewLogsOptions() + flags := new(flag.FlagSet) + l.BindFlags(flags) + err = l.Complete([]string{backupName}, f) + require.NoError(t, err) + + // Verify that the BSL cacert returns empty string when BSL doesn't exist + fetchedBackup := &velerov1api.Backup{} + err = kbClient.Get(t.Context(), kbclient.ObjectKey{Namespace: cmdtest.VeleroNameSpace, Name: backupName}, fetchedBackup) + require.NoError(t, err) + + // Test the cacert fetching logic directly - should not error when BSL is missing + cacertValue, err := cacert.GetCACertFromBackup(t.Context(), kbClient, cmdtest.VeleroNameSpace, fetchedBackup) + require.NoError(t, err) + assert.Empty(t, cacertValue, "BSL cacert should be empty when BSL doesn't exist") + + // The command should still try to run even without BSL + l.Timeout = 100 * time.Millisecond + err = l.Run(c, f) + require.Error(t, err) + assert.Contains(t, err.Error(), "download") + }) } diff --git a/pkg/cmd/cli/restore/logs.go b/pkg/cmd/cli/restore/logs.go index e88945868..f4315c917 100644 --- a/pkg/cmd/cli/restore/logs.go +++ b/pkg/cmd/cli/restore/logs.go @@ -29,6 +29,7 @@ import ( velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" + "github.com/vmware-tanzu/velero/pkg/cmd/util/cacert" "github.com/vmware-tanzu/velero/pkg/cmd/util/downloadrequest" ) @@ -53,7 +54,7 @@ func NewLogsCommand(f client.Factory) *cobra.Command { cmd.CheckError(err) restore := new(velerov1api.Restore) - err = kbClient.Get(context.TODO(), ctrlclient.ObjectKey{Namespace: f.Namespace(), Name: restoreName}, restore) + err = kbClient.Get(context.Background(), ctrlclient.ObjectKey{Namespace: f.Namespace(), Name: restoreName}, restore) if apierrors.IsNotFound(err) { cmd.Exit("Restore %q does not exist.", restoreName) } else if err != nil { @@ -68,14 +69,22 @@ func NewLogsCommand(f client.Factory) *cobra.Command { "until the restore has a phase of Completed or Failed and try again.", restoreName) } - err = downloadrequest.Stream(context.Background(), kbClient, f.Namespace(), restoreName, velerov1api.DownloadTargetKindRestoreLog, os.Stdout, timeout, insecureSkipTLSVerify, caCertFile) + // Get BSL cacert if available + bslCACert, err := cacert.GetCACertFromRestore(context.Background(), kbClient, f.Namespace(), restore) + if err != nil { + // Log the error but don't fail - we can still try to download without the BSL cacert + fmt.Fprintf(os.Stderr, "WARNING: Error getting cacert from BSL: %v\n", err) + bslCACert = "" + } + + err = downloadrequest.StreamWithBSLCACert(context.Background(), kbClient, f.Namespace(), restoreName, velerov1api.DownloadTargetKindRestoreLog, os.Stdout, timeout, insecureSkipTLSVerify, caCertFile, bslCACert) cmd.CheckError(err) }, } c.Flags().DurationVar(&timeout, "timeout", timeout, "How long to wait to receive logs.") c.Flags().BoolVar(&insecureSkipTLSVerify, "insecure-skip-tls-verify", insecureSkipTLSVerify, "If true, the object store's TLS certificate will not be checked for validity. This is insecure and susceptible to man-in-the-middle attacks. Not recommended for production.") - c.Flags().StringVar(&caCertFile, "cacert", caCertFile, "Path to a certificate bundle to use when verifying TLS connections.") + c.Flags().StringVar(&caCertFile, "cacert", caCertFile, "Path to a certificate bundle to use when verifying TLS connections. If not specified, the CA certificate from the BackupStorageLocation will be used if available.") return c } diff --git a/pkg/cmd/cli/restore/logs_test.go b/pkg/cmd/cli/restore/logs_test.go index 007da7044..61c2392b6 100644 --- a/pkg/cmd/cli/restore/logs_test.go +++ b/pkg/cmd/cli/restore/logs_test.go @@ -19,13 +19,18 @@ package restore import ( "os" "testing" + "time" - flag "github.com/spf13/pflag" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/builder" factorymocks "github.com/vmware-tanzu/velero/pkg/client/mocks" cmdtest "github.com/vmware-tanzu/velero/pkg/cmd/test" + "github.com/vmware-tanzu/velero/pkg/cmd/util/cacert" + velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestNewLogsCommand(t *testing.T) { @@ -35,15 +40,124 @@ func TestNewLogsCommand(t *testing.T) { c := NewLogsCommand(f) require.Equal(t, "Get restore logs", c.Short) - flags := new(flag.FlagSet) - timeout := "1m0s" + // Test flag parsing + timeout := "1s" insecureSkipTLSVerify := "true" caCertFile := "testing" - flags.Parse([]string{"--timeout", timeout}) - flags.Parse([]string{"--insecure-skip-tls-verify", insecureSkipTLSVerify}) - flags.Parse([]string{"--cacert", caCertFile}) + c.Flags().Set("timeout", timeout) + c.Flags().Set("insecure-skip-tls-verify", insecureSkipTLSVerify) + c.Flags().Set("cacert", caCertFile) + + timeoutFlag, _ := c.Flags().GetDuration("timeout") + require.Equal(t, 1*time.Second, timeoutFlag) + + insecureFlag, _ := c.Flags().GetBool("insecure-skip-tls-verify") + require.True(t, insecureFlag) + + caCertFlag, _ := c.Flags().GetString("cacert") + require.Equal(t, caCertFile, caCertFlag) + }) + + t.Run("Restore not complete test", func(t *testing.T) { + restoreName := "rs-logs-1" + + // create a factory + f := &factorymocks.Factory{} + + kbClient := velerotest.NewFakeControllerRuntimeClient(t) + restore := builder.ForRestore(cmdtest.VeleroNameSpace, restoreName).Result() + err := kbClient.Create(t.Context(), restore, &kbclient.CreateOptions{}) + require.NoError(t, err) + + f.On("Namespace").Return(cmdtest.VeleroNameSpace) + f.On("KubebuilderClient").Return(kbClient, nil) + + c := NewLogsCommand(f) + assert.Equal(t, "Get restore logs", c.Short) + + // The restore command exits with an error message when restore is not complete + // We can't easily test this since it calls cmd.Exit, which exits the process + // So we'll skip this test case + t.Skip("Cannot test restore not complete case due to cmd.Exit() call") + }) + + t.Run("Restore not exist test", func(t *testing.T) { + // create a factory + f := &factorymocks.Factory{} + + kbClient := velerotest.NewFakeControllerRuntimeClient(t) + + f.On("Namespace").Return(cmdtest.VeleroNameSpace) + f.On("KubebuilderClient").Return(kbClient, nil) + + c := NewLogsCommand(f) + assert.Equal(t, "Get restore logs", c.Short) + + // The restore command exits with an error message when restore doesn't exist + // We can't easily test this since it calls cmd.Exit, which exits the process + // So we'll skip this test case + t.Skip("Cannot test restore not exist case due to cmd.Exit() call") + }) + + t.Run("Restore with BSL cacert test", func(t *testing.T) { + restoreName := "rs-logs-with-cacert" + backupName := "bk-for-restore" + bslName := "test-bsl" + + // create a factory + f := &factorymocks.Factory{} + + kbClient := velerotest.NewFakeControllerRuntimeClient(t) + + // Create BSL with cacert + bsl := builder.ForBackupStorageLocation(cmdtest.VeleroNameSpace, bslName). + Provider("aws"). + Bucket("test-bucket"). + CACert([]byte("test-cacert-content")). + Result() + err := kbClient.Create(t.Context(), bsl, &kbclient.CreateOptions{}) + require.NoError(t, err) + + // Create backup referencing the BSL + backup := builder.ForBackup(cmdtest.VeleroNameSpace, backupName). + StorageLocation(bslName). + Result() + err = kbClient.Create(t.Context(), backup, &kbclient.CreateOptions{}) + require.NoError(t, err) + + // Create restore referencing the backup + restore := builder.ForRestore(cmdtest.VeleroNameSpace, restoreName). + Phase(velerov1api.RestorePhaseCompleted). + Backup(backupName). + Result() + err = kbClient.Create(t.Context(), restore, &kbclient.CreateOptions{}) + require.NoError(t, err) + + f.On("Namespace").Return(cmdtest.VeleroNameSpace) + f.On("KubebuilderClient").Return(kbClient, nil) + + c := NewLogsCommand(f) + assert.Equal(t, "Get restore logs", c.Short) + + // We can verify that BSL cacert fetching logic is in place + // The actual command will call downloadrequest which requires a controller + // to be running, so we'll just verify the command structure + require.NotNil(t, c.Run) + + // Verify the BSL cacert can be fetched + cacertValue, err := cacert.GetCACertFromRestore(t.Context(), kbClient, f.Namespace(), restore) + require.NoError(t, err) + require.Equal(t, "test-cacert-content", cacertValue) + }) + + t.Run("CLI execution test", func(t *testing.T) { + // create a factory + f := &factorymocks.Factory{} + + c := NewLogsCommand(f) + require.Equal(t, "Get restore logs", c.Short) if os.Getenv(cmdtest.CaptureFlag) == "1" { c.SetArgs([]string{"test"}) diff --git a/pkg/cmd/util/cacert/bsl_cacert.go b/pkg/cmd/util/cacert/bsl_cacert.go new file mode 100644 index 000000000..985a8a9f5 --- /dev/null +++ b/pkg/cmd/util/cacert/bsl_cacert.go @@ -0,0 +1,79 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cacert + +import ( + "context" + + "github.com/pkg/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" +) + +// GetCACertFromBackup fetches the BackupStorageLocation for a backup and returns its cacert +func GetCACertFromBackup(ctx context.Context, client kbclient.Client, namespace string, backup *velerov1api.Backup) (string, error) { + return GetCACertFromBSL(ctx, client, namespace, backup.Spec.StorageLocation) +} + +// GetCACertFromRestore fetches the BackupStorageLocation for a restore's backup and returns its cacert +func GetCACertFromRestore(ctx context.Context, client kbclient.Client, namespace string, restore *velerov1api.Restore) (string, error) { + // First get the backup that this restore references + backup := &velerov1api.Backup{} + key := kbclient.ObjectKey{ + Namespace: namespace, + Name: restore.Spec.BackupName, + } + + if err := client.Get(ctx, key, backup); err != nil { + if apierrors.IsNotFound(err) { + // Backup not found is not a fatal error for cacert retrieval + return "", nil + } + return "", errors.Wrapf(err, "error getting backup %s", restore.Spec.BackupName) + } + + return GetCACertFromBackup(ctx, client, namespace, backup) +} + +// GetCACertFromBSL fetches a BackupStorageLocation directly and returns its cacert +func GetCACertFromBSL(ctx context.Context, client kbclient.Client, namespace, bslName string) (string, error) { + if bslName == "" { + return "", nil + } + + bsl := &velerov1api.BackupStorageLocation{} + key := kbclient.ObjectKey{ + Namespace: namespace, + Name: bslName, + } + + if err := client.Get(ctx, key, bsl); err != nil { + if apierrors.IsNotFound(err) { + // BSL not found is not a fatal error, just means no cacert + return "", nil + } + return "", errors.Wrapf(err, "error getting backup storage location %s", bslName) + } + + if bsl.Spec.ObjectStorage != nil && len(bsl.Spec.ObjectStorage.CACert) > 0 { + return string(bsl.Spec.ObjectStorage.CACert), nil + } + + return "", nil +} diff --git a/pkg/cmd/util/cacert/bsl_cacert_test.go b/pkg/cmd/util/cacert/bsl_cacert_test.go new file mode 100644 index 000000000..364978a93 --- /dev/null +++ b/pkg/cmd/util/cacert/bsl_cacert_test.go @@ -0,0 +1,380 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cacert + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/builder" + "github.com/vmware-tanzu/velero/pkg/util" +) + +func TestGetCACertFromBackup(t *testing.T) { + testCases := []struct { + name string + backup *velerov1api.Backup + bsl *velerov1api.BackupStorageLocation + expectedCACert string + expectedError bool + }{ + { + name: "backup with BSL containing cacert", + backup: builder.ForBackup("test-ns", "test-backup"). + StorageLocation("test-bsl"). + Result(), + bsl: builder.ForBackupStorageLocation("test-ns", "test-bsl"). + Provider("aws"). + Bucket("test-bucket"). + CACert([]byte("test-cacert-content")). + Result(), + expectedCACert: "test-cacert-content", + expectedError: false, + }, + { + name: "backup with BSL without cacert", + backup: builder.ForBackup("test-ns", "test-backup"). + StorageLocation("test-bsl"). + Result(), + bsl: builder.ForBackupStorageLocation("test-ns", "test-bsl"). + Provider("aws"). + Bucket("test-bucket"). + Result(), + expectedCACert: "", + expectedError: false, + }, + { + name: "backup without storage location", + backup: builder.ForBackup("test-ns", "test-backup"). + Result(), + bsl: nil, + expectedCACert: "", + expectedError: false, + }, + { + name: "BSL not found", + backup: builder.ForBackup("test-ns", "test-backup"). + StorageLocation("missing-bsl"). + Result(), + bsl: nil, + expectedCACert: "", + expectedError: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var objs []runtime.Object + objs = append(objs, tc.backup) + if tc.bsl != nil { + objs = append(objs, tc.bsl) + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(util.VeleroScheme). + WithRuntimeObjects(objs...). + Build() + + cacert, err := GetCACertFromBackup(t.Context(), fakeClient, "test-ns", tc.backup) + + if tc.expectedError { + assert.Error(t, err) + } else { + require.NoError(t, err) + assert.Equal(t, tc.expectedCACert, cacert) + } + }) + } +} + +func TestGetCACertFromRestore(t *testing.T) { + testCases := []struct { + name string + restore *velerov1api.Restore + backup *velerov1api.Backup + bsl *velerov1api.BackupStorageLocation + expectedCACert string + expectedError bool + }{ + { + name: "restore with backup having BSL containing cacert", + restore: builder.ForRestore("test-ns", "test-restore"). + Backup("test-backup"). + Result(), + backup: builder.ForBackup("test-ns", "test-backup"). + StorageLocation("test-bsl"). + Result(), + bsl: builder.ForBackupStorageLocation("test-ns", "test-bsl"). + Provider("aws"). + Bucket("test-bucket"). + CACert([]byte("test-cacert-content")). + Result(), + expectedCACert: "test-cacert-content", + expectedError: false, + }, + { + name: "restore with backup not found", + restore: builder.ForRestore("test-ns", "test-restore"). + Backup("missing-backup"). + Result(), + backup: nil, + bsl: nil, + expectedCACert: "", + expectedError: false, + }, + { + name: "restore with backup having BSL without cacert", + restore: builder.ForRestore("test-ns", "test-restore"). + Backup("test-backup"). + Result(), + backup: builder.ForBackup("test-ns", "test-backup"). + StorageLocation("test-bsl"). + Result(), + bsl: builder.ForBackupStorageLocation("test-ns", "test-bsl"). + Provider("aws"). + Bucket("test-bucket"). + Result(), + expectedCACert: "", + expectedError: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var objs []runtime.Object + objs = append(objs, tc.restore) + if tc.backup != nil { + objs = append(objs, tc.backup) + } + if tc.bsl != nil { + objs = append(objs, tc.bsl) + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(util.VeleroScheme). + WithRuntimeObjects(objs...). + Build() + + cacert, err := GetCACertFromRestore(t.Context(), fakeClient, "test-ns", tc.restore) + + if tc.expectedError { + assert.Error(t, err) + } else { + require.NoError(t, err) + assert.Equal(t, tc.expectedCACert, cacert) + } + }) + } +} + +func TestGetCACertFromBSL(t *testing.T) { + testCases := []struct { + name string + bslName string + bsl *velerov1api.BackupStorageLocation + expectedCACert string + expectedError bool + }{ + { + name: "BSL with cacert", + bslName: "test-bsl", + bsl: builder.ForBackupStorageLocation("test-ns", "test-bsl"). + Provider("aws"). + Bucket("test-bucket"). + CACert([]byte("test-cacert-content")). + Result(), + expectedCACert: "test-cacert-content", + expectedError: false, + }, + { + name: "BSL without cacert", + bslName: "test-bsl", + bsl: builder.ForBackupStorageLocation("test-ns", "test-bsl"). + Provider("aws"). + Bucket("test-bucket"). + Result(), + expectedCACert: "", + expectedError: false, + }, + { + name: "empty BSL name", + bslName: "", + bsl: nil, + expectedCACert: "", + expectedError: false, + }, + { + name: "BSL not found", + bslName: "missing-bsl", + bsl: nil, + expectedCACert: "", + expectedError: false, + }, + { + name: "BSL with invalid CA cert format", + bslName: "test-bsl", + bsl: builder.ForBackupStorageLocation("test-ns", "test-bsl"). + Provider("aws"). + Bucket("test-bucket"). + CACert([]byte("INVALID CERT DATA WITHOUT PEM HEADERS")). + Result(), + expectedCACert: "INVALID CERT DATA WITHOUT PEM HEADERS", // We still return it, validation happens during TLS handshake + expectedError: false, + }, + { + name: "BSL with malformed PEM certificate", + bslName: "test-bsl", + bsl: builder.ForBackupStorageLocation("test-ns", "test-bsl"). + Provider("aws"). + Bucket("test-bucket"). + CACert([]byte("-----BEGIN CERTIFICATE-----\nINVALID BASE64 DATA!!!\n-----END CERTIFICATE-----\n")). + Result(), + expectedCACert: "-----BEGIN CERTIFICATE-----\nINVALID BASE64 DATA!!!\n-----END CERTIFICATE-----\n", + expectedError: false, + }, + { + name: "BSL with nil config", + bslName: "test-bsl", + bsl: &velerov1api.BackupStorageLocation{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-ns", + Name: "test-bsl", + }, + Spec: velerov1api.BackupStorageLocationSpec{ + Provider: "aws", + Config: nil, + }, + }, + expectedCACert: "", + expectedError: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var objs []runtime.Object + if tc.bsl != nil { + objs = append(objs, tc.bsl) + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(util.VeleroScheme). + WithRuntimeObjects(objs...). + Build() + + cacert, err := GetCACertFromBSL(t.Context(), fakeClient, "test-ns", tc.bslName) + + if tc.expectedError { + assert.Error(t, err) + } else { + require.NoError(t, err) + assert.Equal(t, tc.expectedCACert, cacert) + } + }) + } +} + +// TestGetCACertFromBackup_ClientError tests error scenarios where client.Get returns non-NotFound errors +func TestGetCACertFromBackup_ClientError(t *testing.T) { + testCases := []struct { + name string + backup *velerov1api.Backup + bsl *velerov1api.BackupStorageLocation + expectedError string + }{ + { + name: "client error getting BSL", + backup: builder.ForBackup("test-ns", "test-backup"). + StorageLocation("test-bsl"). + Result(), + bsl: builder.ForBackupStorageLocation("different-ns", "test-bsl"). // Different namespace to trigger error + Provider("aws"). + Bucket("test-bucket"). + CACert([]byte("test-cacert-content")). + Result(), + expectedError: "not found", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var objs []runtime.Object + objs = append(objs, tc.backup) + if tc.bsl != nil { + objs = append(objs, tc.bsl) + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(util.VeleroScheme). + WithRuntimeObjects(objs...). + Build() + + // Try to get BSL from wrong namespace to simulate error + _, err := GetCACertFromBSL(t.Context(), fakeClient, "wrong-ns", tc.backup.Spec.StorageLocation) + + require.NoError(t, err) // Not found errors are handled gracefully + }) + } +} + +// TestGetCACertFromRestore_ClientError tests error scenarios for GetCACertFromRestore +func TestGetCACertFromRestore_ClientError(t *testing.T) { + testCases := []struct { + name string + restore *velerov1api.Restore + backup *velerov1api.Backup + expectedError string + }{ + { + name: "backup in different namespace", + restore: builder.ForRestore("test-ns", "test-restore"). + Backup("test-backup"). + Result(), + backup: builder.ForBackup("different-ns", "test-backup"). // Different namespace + StorageLocation("test-bsl"). + Result(), + expectedError: "not found", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var objs []runtime.Object + objs = append(objs, tc.restore) + if tc.backup != nil { + objs = append(objs, tc.backup) + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(util.VeleroScheme). + WithRuntimeObjects(objs...). + Build() + + // This should not find the backup in the wrong namespace + cacert, err := GetCACertFromRestore(t.Context(), fakeClient, "test-ns", tc.restore) + + require.NoError(t, err) // Not found errors are handled gracefully, returning empty string + assert.Empty(t, cacert) + }) + } +} diff --git a/pkg/cmd/util/downloadrequest/downloadrequest.go b/pkg/cmd/util/downloadrequest/downloadrequest.go index a76d2bb80..6e1d30c37 100644 --- a/pkg/cmd/util/downloadrequest/downloadrequest.go +++ b/pkg/cmd/util/downloadrequest/downloadrequest.go @@ -50,6 +50,22 @@ func Stream( timeout time.Duration, insecureSkipTLSVerify bool, caCertFile string, +) error { + return StreamWithBSLCACert(ctx, kbClient, namespace, name, kind, w, timeout, insecureSkipTLSVerify, caCertFile, "") +} + +// StreamWithBSLCACert is like Stream but accepts an additional bslCACert parameter +// that contains the cacert from the BackupStorageLocation config +func StreamWithBSLCACert( + ctx context.Context, + kbClient kbclient.Client, + namespace, name string, + kind veleroV1api.DownloadTargetKind, + w io.Writer, + timeout time.Duration, + insecureSkipTLSVerify bool, + caCertFile string, + bslCACert string, ) error { ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() @@ -59,7 +75,7 @@ func Stream( return err } - if err := download(ctx, downloadURL, kind, w, insecureSkipTLSVerify, caCertFile); err != nil { + if err := download(ctx, downloadURL, kind, w, insecureSkipTLSVerify, caCertFile, bslCACert); err != nil { return err } @@ -109,21 +125,35 @@ func download( w io.Writer, insecureSkipTLSVerify bool, caCertFile string, + caCertByteString string, ) error { var caPool *x509.CertPool + var err error + + // Initialize caPool once + caPool, err = x509.SystemCertPool() + if err != nil { + caPool = x509.NewCertPool() + } + + // Try to load CA cert from file first if len(caCertFile) > 0 { caCert, err := os.ReadFile(caCertFile) if err != nil { - return errors.Wrapf(err, "couldn't open cacert") + // If caCertFile fails and BSL cert is available, fall back to it + if len(caCertByteString) > 0 { + fmt.Fprintf(os.Stderr, "Warning: Failed to open CA certificate file %s: %v. Using CA certificate from backup storage location instead.\n", caCertFile, err) + caPool.AppendCertsFromPEM([]byte(caCertByteString)) + } else { + // If no BSL cert available, return the original error + return errors.Wrapf(err, "couldn't open cacert") + } + } else { + caPool.AppendCertsFromPEM(caCert) } - // bundle the passed in cert with the system cert pool - // if it's available, otherwise create a new pool just - // for this. - caPool, err = x509.SystemCertPool() - if err != nil { - caPool = x509.NewCertPool() - } - caPool.AppendCertsFromPEM(caCert) + } else if len(caCertByteString) > 0 { + // If no caCertFile specified, use BSL cert if available + caPool.AppendCertsFromPEM([]byte(caCertByteString)) } defaultTransport := http.DefaultTransport.(*http.Transport) diff --git a/pkg/cmd/util/downloadrequest/downloadrequest_test.go b/pkg/cmd/util/downloadrequest/downloadrequest_test.go new file mode 100644 index 000000000..995e83dc6 --- /dev/null +++ b/pkg/cmd/util/downloadrequest/downloadrequest_test.go @@ -0,0 +1,1217 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package downloadrequest + +import ( + "bytes" + "compress/gzip" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "net" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/builder" + "github.com/vmware-tanzu/velero/pkg/cmd/util/cacert" + "github.com/vmware-tanzu/velero/pkg/util" +) + +// createSelfSignedCertificate creates a self-signed certificate for testing. +// This allows us to test the BSL CA certificate functionality by ensuring +// that the client properly validates server certificates against the CA cert +// provided in the BackupStorageLocation configuration. +func createSelfSignedCertificate(t *testing.T) (tls.Certificate, []byte) { + t.Helper() + + // Generate a private key + priv, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + + // Create certificate template for a self-signed certificate + template := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{ + Organization: []string{"Test Org"}, + Country: []string{"US"}, + Province: []string{""}, + Locality: []string{"Test City"}, + StreetAddress: []string{""}, + PostalCode: []string{""}, + }, + NotBefore: time.Now(), + NotAfter: time.Now().Add(365 * 24 * time.Hour), + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1)}, + DNSNames: []string{"localhost"}, + } + + // Create the certificate + certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) + require.NoError(t, err) + + // Encode certificate and key to PEM + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}) + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}) + + // Create tls.Certificate + tlsCert, err := tls.X509KeyPair(certPEM, keyPEM) + require.NoError(t, err) + + return tlsCert, certPEM +} + +func TestStream(t *testing.T) { + // Create a test server that returns download content + testContent := "test backup content" + downloadServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + if strings.Contains(r.URL.Path, "log") { + // For logs, return gzipped content + gzipWriter := gzip.NewWriter(w) + gzipWriter.Write([]byte(testContent)) + gzipWriter.Close() + } else { + w.Write([]byte(testContent)) + } + })) + defer downloadServer.Close() + + testCases := []struct { + name string + target velerov1api.DownloadTargetKind + timeout time.Duration + setupClient func(*testing.T, kbclient.WithWatch) + expectedError bool + expectedErrMessage string + validateContent func(*testing.T, *bytes.Buffer) + }{ + { + name: "successful backup log download", + target: velerov1api.DownloadTargetKindBackupLog, + timeout: 5 * time.Second, + setupClient: func(t *testing.T, client kbclient.WithWatch) { + t.Helper() + // Simulate controller updating the DownloadRequest with URL + go func() { + time.Sleep(10 * time.Millisecond) + list := &velerov1api.DownloadRequestList{} + err := client.List(t.Context(), list) + assert.NoError(t, err) + + for _, dr := range list.Items { + dr.Status.DownloadURL = downloadServer.URL + "/log" + dr.Status.Phase = velerov1api.DownloadRequestPhaseProcessed + err := client.Update(t.Context(), &dr) + assert.NoError(t, err) + } + }() + }, + expectedError: false, + validateContent: func(t *testing.T, buf *bytes.Buffer) { + t.Helper() + // Logs should be decompressed + assert.Equal(t, testContent, buf.String()) + }, + }, + { + name: "successful backup contents download", + target: velerov1api.DownloadTargetKindBackupContents, + timeout: 5 * time.Second, + setupClient: func(t *testing.T, client kbclient.WithWatch) { + t.Helper() + // Simulate controller updating the DownloadRequest with URL + go func() { + time.Sleep(10 * time.Millisecond) + list := &velerov1api.DownloadRequestList{} + err := client.List(t.Context(), list) + assert.NoError(t, err) + + for _, dr := range list.Items { + dr.Status.DownloadURL = downloadServer.URL + "/contents" + dr.Status.Phase = velerov1api.DownloadRequestPhaseProcessed + err := client.Update(t.Context(), &dr) + assert.NoError(t, err) + } + }() + }, + expectedError: false, + validateContent: func(t *testing.T, buf *bytes.Buffer) { + t.Helper() + assert.Equal(t, testContent, buf.String()) + }, + }, + { + name: "timeout waiting for download URL", + target: velerov1api.DownloadTargetKindBackupLog, + timeout: 50 * time.Millisecond, + setupClient: func(t *testing.T, client kbclient.WithWatch) { + t.Helper() + }, + expectedError: true, + expectedErrMessage: "download request download url timeout", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fakeClient := fake.NewClientBuilder(). + WithScheme(util.VeleroScheme). + Build() + + if tc.setupClient != nil { + tc.setupClient(t, fakeClient) + } + + var buf bytes.Buffer + ctx := t.Context() + + err := Stream(ctx, fakeClient, "test-ns", "test-backup", tc.target, &buf, tc.timeout, false, "") + + if tc.expectedError { + require.Error(t, err) + if tc.expectedErrMessage != "" { + assert.Contains(t, err.Error(), tc.expectedErrMessage) + } + } else { + require.NoError(t, err) + if tc.validateContent != nil { + tc.validateContent(t, &buf) + } + } + }) + } +} + +func TestStreamWithBSLCACert(t *testing.T) { + // Create a test server that returns download content + testContent := "test backup content with BSL CA cert" + + // Create self-signed certificate for TLS testing + tlsCert, serverCACertPEM := createSelfSignedCertificate(t) + + // Create TLS test server for testing CA certificate functionality + tlsServer := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + if strings.Contains(r.URL.Path, "log") { + // For logs, return gzipped content + gzipWriter := gzip.NewWriter(w) + gzipWriter.Write([]byte(testContent)) + gzipWriter.Close() + } else { + w.Write([]byte(testContent)) + } + })) + tlsServer.TLS = &tls.Config{ + Certificates: []tls.Certificate{tlsCert}, + } + tlsServer.StartTLS() + defer tlsServer.Close() + + // Also create a regular HTTP server for non-TLS tests + httpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + if strings.Contains(r.URL.Path, "log") { + // For logs, return gzipped content + gzipWriter := gzip.NewWriter(w) + gzipWriter.Write([]byte(testContent)) + gzipWriter.Close() + } else { + w.Write([]byte(testContent)) + } + })) + defer httpServer.Close() + + testCases := []struct { + name string + target velerov1api.DownloadTargetKind + bslCACert string + timeout time.Duration + setupClient func(*testing.T, kbclient.WithWatch) + useTLS bool + expectedError bool + expectedErrMessage string + validateContent func(*testing.T, *bytes.Buffer) + }{ + { + name: "successful TLS backup log download with correct BSL CA cert", + target: velerov1api.DownloadTargetKindBackupLog, + bslCACert: string(serverCACertPEM), + timeout: 5 * time.Second, + useTLS: true, + setupClient: func(t *testing.T, client kbclient.WithWatch) { + t.Helper() + // Simulate controller updating the DownloadRequest with URL + go func() { + time.Sleep(10 * time.Millisecond) + list := &velerov1api.DownloadRequestList{} + err := client.List(t.Context(), list) + assert.NoError(t, err) + + for _, dr := range list.Items { + dr.Status.DownloadURL = tlsServer.URL + "/log" + dr.Status.Phase = velerov1api.DownloadRequestPhaseProcessed + err := client.Update(t.Context(), &dr) + assert.NoError(t, err) + } + }() + }, + expectedError: false, + validateContent: func(t *testing.T, buf *bytes.Buffer) { + t.Helper() + // Logs should be decompressed + assert.Equal(t, testContent, buf.String()) + }, + }, + { + name: "successful TLS backup contents download with correct BSL CA cert", + target: velerov1api.DownloadTargetKindBackupContents, + bslCACert: string(serverCACertPEM), + timeout: 5 * time.Second, + useTLS: true, + setupClient: func(t *testing.T, client kbclient.WithWatch) { + t.Helper() + // Simulate controller updating the DownloadRequest with URL + go func() { + time.Sleep(10 * time.Millisecond) + list := &velerov1api.DownloadRequestList{} + err := client.List(t.Context(), list) + assert.NoError(t, err) + + for _, dr := range list.Items { + dr.Status.DownloadURL = tlsServer.URL + "/contents" + dr.Status.Phase = velerov1api.DownloadRequestPhaseProcessed + err := client.Update(t.Context(), &dr) + assert.NoError(t, err) + } + }() + }, + expectedError: false, + validateContent: func(t *testing.T, buf *bytes.Buffer) { + t.Helper() + assert.Equal(t, testContent, buf.String()) + }, + }, + { + name: "failed TLS download with wrong BSL CA cert", + target: velerov1api.DownloadTargetKindBackupContents, + bslCACert: "-----BEGIN CERTIFICATE-----\nMIIBkTCB+wIJAKHHIgKwERfFMA0GCSqGSIb3DQEBCwUAMBkxFzAVBgNVBAMTDmRp\nZmZlcmVudC1jZXJ0MB4XDTE5MDQwMTAwMDAwMFoXDTI5MDQwMTAwMDAwMFowGTEX\nMBUGA1UEAxMOZGlmZmVyZW50LWNlcnQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ\nAoGBAOO4V+XrhVEGbTqnO2FM5eVFaM3KMKc3M9/C1aeg3vvY+Th3OhqJBxEYFxXL\nZoSqkwL/E6BjQb0NdSyJY9wdM4Ie3gElcZBKYVpHXYYAVhrepRCRVJEIHdBN8ybr\nFoBBDjd/ID1qy8Gdp3RihPFNvCNx0RWWqPAJtNXWJvCiNRCDAgMBAAEwDQYJKoZI\nhvcNAQELBQADgYEAGEwwGz7HAmH0J3pAJzQKPCb8HJG8hTjD6qkMon3Bp6gZ\n-----END CERTIFICATE-----\n", + timeout: 5 * time.Second, + useTLS: true, + setupClient: func(t *testing.T, client kbclient.WithWatch) { + t.Helper() + // Simulate controller updating the DownloadRequest with URL + go func() { + time.Sleep(10 * time.Millisecond) + list := &velerov1api.DownloadRequestList{} + err := client.List(t.Context(), list) + assert.NoError(t, err) + + for _, dr := range list.Items { + dr.Status.DownloadURL = tlsServer.URL + "/contents" + dr.Status.Phase = velerov1api.DownloadRequestPhaseProcessed + err := client.Update(t.Context(), &dr) + assert.NoError(t, err) + } + }() + }, + expectedError: true, + expectedErrMessage: "x509", + }, + { + name: "successful HTTP download with empty BSL CA cert", + target: velerov1api.DownloadTargetKindBackupContents, + bslCACert: "", + timeout: 5 * time.Second, + useTLS: false, + setupClient: func(t *testing.T, client kbclient.WithWatch) { + t.Helper() + // Simulate controller updating the DownloadRequest with URL + go func() { + time.Sleep(10 * time.Millisecond) + list := &velerov1api.DownloadRequestList{} + err := client.List(t.Context(), list) + assert.NoError(t, err) + + for _, dr := range list.Items { + dr.Status.DownloadURL = httpServer.URL + "/contents" + dr.Status.Phase = velerov1api.DownloadRequestPhaseProcessed + err := client.Update(t.Context(), &dr) + assert.NoError(t, err) + } + }() + }, + expectedError: false, + validateContent: func(t *testing.T, buf *bytes.Buffer) { + t.Helper() + assert.Equal(t, testContent, buf.String()) + }, + }, + { + name: "timeout waiting for download URL with BSL CA cert", + target: velerov1api.DownloadTargetKindBackupLog, + bslCACert: "test-ca-cert-content", + timeout: 50 * time.Millisecond, + setupClient: func(t *testing.T, client kbclient.WithWatch) { + t.Helper() + }, + expectedError: true, + expectedErrMessage: "download request download url timeout", + }, + { + name: "failed TLS download with malformed BSL CA cert", + target: velerov1api.DownloadTargetKindBackupLog, + bslCACert: "-----BEGIN CERTIFICATE-----\nINVALID CERT DATA\n-----END CERTIFICATE-----\n", + timeout: 5 * time.Second, + useTLS: true, + setupClient: func(t *testing.T, client kbclient.WithWatch) { + t.Helper() + // Simulate controller updating the DownloadRequest with URL + go func() { + time.Sleep(10 * time.Millisecond) + list := &velerov1api.DownloadRequestList{} + err := client.List(t.Context(), list) + assert.NoError(t, err) + + for _, dr := range list.Items { + dr.Status.DownloadURL = tlsServer.URL + "/log" + dr.Status.Phase = velerov1api.DownloadRequestPhaseProcessed + err := client.Update(t.Context(), &dr) + assert.NoError(t, err) + } + }() + }, + expectedError: true, + expectedErrMessage: "x509", // Should fail due to malformed cert not being added to pool + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fakeClient := fake.NewClientBuilder(). + WithScheme(util.VeleroScheme). + Build() + + if tc.setupClient != nil { + tc.setupClient(t, fakeClient) + } + + var buf bytes.Buffer + ctx := t.Context() + + err := StreamWithBSLCACert(ctx, fakeClient, "test-ns", "test-backup", tc.target, &buf, tc.timeout, false, "", tc.bslCACert) + + if tc.expectedError { + require.Error(t, err) + if tc.expectedErrMessage != "" { + assert.Contains(t, err.Error(), tc.expectedErrMessage) + } + } else { + require.NoError(t, err) + if tc.validateContent != nil { + tc.validateContent(t, &buf) + } + } + }) + } +} + +func TestDownload(t *testing.T) { + testContent := "test content for download" + compressedContent := new(bytes.Buffer) + gzipWriter := gzip.NewWriter(compressedContent) + gzipWriter.Write([]byte(testContent)) + gzipWriter.Close() + + testCases := []struct { + name string + serverHandler http.HandlerFunc + target velerov1api.DownloadTargetKind + insecureSkipTLSVerify bool + caCertFile string + bslCACert string + expectedContent string + expectedError bool + errorType error + }{ + { + name: "successful download with gzip for logs", + serverHandler: func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write(compressedContent.Bytes()) + }, + target: velerov1api.DownloadTargetKindBackupLog, + expectedContent: testContent, + expectedError: false, + }, + { + name: "successful download without gzip for backup contents", + serverHandler: func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte(testContent)) + }, + target: velerov1api.DownloadTargetKindBackupContents, + expectedContent: testContent, + expectedError: false, + }, + { + name: "404 not found error", + serverHandler: func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + }, + target: velerov1api.DownloadTargetKindBackupLog, + expectedError: true, + errorType: ErrNotFound, + }, + { + name: "500 internal server error", + serverHandler: func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte("internal server error")) + }, + target: velerov1api.DownloadTargetKindBackupLog, + expectedError: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + server := httptest.NewServer(tc.serverHandler) + defer server.Close() + + var buf bytes.Buffer + err := download( + t.Context(), + server.URL, + tc.target, + &buf, + tc.insecureSkipTLSVerify, + tc.caCertFile, + tc.bslCACert, + ) + + if tc.expectedError { + require.Error(t, err) + if tc.errorType != nil { + assert.Equal(t, tc.errorType, err) + } + } else { + require.NoError(t, err) + assert.Equal(t, tc.expectedContent, buf.String()) + } + }) + } +} + +// TestStreamWithBSLCACertEndToEnd tests the complete flow from BSL to download with CA cert +func TestStreamWithBSLCACertEndToEnd(t *testing.T) { + testContent := "end-to-end test content" + + // Create self-signed certificate for TLS testing + tlsCert, serverCACertPEM := createSelfSignedCertificate(t) + + // Create TLS test server + tlsServer := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + if strings.Contains(r.URL.Path, "log") { + // For logs, return gzipped content + gzipWriter := gzip.NewWriter(w) + gzipWriter.Write([]byte(testContent)) + gzipWriter.Close() + } else { + w.Write([]byte(testContent)) + } + })) + tlsServer.TLS = &tls.Config{ + Certificates: []tls.Certificate{tlsCert}, + } + tlsServer.StartTLS() + defer tlsServer.Close() + + // Create BSL with CA cert + bsl := builder.ForBackupStorageLocation("test-ns", "test-bsl"). + Provider("aws"). + Bucket("test-bucket"). + CACert(serverCACertPEM). + Result() + + // Create backup that references the BSL + backup := builder.ForBackup("test-ns", "test-backup"). + StorageLocation("test-bsl"). + Result() + + // Setup fake client with BSL and backup + fakeClient := fake.NewClientBuilder(). + WithScheme(util.VeleroScheme). + WithRuntimeObjects(bsl, backup). + Build() + + // Helper function to simulate controller updating the DownloadRequest + simulateControllerUpdate := func() { + go func() { + time.Sleep(10 * time.Millisecond) + list := &velerov1api.DownloadRequestList{} + err := fakeClient.List(t.Context(), list) + assert.NoError(t, err) + + for _, dr := range list.Items { + dr.Status.DownloadURL = tlsServer.URL + "/log" + dr.Status.Phase = velerov1api.DownloadRequestPhaseProcessed + err := fakeClient.Update(t.Context(), &dr) + assert.NoError(t, err) + } + }() + } + + // Test the complete flow + ctx := t.Context() + + // First, try to download WITHOUT the CA cert - this should fail + simulateControllerUpdate() + var bufFail bytes.Buffer + err := StreamWithBSLCACert(ctx, fakeClient, "test-ns", "test-backup", velerov1api.DownloadTargetKindBackupLog, &bufFail, 5*time.Second, false, "", "") + require.Error(t, err) + assert.Contains(t, err.Error(), "x509") // Should fail with certificate validation error + + // Now get CA cert from BSL through backup + cacertStr, err := cacert.GetCACertFromBackup(ctx, fakeClient, "test-ns", backup) + require.NoError(t, err) + require.Equal(t, string(serverCACertPEM), cacertStr) + + // Try again with the CA cert - this should succeed + simulateControllerUpdate() + var bufSuccess bytes.Buffer + err = StreamWithBSLCACert(ctx, fakeClient, "test-ns", "test-backup", velerov1api.DownloadTargetKindBackupLog, &bufSuccess, 5*time.Second, false, "", cacertStr) + require.NoError(t, err) + + // Verify content was downloaded and decompressed correctly + assert.Equal(t, testContent, bufSuccess.String()) +} + +// TestBackwardCompatibilityWithoutBSLCACert tests that old download requests work without BSL CA cert +func TestBackwardCompatibilityWithoutBSLCACert(t *testing.T) { + testContent := "backward compatibility test content" + + // Create HTTP (not HTTPS) server to simulate old behavior where TLS wasn't required + httpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + if strings.Contains(r.URL.Path, "log") { + // For logs, return gzipped content + gzipWriter := gzip.NewWriter(w) + gzipWriter.Write([]byte(testContent)) + gzipWriter.Close() + } else { + w.Write([]byte(testContent)) + } + })) + defer httpServer.Close() + + // Create BSL without CA cert (simulating old configuration) + bsl := builder.ForBackupStorageLocation("test-ns", "test-bsl"). + Provider("aws"). + Bucket("test-bucket"). + // No CACert() call - simulating pre-CA cert support + Result() + + // Create backup that references the BSL + backup := builder.ForBackup("test-ns", "test-backup"). + StorageLocation("test-bsl"). + Result() + + // Setup fake client with BSL and backup + fakeClient := fake.NewClientBuilder(). + WithScheme(util.VeleroScheme). + WithRuntimeObjects(bsl, backup). + Build() + + // Simulate controller updating the DownloadRequest with HTTP URL (old behavior) + go func() { + time.Sleep(10 * time.Millisecond) + list := &velerov1api.DownloadRequestList{} + err := fakeClient.List(t.Context(), list) + assert.NoError(t, err) + + for _, dr := range list.Items { + dr.Status.DownloadURL = httpServer.URL + "/log" + dr.Status.Phase = velerov1api.DownloadRequestPhaseProcessed + err := fakeClient.Update(t.Context(), &dr) + assert.NoError(t, err) + } + }() + + ctx := t.Context() + + // Test 1: Stream function (without BSL CA cert parameter) should work + var buf1 bytes.Buffer + err := Stream(ctx, fakeClient, "test-ns", "test-backup", velerov1api.DownloadTargetKindBackupLog, &buf1, 5*time.Second, false, "") + require.NoError(t, err) + assert.Equal(t, testContent, buf1.String()) + + // Test 2: StreamWithBSLCACert with empty BSL CA cert should also work + go func() { + time.Sleep(10 * time.Millisecond) + list := &velerov1api.DownloadRequestList{} + err := fakeClient.List(t.Context(), list) + assert.NoError(t, err) + + for _, dr := range list.Items { + dr.Status.DownloadURL = httpServer.URL + "/contents" + dr.Status.Phase = velerov1api.DownloadRequestPhaseProcessed + err := fakeClient.Update(t.Context(), &dr) + assert.NoError(t, err) + } + }() + + var buf2 bytes.Buffer + err = StreamWithBSLCACert(ctx, fakeClient, "test-ns", "test-backup", velerov1api.DownloadTargetKindBackupContents, &buf2, 5*time.Second, false, "", "") + require.NoError(t, err) + assert.Equal(t, testContent, buf2.String()) + + // Test 3: Getting CA cert from BSL should return empty string (not error) + cacert, err := cacert.GetCACertFromBackup(ctx, fakeClient, "test-ns", backup) + require.NoError(t, err) + assert.Empty(t, cacert) +} + +// TestMixedEnvironmentHTTPAndHTTPS tests environment with both HTTP and HTTPS endpoints +func TestMixedEnvironmentHTTPAndHTTPS(t *testing.T) { + testContentHTTP := "http content" + testContentHTTPS := "https content" + + // Create HTTP server + httpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte(testContentHTTP)) + })) + defer httpServer.Close() + + // Create HTTPS server with self-signed cert + tlsCert, serverCACertPEM := createSelfSignedCertificate(t) + httpsServer := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte(testContentHTTPS)) + })) + httpsServer.TLS = &tls.Config{ + Certificates: []tls.Certificate{tlsCert}, + } + httpsServer.StartTLS() + defer httpsServer.Close() + + // Create two BSLs - one for HTTP (old) and one for HTTPS (new) + bslHTTP := builder.ForBackupStorageLocation("test-ns", "bsl-http"). + Provider("aws"). + Bucket("http-bucket"). + // No CA cert for HTTP + Result() + + bslHTTPS := builder.ForBackupStorageLocation("test-ns", "bsl-https"). + Provider("aws"). + Bucket("https-bucket"). + CACert(serverCACertPEM). + Result() + + // Create backups for each BSL + backupHTTP := builder.ForBackup("test-ns", "backup-http"). + StorageLocation("bsl-http"). + Result() + + backupHTTPS := builder.ForBackup("test-ns", "backup-https"). + StorageLocation("bsl-https"). + Result() + + // Setup fake client + fakeClient := fake.NewClientBuilder(). + WithScheme(util.VeleroScheme). + WithRuntimeObjects(bslHTTP, bslHTTPS, backupHTTP, backupHTTPS). + Build() + + ctx := t.Context() + + // Test HTTP backup download (backward compatible) + go func() { + time.Sleep(10 * time.Millisecond) + list := &velerov1api.DownloadRequestList{} + err := fakeClient.List(t.Context(), list) + assert.NoError(t, err) + + for _, dr := range list.Items { + if strings.Contains(dr.Name, "backup-http") { + dr.Status.DownloadURL = httpServer.URL + dr.Status.Phase = velerov1api.DownloadRequestPhaseProcessed + err := fakeClient.Update(t.Context(), &dr) + assert.NoError(t, err) + } + } + }() + + var bufHTTP bytes.Buffer + err := Stream(ctx, fakeClient, "test-ns", "backup-http", velerov1api.DownloadTargetKindBackupContents, &bufHTTP, 5*time.Second, false, "") + require.NoError(t, err) + assert.Equal(t, testContentHTTP, bufHTTP.String()) + + // Test HTTPS backup download (requires CA cert) + go func() { + time.Sleep(10 * time.Millisecond) + list := &velerov1api.DownloadRequestList{} + err := fakeClient.List(t.Context(), list) + assert.NoError(t, err) + + for _, dr := range list.Items { + if strings.Contains(dr.Name, "backup-https") { + dr.Status.DownloadURL = httpsServer.URL + dr.Status.Phase = velerov1api.DownloadRequestPhaseProcessed + err := fakeClient.Update(t.Context(), &dr) + assert.NoError(t, err) + } + } + }() + + // First try without CA cert - should fail + var bufHTTPSFail bytes.Buffer + err = Stream(ctx, fakeClient, "test-ns", "backup-https", velerov1api.DownloadTargetKindBackupContents, &bufHTTPSFail, 5*time.Second, false, "") + require.Error(t, err) + assert.Contains(t, err.Error(), "x509") + + // Get CA cert from HTTPS BSL + cacertStr, err := cacert.GetCACertFromBackup(ctx, fakeClient, "test-ns", backupHTTPS) + require.NoError(t, err) + require.Equal(t, string(serverCACertPEM), cacertStr) + + // Try again with CA cert - should succeed + go func() { + time.Sleep(10 * time.Millisecond) + list := &velerov1api.DownloadRequestList{} + err := fakeClient.List(t.Context(), list) + assert.NoError(t, err) + + for _, dr := range list.Items { + if strings.Contains(dr.Name, "backup-https") { + dr.Status.DownloadURL = httpsServer.URL + dr.Status.Phase = velerov1api.DownloadRequestPhaseProcessed + err := fakeClient.Update(t.Context(), &dr) + assert.NoError(t, err) + } + } + }() + + var bufHTTPSSuccess bytes.Buffer + err = StreamWithBSLCACert(ctx, fakeClient, "test-ns", "backup-https", velerov1api.DownloadTargetKindBackupContents, &bufHTTPSSuccess, 5*time.Second, false, "", cacertStr) + require.NoError(t, err) + assert.Equal(t, testContentHTTPS, bufHTTPSSuccess.String()) +} + +// TestBSLUpgradeScenario tests the scenario where a BSL is upgraded to include CA cert +func TestBSLUpgradeScenario(t *testing.T) { + testContent := "bsl upgrade test content" + + // Create self-signed certificate for TLS testing + tlsCert, serverCACertPEM := createSelfSignedCertificate(t) + + // Create HTTPS server + httpsServer := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + if strings.Contains(r.URL.Path, "log") { + // For logs, return gzipped content + gzipWriter := gzip.NewWriter(w) + gzipWriter.Write([]byte(testContent)) + gzipWriter.Close() + } else { + w.Write([]byte(testContent)) + } + })) + httpsServer.TLS = &tls.Config{ + Certificates: []tls.Certificate{tlsCert}, + } + httpsServer.StartTLS() + defer httpsServer.Close() + + // Create BSL initially without CA cert (pre-upgrade state) + bsl := builder.ForBackupStorageLocation("test-ns", "test-bsl"). + Provider("aws"). + Bucket("test-bucket"). + // Initially no CA cert + Result() + + // Create an old backup that references this BSL + oldBackup := builder.ForBackup("test-ns", "old-backup"). + StorageLocation("test-bsl"). + Result() + + // Setup fake client + fakeClient := fake.NewClientBuilder(). + WithScheme(util.VeleroScheme). + WithRuntimeObjects(bsl, oldBackup). + Build() + + ctx := t.Context() + + // Test 1: Old backup with HTTPS URL should fail without CA cert + go func() { + time.Sleep(10 * time.Millisecond) + list := &velerov1api.DownloadRequestList{} + err := fakeClient.List(t.Context(), list) + assert.NoError(t, err) + + for _, dr := range list.Items { + dr.Status.DownloadURL = httpsServer.URL + "/log" + dr.Status.Phase = velerov1api.DownloadRequestPhaseProcessed + err := fakeClient.Update(t.Context(), &dr) + assert.NoError(t, err) + } + }() + + var bufFail bytes.Buffer + err := Stream(ctx, fakeClient, "test-ns", "old-backup", velerov1api.DownloadTargetKindBackupLog, &bufFail, 5*time.Second, false, "") + require.Error(t, err) + assert.Contains(t, err.Error(), "x509") + + // Simulate BSL upgrade - get current BSL and update it + currentBSL := &velerov1api.BackupStorageLocation{} + err = fakeClient.Get(ctx, kbclient.ObjectKey{Namespace: "test-ns", Name: "test-bsl"}, currentBSL) + require.NoError(t, err) + + // Update the BSL to include CA cert + // Ensure ObjectStorage is initialized + if currentBSL.Spec.ObjectStorage == nil { + currentBSL.Spec.ObjectStorage = &velerov1api.ObjectStorageLocation{} + } + currentBSL.Spec.ObjectStorage.CACert = serverCACertPEM // CA cert added after upgrade + + err = fakeClient.Update(ctx, currentBSL) + require.NoError(t, err) + + // Test 2: After BSL upgrade, old backup should work with new CA cert + cacertStr, err := cacert.GetCACertFromBackup(ctx, fakeClient, "test-ns", oldBackup) + require.NoError(t, err) + require.Equal(t, string(serverCACertPEM), cacertStr) + + go func() { + time.Sleep(10 * time.Millisecond) + list := &velerov1api.DownloadRequestList{} + err := fakeClient.List(t.Context(), list) + assert.NoError(t, err) + + for _, dr := range list.Items { + dr.Status.DownloadURL = httpsServer.URL + "/log" + dr.Status.Phase = velerov1api.DownloadRequestPhaseProcessed + err := fakeClient.Update(t.Context(), &dr) + assert.NoError(t, err) + } + }() + + var bufSuccess bytes.Buffer + err = StreamWithBSLCACert(ctx, fakeClient, "test-ns", "old-backup", velerov1api.DownloadTargetKindBackupLog, &bufSuccess, 5*time.Second, false, "", cacertStr) + require.NoError(t, err) + assert.Equal(t, testContent, bufSuccess.String()) + + // Test 3: New backup created after upgrade should also work + newBackup := builder.ForBackup("test-ns", "new-backup"). + StorageLocation("test-bsl"). + Result() + + err = fakeClient.Create(ctx, newBackup) + require.NoError(t, err) + + cacertStr2, err := cacert.GetCACertFromBackup(ctx, fakeClient, "test-ns", newBackup) + require.NoError(t, err) + require.Equal(t, string(serverCACertPEM), cacertStr2) +} + +// TestConcurrentDownloadsWithBSLCACert tests multiple concurrent downloads with CA cert +func TestConcurrentDownloadsWithBSLCACert(t *testing.T) { + testContent := "concurrent test content" + + // Create self-signed certificate for TLS testing + tlsCert, serverCACertPEM := createSelfSignedCertificate(t) + + // Create TLS test server + tlsServer := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Simulate some processing time + time.Sleep(10 * time.Millisecond) + w.WriteHeader(http.StatusOK) + w.Write([]byte(testContent)) + })) + tlsServer.TLS = &tls.Config{ + Certificates: []tls.Certificate{tlsCert}, + } + tlsServer.StartTLS() + defer tlsServer.Close() + + // Run multiple concurrent downloads + numConcurrent := 5 + errors := make(chan error, numConcurrent) + results := make(chan string, numConcurrent) + + for i := range numConcurrent { + go func(idx int) { + var buf bytes.Buffer + err := download( + t.Context(), + tlsServer.URL, + velerov1api.DownloadTargetKindBackupContents, + &buf, + false, + "", + string(serverCACertPEM), + ) + if err != nil { + errors <- err + return + } + results <- buf.String() + }(i) + } + + // Collect results + for i := range numConcurrent { + select { + case err := <-errors: + t.Fatalf("Concurrent download %d failed: %v", i, err) + case result := <-results: + assert.Equal(t, testContent, result) + case <-time.After(5 * time.Second): + t.Fatal("Timeout waiting for concurrent downloads") + } + } +} + +func TestDownloadWithBSLCACert(t *testing.T) { + testContent := "test content with BSL CA cert" + + // Create self-signed certificate for TLS testing + tlsCert, serverCACertPEM := createSelfSignedCertificate(t) + + // Create TLS test server with self-signed cert + tlsServer := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte(testContent)) + })) + tlsServer.TLS = &tls.Config{ + Certificates: []tls.Certificate{tlsCert}, + } + tlsServer.StartTLS() + defer tlsServer.Close() + + // Create HTTP test server for non-TLS tests + httpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte(testContent)) + })) + defer httpServer.Close() + + testCases := []struct { + name string + url string + target velerov1api.DownloadTargetKind + insecureSkipTLSVerify bool + bslCACert string + expectedContent string + expectedError bool + expectedErrContains string + }{ + { + name: "successful TLS download with correct BSL CA cert", + url: tlsServer.URL, + target: velerov1api.DownloadTargetKindBackupContents, + bslCACert: string(serverCACertPEM), + expectedContent: testContent, + expectedError: false, + }, + { + name: "successful TLS download with insecureSkipTLSVerify", + url: tlsServer.URL, + target: velerov1api.DownloadTargetKindBackupContents, + insecureSkipTLSVerify: true, + bslCACert: "", + expectedContent: testContent, + expectedError: false, + }, + { + name: "failed TLS download without CA cert or insecureSkipTLSVerify", + url: tlsServer.URL, + target: velerov1api.DownloadTargetKindBackupContents, + bslCACert: "", + expectedError: true, + expectedErrContains: "x509", + }, + { + name: "failed TLS download with wrong BSL CA cert", + url: tlsServer.URL, + target: velerov1api.DownloadTargetKindBackupContents, + bslCACert: "-----BEGIN CERTIFICATE-----\nMIIBkTCB+wIJAKHHIgKwERfFMA0GCSqGSIb3DQEBCwUAMBkxFzAVBgNVBAMTDmRp\nZmZlcmVudC1jZXJ0MB4XDTE5MDQwMTAwMDAwMFoXDTI5MDQwMTAwMDAwMFowGTEX\nMBUGA1UEAxMOZGlmZmVyZW50LWNlcnQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ\nAoGBAOO4V+XrhVEGbTqnO2FM5eVFaM3KMKc3M9/C1aeg3vvY+Th3OhqJBxEYFxXL\nZoSqkwL/E6BjQb0NdSyJY9wdM4Ie3gElcZBKYVpHXYYAVhrepRCRVJEIHdBN8ybr\nFoBBDjd/ID1qy8Gdp3RihPFNvCNx0RWWqPAJtNXWJvCiNRCDAgMBAAEwDQYJKoZI\nhvcNAQELBQADgYEAGEwwGz7HAmH0J3pAJzQKPCb8HJG8hTjD6qkMon3Bp6gZ\n-----END CERTIFICATE-----\n", + expectedError: true, + expectedErrContains: "x509", + }, + { + name: "successful HTTP download with empty BSL CA cert", + url: httpServer.URL, + target: velerov1api.DownloadTargetKindBackupContents, + bslCACert: "", + expectedContent: testContent, + expectedError: false, + }, + { + name: "successful TLS download with multiple CA certs in PEM block", + url: tlsServer.URL, + target: velerov1api.DownloadTargetKindBackupContents, + bslCACert: "-----BEGIN CERTIFICATE-----\nMIIBkTCB+wIJAKHHIgKwERfFMA0GCSqGSIb3DQEBCwUAMBkxFzAVBgNVBAMTDmRp\nZmZlcmVudC1jZXJ0MB4XDTE5MDQwMTAwMDAwMFoXDTI5MDQwMTAwMDAwMFowGTEX\nMBUGA1UEAxMOZGlmZmVyZW50LWNlcnQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ\nAoGBAOO4V+XrhVEGbTqnO2FM5eVFaM3KMKc3M9/C1aeg3vvY+Th3OhqJBxEYFxXL\nZoSqkwL/E6BjQb0NdSyJY9wdM4Ie3gElcZBKYVpHXYYAVhrepRCRVJEIHdBN8ybr\nFoBBDjd/ID1qy8Gdp3RihPFNvCNx0RWWqPAJtNXWJvCiNRCDAgMBAAEwDQYJKoZI\nhvcNAQELBQADgYEAGEwwGz7HAmH0J3pAJzQKPCb8HJG8hTjD6qkMon3Bp6gZ\n-----END CERTIFICATE-----\n" + string(serverCACertPEM), // First cert is wrong, but second is correct + expectedContent: testContent, + expectedError: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var buf bytes.Buffer + err := download( + t.Context(), + tc.url, + tc.target, + &buf, + tc.insecureSkipTLSVerify, + "", + tc.bslCACert, + ) + + if tc.expectedError { + require.Error(t, err) + if tc.expectedErrContains != "" { + assert.Contains(t, err.Error(), tc.expectedErrContains) + } + } else { + require.NoError(t, err) + assert.Equal(t, tc.expectedContent, buf.String()) + } + }) + } +} + +// TestCACertFallback tests the fallback behavior when caCertFile fails +func TestCACertFallback(t *testing.T) { + testContent := "test content for CA cert fallback" + + // Create self-signed certificate for TLS testing + tlsCert, serverCACertPEM := createSelfSignedCertificate(t) + + // Create TLS test server + tlsServer := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte(testContent)) + })) + tlsServer.TLS = &tls.Config{ + Certificates: []tls.Certificate{tlsCert}, + } + tlsServer.StartTLS() + defer tlsServer.Close() + + // Create a temporary file path that doesn't exist + nonExistentFile := "/tmp/non-existent-ca-cert-file.pem" + + testCases := []struct { + name string + caCertFile string + bslCACert string + expectedError bool + expectedErrContains string + expectedContent string + }{ + { + name: "successful download with BSL cert when caCertFile fails", + caCertFile: nonExistentFile, + bslCACert: string(serverCACertPEM), + expectedError: false, + expectedContent: testContent, + }, + { + name: "failed download when both caCertFile and BSL cert are invalid", + caCertFile: nonExistentFile, + bslCACert: "", + expectedError: true, + expectedErrContains: "couldn't open cacert", + }, + { + name: "BSL cert used when caCertFile is empty", + caCertFile: "", + bslCACert: string(serverCACertPEM), + expectedError: false, + expectedContent: testContent, + }, + { + name: "successful download with valid caCertFile (BSL cert ignored)", + caCertFile: "", // Will be set to a valid temp file in test + bslCACert: "invalid cert that should be ignored", + expectedError: false, + expectedContent: testContent, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + caCertFile := tc.caCertFile + + // For the last test case, create a valid temp file + if tc.name == "successful download with valid caCertFile (BSL cert ignored)" { + tmpFile, err := os.CreateTemp(t.TempDir(), "test-ca-cert-*.pem") + require.NoError(t, err) + defer os.Remove(tmpFile.Name()) + + _, err = tmpFile.Write(serverCACertPEM) + require.NoError(t, err) + tmpFile.Close() + + caCertFile = tmpFile.Name() + } + + var buf bytes.Buffer + err := download( + t.Context(), + tlsServer.URL, + velerov1api.DownloadTargetKindBackupContents, + &buf, + false, + caCertFile, + tc.bslCACert, + ) + + if tc.expectedError { + require.Error(t, err) + if tc.expectedErrContains != "" { + assert.Contains(t, err.Error(), tc.expectedErrContains) + } + } else { + require.NoError(t, err) + assert.Equal(t, tc.expectedContent, buf.String()) + } + }) + } +} diff --git a/pkg/cmd/util/output/backup_describer.go b/pkg/cmd/util/output/backup_describer.go index cf5b1b753..3b6c5ae19 100644 --- a/pkg/cmd/util/output/backup_describer.go +++ b/pkg/cmd/util/output/backup_describer.go @@ -36,6 +36,7 @@ import ( veleroapishared "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/cmd/util/cacert" "github.com/vmware-tanzu/velero/pkg/cmd/util/downloadrequest" "github.com/vmware-tanzu/velero/pkg/itemoperation" @@ -377,8 +378,16 @@ func describeBackupItemOperations(ctx context.Context, kbClient kbclient.Client, return } + // Get BSL cacert if available + bslCACert, err := cacert.GetCACertFromBackup(ctx, kbClient, backup.Namespace, backup) + if err != nil { + // Log the error but don't fail - we can still try to download without the BSL cacert + d.Printf("WARNING: Error getting cacert from BSL: %v\n", err) + bslCACert = "" + } + buf := new(bytes.Buffer) - if err := downloadrequest.Stream(ctx, kbClient, backup.Namespace, backup.Name, velerov1api.DownloadTargetKindBackupItemOperations, buf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath); err != nil { + if err := downloadrequest.StreamWithBSLCACert(ctx, kbClient, backup.Namespace, backup.Name, velerov1api.DownloadTargetKindBackupItemOperations, buf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath, bslCACert); err != nil { d.Printf("Backup Item Operations:\t\n", err) return } @@ -397,8 +406,16 @@ func describeBackupItemOperations(ctx context.Context, kbClient kbclient.Client, } func describeBackupResourceList(ctx context.Context, kbClient kbclient.Client, d *Describer, backup *velerov1api.Backup, insecureSkipTLSVerify bool, caCertPath string) { + // Get BSL cacert if available + bslCACert, err := cacert.GetCACertFromBackup(ctx, kbClient, backup.Namespace, backup) + if err != nil { + // Log the error but don't fail - we can still try to download without the BSL cacert + d.Printf("WARNING: Error getting cacert from BSL: %v\n", err) + bslCACert = "" + } + buf := new(bytes.Buffer) - if err := downloadrequest.Stream(ctx, kbClient, backup.Namespace, backup.Name, velerov1api.DownloadTargetKindBackupResourceList, buf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath); err != nil { + if err := downloadrequest.StreamWithBSLCACert(ctx, kbClient, backup.Namespace, backup.Name, velerov1api.DownloadTargetKindBackupResourceList, buf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath, bslCACert); err != nil { if err == downloadrequest.ErrNotFound { // the backup resource list could be missing if (other reasons may exist as well): // - the backup was taken prior to v1.1; or @@ -444,20 +461,28 @@ func describeBackupVolumes( ) { d.Println("Backup Volumes:") + // Get BSL cacert if available + bslCACert, err := cacert.GetCACertFromBackup(ctx, kbClient, backup.Namespace, backup) + if err != nil { + // Log the error but don't fail - we can still try to download without the BSL cacert + d.Printf("WARNING: Error getting cacert from BSL: %v\n", err) + bslCACert = "" + } + nativeSnapshots := []*volume.BackupVolumeInfo{} csiSnapshots := []*volume.BackupVolumeInfo{} legacyInfoSource := false buf := new(bytes.Buffer) - err := downloadrequest.Stream(ctx, kbClient, backup.Namespace, backup.Name, velerov1api.DownloadTargetKindBackupVolumeInfos, buf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath) + err = downloadrequest.StreamWithBSLCACert(ctx, kbClient, backup.Namespace, backup.Name, velerov1api.DownloadTargetKindBackupVolumeInfos, buf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath, bslCACert) if err == downloadrequest.ErrNotFound { - nativeSnapshots, err = retrieveNativeSnapshotLegacy(ctx, kbClient, backup, insecureSkipTLSVerify, caCertPath) + nativeSnapshots, err = retrieveNativeSnapshotLegacy(ctx, kbClient, backup, insecureSkipTLSVerify, caCertPath, bslCACert) if err != nil { d.Printf("\t\n", err) return } - csiSnapshots, err = retrieveCSISnapshotLegacy(ctx, kbClient, backup, insecureSkipTLSVerify, caCertPath) + csiSnapshots, err = retrieveCSISnapshotLegacy(ctx, kbClient, backup, insecureSkipTLSVerify, caCertPath, bslCACert) if err != nil { d.Printf("\t\n", err) return @@ -493,7 +518,7 @@ func describeBackupVolumes( describePodVolumeBackups(d, details, podVolumeBackupCRs) } -func retrieveNativeSnapshotLegacy(ctx context.Context, kbClient kbclient.Client, backup *velerov1api.Backup, insecureSkipTLSVerify bool, caCertPath string) ([]*volume.BackupVolumeInfo, error) { +func retrieveNativeSnapshotLegacy(ctx context.Context, kbClient kbclient.Client, backup *velerov1api.Backup, insecureSkipTLSVerify bool, caCertPath string, bslCACert string) ([]*volume.BackupVolumeInfo, error) { status := backup.Status nativeSnapshots := []*volume.BackupVolumeInfo{} @@ -502,7 +527,7 @@ func retrieveNativeSnapshotLegacy(ctx context.Context, kbClient kbclient.Client, } buf := new(bytes.Buffer) - if err := downloadrequest.Stream(ctx, kbClient, backup.Namespace, backup.Name, velerov1api.DownloadTargetKindBackupVolumeSnapshots, buf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath); err != nil { + if err := downloadrequest.StreamWithBSLCACert(ctx, kbClient, backup.Namespace, backup.Name, velerov1api.DownloadTargetKindBackupVolumeSnapshots, buf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath, bslCACert); err != nil { return nativeSnapshots, errors.Wrapf(err, "error to download native snapshot info") } @@ -531,7 +556,7 @@ func retrieveNativeSnapshotLegacy(ctx context.Context, kbClient kbclient.Client, return nativeSnapshots, nil } -func retrieveCSISnapshotLegacy(ctx context.Context, kbClient kbclient.Client, backup *velerov1api.Backup, insecureSkipTLSVerify bool, caCertPath string) ([]*volume.BackupVolumeInfo, error) { +func retrieveCSISnapshotLegacy(ctx context.Context, kbClient kbclient.Client, backup *velerov1api.Backup, insecureSkipTLSVerify bool, caCertPath string, bslCACert string) ([]*volume.BackupVolumeInfo, error) { status := backup.Status csiSnapshots := []*volume.BackupVolumeInfo{} @@ -540,7 +565,7 @@ func retrieveCSISnapshotLegacy(ctx context.Context, kbClient kbclient.Client, ba } vsBuf := new(bytes.Buffer) - err := downloadrequest.Stream(ctx, kbClient, backup.Namespace, backup.Name, velerov1api.DownloadTargetKindCSIBackupVolumeSnapshots, vsBuf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath) + err := downloadrequest.StreamWithBSLCACert(ctx, kbClient, backup.Namespace, backup.Name, velerov1api.DownloadTargetKindCSIBackupVolumeSnapshots, vsBuf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath, bslCACert) if err != nil { return csiSnapshots, errors.Wrapf(err, "error to download vs list") } @@ -551,7 +576,7 @@ func retrieveCSISnapshotLegacy(ctx context.Context, kbClient kbclient.Client, ba } vscBuf := new(bytes.Buffer) - err = downloadrequest.Stream(ctx, kbClient, backup.Namespace, backup.Name, velerov1api.DownloadTargetKindCSIBackupVolumeSnapshotContents, vscBuf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath) + err = downloadrequest.StreamWithBSLCACert(ctx, kbClient, backup.Namespace, backup.Name, velerov1api.DownloadTargetKindCSIBackupVolumeSnapshotContents, vscBuf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath, bslCACert) if err != nil { return csiSnapshots, errors.Wrapf(err, "error to download vsc list") } @@ -901,12 +926,20 @@ func DescribeBackupResults(ctx context.Context, kbClient kbclient.Client, d *Des return } + // Get BSL cacert if available + bslCACert, err := cacert.GetCACertFromBackup(ctx, kbClient, backup.Namespace, backup) + if err != nil { + // Log the error but don't fail - we can still try to download without the BSL cacert + d.Printf("WARNING: Error getting cacert from BSL: %v\n", err) + bslCACert = "" + } + var buf bytes.Buffer var resultMap map[string]results.Result // If err 'ErrNotFound' occurs, it means the backup bundle in the bucket has already been there before the backup-result file is introduced. // We only display the count of errors and warnings in this case. - err := downloadrequest.Stream(ctx, kbClient, backup.Namespace, backup.Name, velerov1api.DownloadTargetKindBackupResults, &buf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath) + err = downloadrequest.StreamWithBSLCACert(ctx, kbClient, backup.Namespace, backup.Name, velerov1api.DownloadTargetKindBackupResults, &buf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath, bslCACert) if err == downloadrequest.ErrNotFound { d.Printf("Errors:\t%d\n", backup.Status.Errors) d.Printf("Warnings:\t%d\n", backup.Status.Warnings) diff --git a/pkg/cmd/util/output/backup_structured_describer.go b/pkg/cmd/util/output/backup_structured_describer.go index f27526718..ad8d8381d 100644 --- a/pkg/cmd/util/output/backup_structured_describer.go +++ b/pkg/cmd/util/output/backup_structured_describer.go @@ -30,6 +30,7 @@ import ( "github.com/vmware-tanzu/velero/internal/volume" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/cmd/util/cacert" "github.com/vmware-tanzu/velero/pkg/cmd/util/downloadrequest" "github.com/vmware-tanzu/velero/pkg/util/results" ) @@ -272,11 +273,19 @@ func DescribeBackupStatusInSF(ctx context.Context, kbClient kbclient.Client, d * } func describeBackupResourceListInSF(ctx context.Context, kbClient kbclient.Client, backupStatusInfo map[string]any, backup *velerov1api.Backup, insecureSkipTLSVerify bool, caCertPath string) { + // Get BSL cacert if available + bslCACert, err := cacert.GetCACertFromBackup(ctx, kbClient, backup.Namespace, backup) + if err != nil { + // Log the error but don't fail - we can still try to download without the BSL cacert + backupStatusInfo["warningGettingBSLCACert"] = fmt.Sprintf("Warning: Error getting cacert from BSL: %v", err) + bslCACert = "" + } + // In consideration of decoding structured output conveniently, the two separate fields were created here(in func describeBackupResourceList, there is only one field describing either error message or resource list) // the field of 'errorGettingResourceList' gives specific error message when it fails to get resources list // the field of 'resourceList' lists the rearranged resources buf := new(bytes.Buffer) - if err := downloadrequest.Stream(ctx, kbClient, backup.Namespace, backup.Name, velerov1api.DownloadTargetKindBackupResourceList, buf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath); err != nil { + if err := downloadrequest.StreamWithBSLCACert(ctx, kbClient, backup.Namespace, backup.Name, velerov1api.DownloadTargetKindBackupResourceList, buf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath, bslCACert); err != nil { if err == downloadrequest.ErrNotFound { // the backup resource list could be missing if (other reasons may exist as well): // - the backup was taken prior to v1.1; or @@ -302,20 +311,28 @@ func describeBackupVolumesInSF(ctx context.Context, kbClient kbclient.Client, ba insecureSkipTLSVerify bool, caCertPath string, podVolumeBackupCRs []velerov1api.PodVolumeBackup, backupStatusInfo map[string]any) { backupVolumes := make(map[string]any) + // Get BSL cacert if available + bslCACert, err := cacert.GetCACertFromBackup(ctx, kbClient, backup.Namespace, backup) + if err != nil { + // Log the error but don't fail - we can still try to download without the BSL cacert + backupVolumes["warningGettingBSLCACert"] = fmt.Sprintf("Warning: Error getting cacert from BSL: %v", err) + bslCACert = "" + } + nativeSnapshots := []*volume.BackupVolumeInfo{} csiSnapshots := []*volume.BackupVolumeInfo{} legacyInfoSource := false buf := new(bytes.Buffer) - err := downloadrequest.Stream(ctx, kbClient, backup.Namespace, backup.Name, velerov1api.DownloadTargetKindBackupVolumeInfos, buf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath) + err = downloadrequest.StreamWithBSLCACert(ctx, kbClient, backup.Namespace, backup.Name, velerov1api.DownloadTargetKindBackupVolumeInfos, buf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath, bslCACert) if err == downloadrequest.ErrNotFound { - nativeSnapshots, err = retrieveNativeSnapshotLegacy(ctx, kbClient, backup, insecureSkipTLSVerify, caCertPath) + nativeSnapshots, err = retrieveNativeSnapshotLegacy(ctx, kbClient, backup, insecureSkipTLSVerify, caCertPath, bslCACert) if err != nil { backupVolumes["errorConcludeNativeSnapshot"] = fmt.Sprintf("error concluding native snapshot info: %v", err) return } - csiSnapshots, err = retrieveCSISnapshotLegacy(ctx, kbClient, backup, insecureSkipTLSVerify, caCertPath) + csiSnapshots, err = retrieveCSISnapshotLegacy(ctx, kbClient, backup, insecureSkipTLSVerify, caCertPath, bslCACert) if err != nil { backupVolumes["errorConcludeCSISnapshot"] = fmt.Sprintf("error concluding CSI snapshot info: %v", err) return @@ -538,6 +555,16 @@ func DescribeBackupResultsInSF(ctx context.Context, kbClient kbclient.Client, d return } + // Get BSL cacert if available + bslCACert, err := cacert.GetCACertFromBackup(ctx, kbClient, backup.Namespace, backup) + if err != nil { + // Log the error but don't fail - we can still try to download without the BSL cacert + warnings := make(map[string]any) + warnings["warningGettingBSLCACert"] = fmt.Sprintf("Warning: Error getting cacert from BSL: %v", err) + d.Describe("warningsGettingBSLCACert", warnings) + bslCACert = "" + } + var buf bytes.Buffer var resultMap map[string]results.Result @@ -549,7 +576,7 @@ func DescribeBackupResultsInSF(ctx context.Context, kbClient kbclient.Client, d // If 'ErrNotFound' occurs, it means the backup bundle in the bucket has already been there before the backup-result file is introduced. // We only display the count of errors and warnings in this case. - err := downloadrequest.Stream(ctx, kbClient, backup.Namespace, backup.Name, velerov1api.DownloadTargetKindBackupResults, &buf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath) + err = downloadrequest.StreamWithBSLCACert(ctx, kbClient, backup.Namespace, backup.Name, velerov1api.DownloadTargetKindBackupResults, &buf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath, bslCACert) if err == downloadrequest.ErrNotFound { errors["count"] = backup.Status.Errors warnings["count"] = backup.Status.Warnings diff --git a/pkg/cmd/util/output/restore_describer.go b/pkg/cmd/util/output/restore_describer.go index 76522d49f..4bd3bff1f 100644 --- a/pkg/cmd/util/output/restore_describer.go +++ b/pkg/cmd/util/output/restore_describer.go @@ -34,6 +34,7 @@ import ( "github.com/fatih/color" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/cmd/util/cacert" "github.com/vmware-tanzu/velero/pkg/cmd/util/downloadrequest" "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/util/boolptr" @@ -179,9 +180,17 @@ func DescribeRestore( describePodVolumeRestores(d, podVolumeRestores, details) } + // Get BSL cacert if available + bslCACert, err := cacert.GetCACertFromRestore(ctx, kbClient, restore.Namespace, restore) + if err != nil { + // Log the error but don't fail - we can still try to download without the BSL cacert + d.Printf("WARNING: Error getting cacert from BSL: %v\n", err) + bslCACert = "" + } + buf := new(bytes.Buffer) - if err := downloadrequest.Stream(ctx, kbClient, restore.Namespace, restore.Name, velerov1api.DownloadTargetKindRestoreVolumeInfo, - buf, downloadRequestTimeout, insecureSkipTLSVerify, caCertFile); err == nil { + if err := downloadrequest.StreamWithBSLCACert(ctx, kbClient, restore.Namespace, restore.Name, velerov1api.DownloadTargetKindRestoreVolumeInfo, + buf, downloadRequestTimeout, insecureSkipTLSVerify, caCertFile, bslCACert); err == nil { var restoreVolInfo []volume.RestoreVolumeInfo if err := json.NewDecoder(buf).Decode(&restoreVolInfo); err != nil { d.Printf("\t\n", err) @@ -250,8 +259,16 @@ func describeRestoreItemOperations(ctx context.Context, kbClient kbclient.Client return } + // Get BSL cacert if available + bslCACert, err := cacert.GetCACertFromRestore(ctx, kbClient, restore.Namespace, restore) + if err != nil { + // Log the error but don't fail - we can still try to download without the BSL cacert + d.Printf("WARNING: Error getting cacert from BSL: %v\n", err) + bslCACert = "" + } + buf := new(bytes.Buffer) - if err := downloadrequest.Stream(ctx, kbClient, restore.Namespace, restore.Name, velerov1api.DownloadTargetKindRestoreItemOperations, buf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath); err != nil { + if err := downloadrequest.StreamWithBSLCACert(ctx, kbClient, restore.Namespace, restore.Name, velerov1api.DownloadTargetKindRestoreItemOperations, buf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath, bslCACert); err != nil { d.Printf("Restore Item Operations:\t\n", err) return } @@ -274,10 +291,18 @@ func describeRestoreResults(ctx context.Context, kbClient kbclient.Client, d *De return } + // Get BSL cacert if available + bslCACert, err := cacert.GetCACertFromRestore(ctx, kbClient, restore.Namespace, restore) + if err != nil { + // Log the error but don't fail - we can still try to download without the BSL cacert + d.Printf("WARNING: Error getting cacert from BSL: %v\n", err) + bslCACert = "" + } + var buf bytes.Buffer var resultMap map[string]results.Result - if err := downloadrequest.Stream(ctx, kbClient, restore.Namespace, restore.Name, velerov1api.DownloadTargetKindRestoreResults, &buf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath); err != nil { + if err := downloadrequest.StreamWithBSLCACert(ctx, kbClient, restore.Namespace, restore.Name, velerov1api.DownloadTargetKindRestoreResults, &buf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath, bslCACert); err != nil { d.Printf("Warnings:\t\n\nErrors:\t\n", err, err) return } @@ -463,8 +488,16 @@ func groupRestoresByPhase(restores []velerov1api.PodVolumeRestore) map[string][] } func describeRestoreResourceList(ctx context.Context, kbClient kbclient.Client, d *Describer, restore *velerov1api.Restore, insecureSkipTLSVerify bool, caCertPath string) { + // Get BSL cacert if available + bslCACert, err := cacert.GetCACertFromRestore(ctx, kbClient, restore.Namespace, restore) + if err != nil { + // Log the error but don't fail - we can still try to download without the BSL cacert + d.Printf("WARNING: Error getting cacert from BSL: %v\n", err) + bslCACert = "" + } + buf := new(bytes.Buffer) - if err := downloadrequest.Stream(ctx, kbClient, restore.Namespace, restore.Name, velerov1api.DownloadTargetKindRestoreResourceList, buf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath); err != nil { + if err := downloadrequest.StreamWithBSLCACert(ctx, kbClient, restore.Namespace, restore.Name, velerov1api.DownloadTargetKindRestoreResourceList, buf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath, bslCACert); err != nil { if err == downloadrequest.ErrNotFound { d.Println("Resource List:\t") } else { diff --git a/site/content/docs/main/api-types/backupstoragelocation.md b/site/content/docs/main/api-types/backupstoragelocation.md index b6c58ece7..974d120b3 100644 --- a/site/content/docs/main/api-types/backupstoragelocation.md +++ b/site/content/docs/main/api-types/backupstoragelocation.md @@ -30,6 +30,27 @@ spec: profile: "default" ``` +### Example with self-signed certificate + +When using object storage with self-signed certificates, you can specify the CA certificate: + +```yaml +apiVersion: velero.io/v1 +kind: BackupStorageLocation +metadata: + name: default + namespace: velero +spec: + provider: aws + objectStorage: + bucket: velero-backups + # Base64 encoded CA certificate + caCert: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUR1VENDQXFHZ0F3SUJBZ0lVTWRiWkNaYnBhcE9lYThDR0NMQnhhY3dVa213d0RRWUpLb1pJaHZjTkFRRUwKQlFBd2JERUxNQWtHQTFVRUJoTUNWVk14RXpBUkJnTlZCQWdNQ2tOaGJHbG1iM0p1YVdFeEZqQVVCZ05WQkFjTQpEVk5oYmlCR2NtRnVZMmx6WTI4eEdEQVdCZ05WQkFvTUQwVjRZVzF3YkdVZ1EyOXRjR0Z1ZVRFV01CUUdBMVVFCkF3d05aWGhoYlhCc1pTNXNiMk5oYkRBZUZ3MHlNekEzTVRBeE9UVXlNVGhhRncweU5EQTNNRGt4T1RVeU1UaGEKTUd3eEN6QUpCZ05WQkFZVEFsVlRNUk13RVFZRFZRUUNEQXBEWEJ4cG1iM0p1YVdFeEZqQVVCZ05WQkFjTURWTmgKYmlCR2NtRnVZMmx6WTI4eEdEQVdCZ05WQkFvTUQwVjRZVzF3YkdVZ1EyOXRjR0Z1ZVRFV01CUUdBMVVFQXd3TgpaWGhoYlhCc1pTNXNiMk5oYkRDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBS1dqCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + config: + region: us-east-1 + s3Url: https://minio.example.com +``` + ### Parameter Reference The configurable parameters are as follows: diff --git a/site/content/docs/main/self-signed-certificates.md b/site/content/docs/main/self-signed-certificates.md index b85924c73..ac088b694 100644 --- a/site/content/docs/main/self-signed-certificates.md +++ b/site/content/docs/main/self-signed-certificates.md @@ -25,19 +25,39 @@ that storage provider when backing up and restoring. ## Trusting a self-signed certificate with the Velero client -To use the describe, download, or logs commands to access a backup or restore contained -in storage secured by a self-signed certificate as in the above example, you must use -the `--cacert` flag to provide a path to the certificate to be trusted. +When using Velero client commands like describe, download, or logs to access backups or restores +in storage secured by a self-signed certificate, the CA certificate can be configured in two ways: -```bash -velero backup describe my-backup --cacert -``` +1. **Using the `--cacert` flag** (legacy method): + + ```bash + velero backup describe my-backup --cacert + ``` + +2. **Configuring the CA certificate in the BackupStorageLocation**: + + ```yaml + apiVersion: velero.io/v1 + kind: BackupStorageLocation + metadata: + name: default + namespace: velero + spec: + provider: aws + objectStorage: + bucket: velero-backups + caCert: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCi4uLiAoYmFzZTY0IGVuY29kZWQgY2VydGlmaWNhdGUgY29udGVudCkgLi4uCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + config: + region: us-east-1 + ``` + +When the CA certificate is configured in the BackupStorageLocation, Velero client commands will automatically use it without requiring the `--cacert` flag. ## Error with client certificate with custom S3 server In case you are using a custom S3-compatible server, you may encounter that the backup fails with an error similar to one below. -``` +```text rpc error: code = Unknown desc = RequestError: send request failed caused by: Get https://minio.com:3000/k8s-backup-bucket?delimiter=%2F&list-type=2&prefix=: remote error: tls: alert(116) ``` @@ -47,7 +67,6 @@ Velero as a client does not include its certificate while performing SSL handsha From [TLS 1.3 spec](https://tools.ietf.org/html/rfc8446), verifying client certificate is optional on the server. You will need to change this setting on the server to make it work. - ## Skipping TLS verification **Note:** The `--insecure-skip-tls-verify` flag is insecure and susceptible to man-in-the-middle attacks and meant to help your testing and developing scenarios in an on-premises environment. Using this flag in production is not recommended.