re-instantiate backup store just before persisting artifacts (#2550)

Signed-off-by: Steve Kriss <krisss@vmware.com>

changelogs/unreleased/2550-skriss

@@ -0,0 +1 @@
+backup/restore: reinstantiate backup store just before uploading artifacts to ensure credentials are up-to-date

pkg/controller/backup_controller.go

@@ -533,7 +533,7 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error {
 		return err
 	}
 
-	backupLog.Info("Setting up backup store")
+	backupLog.Info("Setting up backup store to check for backup existence")
 	backupStore, err := c.newBackupStore(backup.StorageLocation, pluginManager, backupLog)
 	if err != nil {
 		return err
@@ -612,6 +612,14 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error {
 		backup.Status.Phase = velerov1api.BackupPhaseCompleted
 	}
 
+	// re-instantiate the backup store because credentials could have changed since the original
+	// instantiation, if this was a long-running backup
+	backupLog.Info("Setting up backup store to persist the backup")
+	backupStore, err = c.newBackupStore(backup.StorageLocation, pluginManager, backupLog)
+	if err != nil {
+		return err
+	}
+
 	if errs := persistBackup(backup, backupFile, logFile, backupStore, c.logger, volumeSnapshots, volumeSnapshotContents); len(errs) > 0 {
 		fatalErrs = append(fatalErrs, errs...)
 	}

pkg/controller/restore_controller.go

@@ -274,6 +274,7 @@ func (c *restoreController) processRestore(restore *api.Restore) error {
 
 type backupInfo struct {
 	backup      *api.Backup
+	location    *api.BackupStorageLocation
 	backupStore persistence.BackupStore
 }
 
@@ -407,6 +408,7 @@ func (c *restoreController) fetchBackupInfo(backupName string, pluginManager cli
 
 	return backupInfo{
 		backup:      backup,
+		location:    location,
 		backupStore: backupStore,
 	}, nil
 }
@@ -467,6 +469,13 @@ func (c *restoreController) runValidatedRestore(restore *api.Restore, info backu
 	restoreWarnings, restoreErrors := c.restorer.Restore(restoreReq, actions, c.snapshotLocationLister, pluginManager)
 	restoreLog.Info("restore completed")
 
+	// re-instantiate the backup store because credentials could have changed since the original
+	// instantiation, if this was a long-running restore
+	info.backupStore, err = c.newBackupStore(info.location, pluginManager, c.logger)
+	if err != nil {
+		return errors.Wrap(err, "error setting up backup store to persist log and results files")
+	}
+
 	if logReader, err := restoreLog.done(c.logger); err != nil {
 		restoreErrors.Velero = append(restoreErrors.Velero, fmt.Sprintf("error getting restore log reader: %v", err))
 	} else {