From fca4d405b16814d7c831ad24f99c7274280504b9 Mon Sep 17 00:00:00 2001 From: Lyndon-Li Date: Tue, 7 Apr 2026 16:51:13 +0800 Subject: [PATCH] remove restic for uploader Signed-off-by: Lyndon-Li --- changelogs/unreleased/9677-Lyndon-Li‎‎ | 1 + pkg/cmd/server/server_test.go | 4 +- .../backup_deletion_controller_test.go | 6 +- .../pod_volume_restore_controller_legacy.go | 2 +- pkg/podvolume/backupper.go | 2 +- pkg/podvolume/backupper_test.go | 2 +- pkg/podvolume/restorer_test.go | 18 - pkg/podvolume/util.go | 35 +- pkg/uploader/provider/kopia_test.go | 4 + pkg/uploader/provider/provider.go | 2 +- pkg/uploader/provider/provider_test.go | 2 +- pkg/uploader/provider/restic.go | 269 ---------- pkg/uploader/provider/restic_test.go | 464 ------------------ pkg/uploader/types.go | 1 - 14 files changed, 26 insertions(+), 786 deletions(-) create mode 100644 changelogs/unreleased/9677-Lyndon-Li‎‎ delete mode 100644 pkg/uploader/provider/restic.go delete mode 100644 pkg/uploader/provider/restic_test.go diff --git a/changelogs/unreleased/9677-Lyndon-Li‎‎ b/changelogs/unreleased/9677-Lyndon-Li‎‎ new file mode 100644 index 000000000..f722008e9 --- /dev/null +++ b/changelogs/unreleased/9677-Lyndon-Li‎‎ @@ -0,0 +1 @@ +Fix issue #9469, remove restic for uploader \ No newline at end of file diff --git a/pkg/cmd/server/server_test.go b/pkg/cmd/server/server_test.go index 1ea9d0022..c602f7c9e 100644 --- a/pkg/cmd/server/server_test.go +++ b/pkg/cmd/server/server_test.go @@ -204,9 +204,9 @@ func Test_newServer(t *testing.T) { }, logger) require.Error(t, err) - // invalid clientQPS Restic uploader + // invalid clientQPS Kopia uploader _, err = newServer(factory, &config.Config{ - UploaderType: uploader.ResticType, + UploaderType: uploader.KopiaType, ClientQPS: -1, }, logger) require.Error(t, err) diff --git a/pkg/controller/backup_deletion_controller_test.go b/pkg/controller/backup_deletion_controller_test.go index ab3687438..58d9b0420 100644 --- a/pkg/controller/backup_deletion_controller_test.go +++ b/pkg/controller/backup_deletion_controller_test.go @@ -821,12 +821,12 @@ func TestGetSnapshotsInBackup(t *testing.T) { { VolumeNamespace: "ns-1", SnapshotID: "snap-3", - RepositoryType: "restic", + RepositoryType: "kopia", }, { VolumeNamespace: "ns-1", SnapshotID: "snap-4", - RepositoryType: "restic", + RepositoryType: "kopia", }, }, }, @@ -876,7 +876,7 @@ func TestGetSnapshotsInBackup(t *testing.T) { { VolumeNamespace: "ns-1", SnapshotID: "snap-3", - RepositoryType: "restic", + RepositoryType: "kopia", }, }, }, diff --git a/pkg/controller/pod_volume_restore_controller_legacy.go b/pkg/controller/pod_volume_restore_controller_legacy.go index 731b70db9..9ddececf5 100644 --- a/pkg/controller/pod_volume_restore_controller_legacy.go +++ b/pkg/controller/pod_volume_restore_controller_legacy.go @@ -360,5 +360,5 @@ func (c *PodVolumeRestoreReconcilerLegacy) closeDataPath(ctx context.Context, pv } func IsLegacyPVR(pvr *velerov1api.PodVolumeRestore) bool { - return pvr.Spec.UploaderType == uploader.ResticType + return pvr.Spec.UploaderType == "restic" } diff --git a/pkg/podvolume/backupper.go b/pkg/podvolume/backupper.go index 1747f1b33..1dc88a9e5 100644 --- a/pkg/podvolume/backupper.go +++ b/pkg/podvolume/backupper.go @@ -272,7 +272,7 @@ func (b *backupper) BackupPodVolumes(backup *velerov1api.Backup, pod *corev1api. return nil, pvcSummary, []error{err} } - repositoryType := funcGetRepositoryType(b.uploaderType) + repositoryType := funcGetRepositoryType() if repositoryType == "" { err := errors.Errorf("empty repository type, uploader %s", b.uploaderType) skipAllPodVolumes(pod, volumesToBackup, err, pvcSummary, log) diff --git a/pkg/podvolume/backupper_test.go b/pkg/podvolume/backupper_test.go index 846f65796..f7686978a 100644 --- a/pkg/podvolume/backupper_test.go +++ b/pkg/podvolume/backupper_test.go @@ -580,7 +580,7 @@ func TestBackupPodVolumes(t *testing.T) { require.NoError(t, err) if test.mockGetRepositoryType { - funcGetRepositoryType = func(string) string { return "" } + funcGetRepositoryType = func() string { return "" } } else { funcGetRepositoryType = getRepositoryType } diff --git a/pkg/podvolume/restorer_test.go b/pkg/podvolume/restorer_test.go index 36a1fc034..e10146578 100644 --- a/pkg/podvolume/restorer_test.go +++ b/pkg/podvolume/restorer_test.go @@ -204,24 +204,6 @@ func TestRestorePodVolumes(t *testing.T) { }, }, }, - { - name: "get repository type fail", - pvbs: []*velerov1api.PodVolumeBackup{ - createPVBObj(true, true, 1, "restic"), - createPVBObj(true, true, 2, "kopia"), - }, - kubeClientObj: []runtime.Object{ - createNodeAgentDaemonset(), - }, - restoredPod: createPodObj(false, false, false, 2), - sourceNamespace: "fake-ns", - errs: []expectError{ - { - err: "multiple repository type in one backup", - prefixOnly: true, - }, - }, - }, { name: "ensure repo fail", pvbs: []*velerov1api.PodVolumeBackup{ diff --git a/pkg/podvolume/util.go b/pkg/podvolume/util.go index 1864e9615..9bf6f81ca 100644 --- a/pkg/podvolume/util.go +++ b/pkg/podvolume/util.go @@ -62,12 +62,12 @@ func GetVolumeBackupsForPod(podVolumeBackups []*velerov1api.PodVolumeBackup, pod // GetPvbRepositoryType returns the repositoryType according to the PVB information func GetPvbRepositoryType(pvb *velerov1api.PodVolumeBackup) string { - return getRepositoryType(pvb.Spec.UploaderType) + return getRepositoryType() } // GetPvrRepositoryType returns the repositoryType according to the PVR information func GetPvrRepositoryType(pvr *velerov1api.PodVolumeRestore) string { - return getRepositoryType(pvr.Spec.UploaderType) + return getRepositoryType() } // getVolumeBackupInfoForPod returns a map, of volume name -> VolumeBackupInfo, @@ -97,7 +97,7 @@ func getVolumeBackupInfoForPod(podVolumeBackups []*velerov1api.PodVolumeBackup, snapshotID: pvb.Status.SnapshotID, snapshotSize: pvb.Status.Progress.TotalBytes, uploaderType: getUploaderTypeOrDefault(pvb.Spec.UploaderType), - repositoryType: getRepositoryType(pvb.Spec.UploaderType), + repositoryType: getRepositoryType(), } } @@ -111,7 +111,7 @@ func getVolumeBackupInfoForPod(podVolumeBackups []*velerov1api.PodVolumeBackup, } for k, v := range fromAnnntation { - volumes[k] = volumeBackupInfo{v, 0, uploader.ResticType, velerov1api.BackupRepositoryTypeRestic} + volumes[k] = volumeBackupInfo{v, 0, uploader.KopiaType, velerov1api.BackupRepositoryTypeKopia} } return volumes @@ -135,7 +135,7 @@ func GetSnapshotIdentifier(podVolumeBackups *velerov1api.PodVolumeBackupList) ma VolumeNamespace: item.Spec.Pod.Namespace, BackupStorageLocation: item.Spec.BackupStorageLocation, SnapshotID: item.Status.SnapshotID, - RepositoryType: getRepositoryType(item.Spec.UploaderType), + RepositoryType: getRepositoryType(), UploaderType: item.Spec.UploaderType, Source: item.Status.Path, RepoIdentifier: item.Spec.RepoIdentifier, @@ -164,27 +164,14 @@ func getUploaderTypeOrDefault(uploaderType string) string { if uploaderType != "" { return uploaderType } - return uploader.ResticType + return uploader.KopiaType } -// getRepositoryType returns the hardcode repositoryType for different backup methods - Restic or Kopia,uploaderType -// indicates the method. -// For Restic backup method, it is always hardcode to BackupRepositoryTypeRestic, never changed. -// For Kopia backup method, this means we hardcode repositoryType as BackupRepositoryTypeKopia for Unified Repo, -// at present (Kopia backup method is using Unified Repo). However, it doesn't mean we could deduce repositoryType -// from uploaderType for Unified Repo. -// TODO: post v1.10, refactor this function for Kopia backup method. In future, when we have multiple implementations of -// Unified Repo (besides Kopia), we will add the repositoryType to BSL, because by then, we are not able to hardcode -// the repositoryType to BackupRepositoryTypeKopia for Unified Repo. -func getRepositoryType(uploaderType string) string { - switch uploaderType { - case "", uploader.ResticType: - return velerov1api.BackupRepositoryTypeRestic - case uploader.KopiaType: - return velerov1api.BackupRepositoryTypeKopia - default: - return "" - } +// getRepositoryType returns the hardcode repositoryType +// TODO: In future, when we have multiple implementations of Unified Repo (besides Kopia), we will add the repositoryType to BSL, +// because by then, we are not able to hardcode the repositoryType to BackupRepositoryTypeKopia for Unified Repo. +func getRepositoryType() string { + return velerov1api.BackupRepositoryTypeKopia } func isPVBMatchPod(pvb *velerov1api.PodVolumeBackup, podName string, namespace string) bool { diff --git a/pkg/uploader/provider/kopia_test.go b/pkg/uploader/provider/kopia_test.go index 74eaa67f7..734bdb176 100644 --- a/pkg/uploader/provider/kopia_test.go +++ b/pkg/uploader/provider/kopia_test.go @@ -294,6 +294,10 @@ func TestGetPassword(t *testing.T) { } } +type MockCredentialGetter struct { + mock.Mock +} + func (m *MockCredentialGetter) GetCredentials() (string, error) { args := m.Called() return args.String(0), args.Error(1) diff --git a/pkg/uploader/provider/provider.go b/pkg/uploader/provider/provider.go index fe1dd3091..95a34b1a0 100644 --- a/pkg/uploader/provider/provider.go +++ b/pkg/uploader/provider/provider.go @@ -87,6 +87,6 @@ func NewUploaderProvider( if uploaderType == uploader.KopiaType { return NewKopiaUploaderProvider(requesterType, ctx, credGetter, backupRepo, log) } else { - return NewResticUploaderProvider(repoIdentifier, bsl, credGetter, repoKeySelector, log) + return nil, errors.Errorf("unsupported uploader type %v", uploaderType) } } diff --git a/pkg/uploader/provider/provider_test.go b/pkg/uploader/provider/provider_test.go index 199091e32..8f447725b 100644 --- a/pkg/uploader/provider/provider_test.go +++ b/pkg/uploader/provider/provider_test.go @@ -75,7 +75,7 @@ func TestNewUploaderProvider(t *testing.T) { UploaderType: "restic", RequestorType: "requester", needFromFile: true, - ExpectedError: "", + ExpectedError: "unsupported uploader type restic", }, } diff --git a/pkg/uploader/provider/restic.go b/pkg/uploader/provider/restic.go deleted file mode 100644 index 93b907be9..000000000 --- a/pkg/uploader/provider/restic.go +++ /dev/null @@ -1,269 +0,0 @@ -/* -Copyright The Velero Contributors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package provider - -import ( - "context" - "fmt" - "os" - "strings" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - corev1api "k8s.io/api/core/v1" - - "github.com/vmware-tanzu/velero/internal/credentials" - velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" - "github.com/vmware-tanzu/velero/pkg/restic" - "github.com/vmware-tanzu/velero/pkg/uploader" - uploaderutil "github.com/vmware-tanzu/velero/pkg/uploader/util" - "github.com/vmware-tanzu/velero/pkg/util/filesystem" -) - -// resticBackupCMDFunc and resticRestoreCMDFunc are mainly used to make testing more convenient -var resticBackupCMDFunc = restic.BackupCommand -var resticBackupFunc = restic.RunBackup -var resticGetSnapshotFunc = restic.GetSnapshotCommand -var resticGetSnapshotIDFunc = restic.GetSnapshotID -var resticRestoreCMDFunc = restic.RestoreCommand -var resticTempCACertFileFunc = restic.TempCACertFile -var resticCmdEnvFunc = restic.CmdEnv - -type resticProvider struct { - repoIdentifier string - credentialsFile string - caCertFile string - cmdEnv []string - extraFlags []string - bsl *velerov1api.BackupStorageLocation - log logrus.FieldLogger -} - -func NewResticUploaderProvider( - repoIdentifier string, - bsl *velerov1api.BackupStorageLocation, - credGetter *credentials.CredentialGetter, - repoKeySelector *corev1api.SecretKeySelector, - log logrus.FieldLogger, -) (Provider, error) { - provider := resticProvider{ - repoIdentifier: repoIdentifier, - bsl: bsl, - log: log, - } - - var err error - provider.credentialsFile, err = credGetter.FromFile.Path(repoKeySelector) - if err != nil { - return nil, errors.Wrap(err, "error creating temp restic credentials file") - } - - // if there's a caCert on the ObjectStorage, write it to disk so that it can be passed to restic - if bsl.Spec.ObjectStorage != nil { - var caCertData []byte - - // Try CACertRef first (new method), then fall back to CACert (deprecated) - if bsl.Spec.ObjectStorage.CACertRef != nil { - caCertString, err := credGetter.FromSecret.Get(bsl.Spec.ObjectStorage.CACertRef) - if err != nil { - return nil, errors.Wrap(err, "error getting CA certificate from secret") - } - caCertData = []byte(caCertString) - } else if bsl.Spec.ObjectStorage.CACert != nil { - caCertData = bsl.Spec.ObjectStorage.CACert - } - - if caCertData != nil { - provider.caCertFile, err = resticTempCACertFileFunc(caCertData, bsl.Name, filesystem.NewFileSystem()) - if err != nil { - return nil, errors.Wrap(err, "error create temp cert file") - } - } - } - - provider.cmdEnv, err = resticCmdEnvFunc(bsl, credGetter.FromFile) - if err != nil { - return nil, errors.Wrap(err, "error generating repository cmnd env") - } - - // #4820: restrieve insecureSkipTLSVerify from BSL configuration for - // AWS plugin. If nothing is return, that means insecureSkipTLSVerify - // is not enable for Restic command. - skipTLSRet := restic.GetInsecureSkipTLSVerifyFromBSL(bsl, log) - if len(skipTLSRet) > 0 { - provider.extraFlags = append(provider.extraFlags, skipTLSRet) - } - - return &provider, nil -} - -func (rp *resticProvider) Close(ctx context.Context) error { - _, err := os.Stat(rp.credentialsFile) - if err == nil { - return os.Remove(rp.credentialsFile) - } else if !os.IsNotExist(err) { - return errors.Errorf("failed to get file %s info with error %v", rp.credentialsFile, err) - } - - _, err = os.Stat(rp.caCertFile) - if err == nil { - return os.Remove(rp.caCertFile) - } else if !os.IsNotExist(err) { - return errors.Errorf("failed to get file %s info with error %v", rp.caCertFile, err) - } - return nil -} - -// RunBackup runs a `backup` command and watches the output to provide -// progress updates to the caller and return snapshotID, isEmptySnapshot, error -func (rp *resticProvider) RunBackup( - ctx context.Context, - path string, - realSource string, - tags map[string]string, - forceFull bool, - parentSnapshot string, - volMode uploader.PersistentVolumeMode, - uploaderCfg map[string]string, - updater uploader.ProgressUpdater) (string, bool, int64, int64, error) { - if updater == nil { - return "", false, 0, 0, errors.New("Need to initial backup progress updater first") - } - - if path == "" { - return "", false, 0, 0, errors.New("path is empty") - } - - if realSource != "" { - return "", false, 0, 0, errors.New("real source is not empty, this is not supported by restic uploader") - } - - if volMode == uploader.PersistentVolumeBlock { - return "", false, 0, 0, errors.New("unable to support block mode") - } - - log := rp.log.WithFields(logrus.Fields{ - "path": path, - "parentSnapshot": parentSnapshot, - }) - - if len(uploaderCfg) > 0 { - parallelFilesUpload, err := uploaderutil.GetParallelFilesUpload(uploaderCfg) - if err != nil { - return "", false, 0, 0, errors.Wrap(err, "failed to get uploader config") - } - if parallelFilesUpload > 0 { - log.Warnf("ParallelFilesUpload is set to %d, but restic does not support parallel file uploads. Ignoring.", parallelFilesUpload) - } - } - - backupCmd := resticBackupCMDFunc(rp.repoIdentifier, rp.credentialsFile, path, tags) - backupCmd.Env = rp.cmdEnv - backupCmd.CACertFile = rp.caCertFile - if len(rp.extraFlags) != 0 { - backupCmd.ExtraFlags = append(backupCmd.ExtraFlags, rp.extraFlags...) - } - - if parentSnapshot != "" { - backupCmd.ExtraFlags = append(backupCmd.ExtraFlags, fmt.Sprintf("--parent=%s", parentSnapshot)) - } - - summary, stderrBuf, err := resticBackupFunc(backupCmd, log, updater) - if err != nil { - if strings.Contains(stderrBuf, "snapshot is empty") { - log.Debugf("Restic backup got empty dir with %s path", path) - return "", true, 0, 0, nil - } - return "", false, 0, 0, errors.WithStack(fmt.Errorf("error running restic backup command %s with error: %v stderr: %v", backupCmd.String(), err, stderrBuf)) - } - // GetSnapshotID - snapshotIDCmd := resticGetSnapshotFunc(rp.repoIdentifier, rp.credentialsFile, tags) - snapshotIDCmd.Env = rp.cmdEnv - snapshotIDCmd.CACertFile = rp.caCertFile - if len(rp.extraFlags) != 0 { - snapshotIDCmd.ExtraFlags = append(snapshotIDCmd.ExtraFlags, rp.extraFlags...) - } - snapshotID, err := resticGetSnapshotIDFunc(snapshotIDCmd) - if err != nil { - return "", false, 0, 0, errors.WithStack(fmt.Errorf("error getting snapshot id with error: %v", err)) - } - log.Infof("Run command=%s, stdout=%s, stderr=%s", backupCmd.String(), summary, stderrBuf) - return snapshotID, false, 0, 0, nil -} - -// RunRestore runs a `restore` command and monitors the volume size to -// provide progress updates to the caller. -func (rp *resticProvider) RunRestore( - ctx context.Context, - snapshotID string, - volumePath string, - volMode uploader.PersistentVolumeMode, - uploaderCfg map[string]string, - updater uploader.ProgressUpdater) (int64, error) { - if updater == nil { - return 0, errors.New("Need to initial backup progress updater first") - } - log := rp.log.WithFields(logrus.Fields{ - "snapshotID": snapshotID, - "volumePath": volumePath, - }) - - if volMode == uploader.PersistentVolumeBlock { - return 0, errors.New("unable to support block mode") - } - - restoreCmd := resticRestoreCMDFunc(rp.repoIdentifier, rp.credentialsFile, snapshotID, volumePath) - restoreCmd.Env = rp.cmdEnv - restoreCmd.CACertFile = rp.caCertFile - if len(rp.extraFlags) != 0 { - restoreCmd.ExtraFlags = append(restoreCmd.ExtraFlags, rp.extraFlags...) - } - - extraFlags, err := rp.parseRestoreExtraFlags(uploaderCfg) - if err != nil { - return 0, errors.Wrap(err, "failed to parse uploader config") - } else if len(extraFlags) != 0 { - restoreCmd.ExtraFlags = append(restoreCmd.ExtraFlags, extraFlags...) - } - - stdout, stderr, err := restic.RunRestore(restoreCmd, log, updater) - - log.Infof("Run command=%v, stdout=%s, stderr=%s", restoreCmd, stdout, stderr) - return 0, err -} - -func (rp *resticProvider) parseRestoreExtraFlags(uploaderCfg map[string]string) ([]string, error) { - extraFlags := []string{} - if len(uploaderCfg) == 0 { - return extraFlags, nil - } - - writeSparseFiles, err := uploaderutil.GetWriteSparseFiles(uploaderCfg) - if err != nil { - return extraFlags, errors.Wrap(err, "failed to get uploader config") - } - - if writeSparseFiles { - extraFlags = append(extraFlags, "--sparse") - } - - if restoreConcurrency, err := uploaderutil.GetRestoreConcurrency(uploaderCfg); err == nil && restoreConcurrency > 0 { - return extraFlags, errors.New("restic does not support parallel restore") - } - - return extraFlags, nil -} diff --git a/pkg/uploader/provider/restic_test.go b/pkg/uploader/provider/restic_test.go deleted file mode 100644 index 24eb11e04..000000000 --- a/pkg/uploader/provider/restic_test.go +++ /dev/null @@ -1,464 +0,0 @@ -/* -Copyright The Velero Contributors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package provider - -import ( - "errors" - "os" - "reflect" - "strings" - "testing" - - "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - corev1api "k8s.io/api/core/v1" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - - "github.com/vmware-tanzu/velero/internal/credentials" - velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" - "github.com/vmware-tanzu/velero/pkg/builder" - "github.com/vmware-tanzu/velero/pkg/restic" - "github.com/vmware-tanzu/velero/pkg/uploader" - "github.com/vmware-tanzu/velero/pkg/util" - "github.com/vmware-tanzu/velero/pkg/util/filesystem" -) - -func TestResticRunBackup(t *testing.T) { - testCases := []struct { - name string - nilUpdater bool - parentSnapshot string - rp *resticProvider - volMode uploader.PersistentVolumeMode - hookBackupFunc func(string, string, string, map[string]string) *restic.Command - hookResticBackupFunc func(*restic.Command, logrus.FieldLogger, uploader.ProgressUpdater) (string, string, error) - hookResticGetSnapshotFunc func(string, string, map[string]string) *restic.Command - hookResticGetSnapshotIDFunc func(*restic.Command) (string, error) - errorHandleFunc func(err error) bool - }{ - { - name: "nil uploader", - rp: &resticProvider{log: logrus.New()}, - nilUpdater: true, - hookBackupFunc: func(repoIdentifier string, passwordFile string, path string, tags map[string]string) *restic.Command { - return &restic.Command{Command: "date"} - }, - errorHandleFunc: func(err error) bool { - return strings.Contains(err.Error(), "Need to initial backup progress updater first") - }, - }, - { - name: "wrong restic execute command", - rp: &resticProvider{log: logrus.New()}, - hookBackupFunc: func(repoIdentifier string, passwordFile string, path string, tags map[string]string) *restic.Command { - return &restic.Command{Command: "date"} - }, - errorHandleFunc: func(err error) bool { - return strings.Contains(err.Error(), "error running") - }, - }, { - name: "has parent snapshot", - rp: &resticProvider{log: logrus.New()}, - parentSnapshot: "parentSnapshot", - hookBackupFunc: func(repoIdentifier string, passwordFile string, path string, tags map[string]string) *restic.Command { - return &restic.Command{Command: "date"} - }, - hookResticBackupFunc: func(*restic.Command, logrus.FieldLogger, uploader.ProgressUpdater) (string, string, error) { - return "", "", nil - }, - - hookResticGetSnapshotIDFunc: func(*restic.Command) (string, error) { return "test-snapshot-id", nil }, - errorHandleFunc: func(err error) bool { - return err == nil - }, - }, - { - name: "has extra flags", - rp: &resticProvider{log: logrus.New(), extraFlags: []string{"testFlags"}}, - hookBackupFunc: func(string, string, string, map[string]string) *restic.Command { - return &restic.Command{Command: "date"} - }, - hookResticBackupFunc: func(*restic.Command, logrus.FieldLogger, uploader.ProgressUpdater) (string, string, error) { - return "", "", nil - }, - hookResticGetSnapshotIDFunc: func(*restic.Command) (string, error) { return "test-snapshot-id", nil }, - errorHandleFunc: func(err error) bool { - return err == nil - }, - }, - { - name: "failed to get snapshot id", - rp: &resticProvider{log: logrus.New(), extraFlags: []string{"testFlags"}}, - hookBackupFunc: func(string, string, string, map[string]string) *restic.Command { - return &restic.Command{Command: "date"} - }, - hookResticBackupFunc: func(*restic.Command, logrus.FieldLogger, uploader.ProgressUpdater) (string, string, error) { - return "", "", nil - }, - hookResticGetSnapshotIDFunc: func(*restic.Command) (string, error) { - return "test-snapshot-id", errors.New("failed to get snapshot id") - }, - errorHandleFunc: func(err error) bool { - return strings.Contains(err.Error(), "failed to get snapshot id") - }, - }, - { - name: "failed to use block mode", - rp: &resticProvider{log: logrus.New(), extraFlags: []string{"testFlags"}}, - volMode: uploader.PersistentVolumeBlock, - errorHandleFunc: func(err error) bool { - return strings.Contains(err.Error(), "unable to support block mode") - }, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - var err error - parentSnapshot := tc.parentSnapshot - if tc.hookBackupFunc != nil { - resticBackupCMDFunc = tc.hookBackupFunc - } - if tc.hookResticBackupFunc != nil { - resticBackupFunc = tc.hookResticBackupFunc - } - if tc.hookResticGetSnapshotFunc != nil { - resticGetSnapshotFunc = tc.hookResticGetSnapshotFunc - } - if tc.hookResticGetSnapshotIDFunc != nil { - resticGetSnapshotIDFunc = tc.hookResticGetSnapshotIDFunc - } - if tc.volMode == "" { - tc.volMode = uploader.PersistentVolumeFilesystem - } - if !tc.nilUpdater { - updater := FakeBackupProgressUpdater{PodVolumeBackup: &velerov1api.PodVolumeBackup{}, Log: tc.rp.log, Ctx: t.Context(), Cli: fake.NewClientBuilder().WithScheme(util.VeleroScheme).Build()} - _, _, _, _, err = tc.rp.RunBackup(t.Context(), "var", "", map[string]string{}, false, parentSnapshot, tc.volMode, map[string]string{}, &updater) - } else { - _, _, _, _, err = tc.rp.RunBackup(t.Context(), "var", "", map[string]string{}, false, parentSnapshot, tc.volMode, map[string]string{}, nil) - } - - tc.rp.log.Infof("test name %v error %v", tc.name, err) - require.True(t, tc.errorHandleFunc(err)) - }) - } -} - -func TestResticRunRestore(t *testing.T) { - resticRestoreCMDFunc = func(repoIdentifier, passwordFile, snapshotID, target string) *restic.Command { - return &restic.Command{Args: []string{""}} - } - testCases := []struct { - name string - rp *resticProvider - nilUpdater bool - hookResticRestoreFunc func(repoIdentifier, passwordFile, snapshotID, target string) *restic.Command - errorHandleFunc func(err error) bool - volMode uploader.PersistentVolumeMode - }{ - { - name: "wrong restic execute command", - rp: &resticProvider{log: logrus.New()}, - nilUpdater: true, - errorHandleFunc: func(err error) bool { - return strings.Contains(err.Error(), "Need to initial backup progress updater first") - }, - }, - { - name: "has extral flags", - rp: &resticProvider{log: logrus.New(), extraFlags: []string{"test-extra-flags"}}, - hookResticRestoreFunc: func(repoIdentifier, passwordFile, snapshotID, target string) *restic.Command { - return &restic.Command{Args: []string{"date"}} - }, - errorHandleFunc: func(err error) bool { - return strings.Contains(err.Error(), "error running command") - }, - }, - { - name: "wrong restic execute command", - rp: &resticProvider{log: logrus.New()}, - hookResticRestoreFunc: func(repoIdentifier, passwordFile, snapshotID, target string) *restic.Command { - return &restic.Command{Args: []string{"date"}} - }, - errorHandleFunc: func(err error) bool { - return strings.Contains(err.Error(), "error running command") - }, - }, - { - name: "error block volume mode", - rp: &resticProvider{log: logrus.New()}, - errorHandleFunc: func(err error) bool { - return strings.Contains(err.Error(), "unable to support block mode") - }, - volMode: uploader.PersistentVolumeBlock, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - if tc.volMode == "" { - tc.volMode = uploader.PersistentVolumeFilesystem - } - resticRestoreCMDFunc = tc.hookResticRestoreFunc - if tc.volMode == "" { - tc.volMode = uploader.PersistentVolumeFilesystem - } - var err error - if !tc.nilUpdater { - updater := FakeBackupProgressUpdater{PodVolumeBackup: &velerov1api.PodVolumeBackup{}, Log: tc.rp.log, Ctx: t.Context(), Cli: fake.NewClientBuilder().WithScheme(util.VeleroScheme).Build()} - _, err = tc.rp.RunRestore(t.Context(), "", "var", tc.volMode, map[string]string{}, &updater) - } else { - _, err = tc.rp.RunRestore(t.Context(), "", "var", tc.volMode, map[string]string{}, nil) - } - - tc.rp.log.Infof("test name %v error %v", tc.name, err) - require.True(t, tc.errorHandleFunc(err)) - }) - } -} - -func TestClose(t *testing.T) { - t.Run("Delete existing credentials file", func(t *testing.T) { - // Create temporary files for the credentials and caCert - credentialsFile, err := os.CreateTemp(t.TempDir(), "credentialsFile") - if err != nil { - t.Fatalf("failed to create temp file: %v", err) - } - defer os.Remove(credentialsFile.Name()) - - caCertFile, err := os.CreateTemp(t.TempDir(), "caCertFile") - if err != nil { - t.Fatalf("failed to create temp file: %v", err) - } - defer os.Remove(caCertFile.Name()) - rp := &resticProvider{ - credentialsFile: credentialsFile.Name(), - caCertFile: caCertFile.Name(), - } - // Test deleting an existing credentials file - err = rp.Close(t.Context()) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - - _, err = os.Stat(rp.credentialsFile) - if !os.IsNotExist(err) { - t.Errorf("expected credentials file to be deleted, got error: %v", err) - } - }) - - t.Run("Delete existing caCert file", func(t *testing.T) { - // Create temporary files for the credentials and caCert - caCertFile, err := os.CreateTemp(t.TempDir(), "caCertFile") - if err != nil { - t.Fatalf("failed to create temp file: %v", err) - } - defer os.Remove(caCertFile.Name()) - rp := &resticProvider{ - credentialsFile: "", - caCertFile: "", - } - err = rp.Close(t.Context()) - // Test deleting an existing caCert file - if err != nil { - t.Errorf("unexpected error: %v", err) - } - - _, err = os.Stat(rp.caCertFile) - if !os.IsNotExist(err) { - t.Errorf("expected caCert file to be deleted, got error: %v", err) - } - }) -} - -type MockCredentialGetter struct { - mock.Mock -} - -func (m *MockCredentialGetter) Path(selector *corev1api.SecretKeySelector) (string, error) { - args := m.Called(selector) - return args.Get(0).(string), args.Error(1) -} - -func TestNewResticUploaderProvider(t *testing.T) { - testCases := []struct { - name string - emptyBSL bool - mockCredFunc func(*MockCredentialGetter, *corev1api.SecretKeySelector) - resticCmdEnvFunc func(backupLocation *velerov1api.BackupStorageLocation, credentialFileStore credentials.FileStore) ([]string, error) - resticTempCACertFileFunc func(caCert []byte, bsl string, fs filesystem.Interface) (string, error) - checkFunc func(t *testing.T, provider Provider, err error) - }{ - { - name: "No error in creating temp credentials file", - mockCredFunc: func(credGetter *MockCredentialGetter, repoKeySelector *corev1api.SecretKeySelector) { - credGetter.On("Path", repoKeySelector).Return("temp-credentials", nil) - }, - checkFunc: func(t *testing.T, provider Provider, err error) { - t.Helper() - require.NoError(t, err) - assert.NotNil(t, provider) - }, - }, { - name: "Error in creating temp credentials file", - mockCredFunc: func(credGetter *MockCredentialGetter, repoKeySelector *corev1api.SecretKeySelector) { - credGetter.On("Path", repoKeySelector).Return("", errors.New("error creating temp credentials file")) - }, - checkFunc: func(t *testing.T, provider Provider, err error) { - t.Helper() - require.Error(t, err) - assert.Nil(t, provider) - }, - }, { - name: "ObjectStorage with CACert present and creating CACert file failed", - mockCredFunc: func(credGetter *MockCredentialGetter, repoKeySelector *corev1api.SecretKeySelector) { - credGetter.On("Path", repoKeySelector).Return("temp-credentials", nil) - }, - resticTempCACertFileFunc: func(caCert []byte, bsl string, fs filesystem.Interface) (string, error) { - return "", errors.New("error writing CACert file") - }, - checkFunc: func(t *testing.T, provider Provider, err error) { - t.Helper() - require.Error(t, err) - assert.Nil(t, provider) - }, - }, { - name: "Generating repository cmd failed", - mockCredFunc: func(credGetter *MockCredentialGetter, repoKeySelector *corev1api.SecretKeySelector) { - credGetter.On("Path", repoKeySelector).Return("temp-credentials", nil) - }, - resticTempCACertFileFunc: func(caCert []byte, bsl string, fs filesystem.Interface) (string, error) { - return "test-ca", nil - }, - resticCmdEnvFunc: func(backupLocation *velerov1api.BackupStorageLocation, credentialFileStore credentials.FileStore) ([]string, error) { - return nil, errors.New("error generating repository cmnd env") - }, - checkFunc: func(t *testing.T, provider Provider, err error) { - t.Helper() - require.Error(t, err) - assert.Nil(t, provider) - }, - }, { - name: "New provider with not nil bsl", - mockCredFunc: func(credGetter *MockCredentialGetter, repoKeySelector *corev1api.SecretKeySelector) { - credGetter.On("Path", repoKeySelector).Return("temp-credentials", nil) - }, - resticTempCACertFileFunc: func(caCert []byte, bsl string, fs filesystem.Interface) (string, error) { - return "test-ca", nil - }, - resticCmdEnvFunc: func(backupLocation *velerov1api.BackupStorageLocation, credentialFileStore credentials.FileStore) ([]string, error) { - return nil, nil - }, - checkFunc: func(t *testing.T, provider Provider, err error) { - t.Helper() - require.NoError(t, err) - assert.NotNil(t, provider) - }, - }, - { - name: "New provider with nil bsl", - emptyBSL: true, - mockCredFunc: func(credGetter *MockCredentialGetter, repoKeySelector *corev1api.SecretKeySelector) { - credGetter.On("Path", repoKeySelector).Return("temp-credentials", nil) - }, - resticTempCACertFileFunc: func(caCert []byte, bsl string, fs filesystem.Interface) (string, error) { - return "test-ca", nil - }, - resticCmdEnvFunc: func(backupLocation *velerov1api.BackupStorageLocation, credentialFileStore credentials.FileStore) ([]string, error) { - return nil, nil - }, - checkFunc: func(t *testing.T, provider Provider, err error) { - t.Helper() - require.NoError(t, err) - assert.NotNil(t, provider) - }, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - repoIdentifier := "my-repo" - bsl := &velerov1api.BackupStorageLocation{} - if !tc.emptyBSL { - bsl = builder.ForBackupStorageLocation("test-ns", "test-name").CACert([]byte("my-cert")).Result() - } - credGetter := &credentials.CredentialGetter{} - repoKeySelector := &corev1api.SecretKeySelector{} - log := logrus.New() - - // Mock CredentialGetter - mockCredGetter := &MockCredentialGetter{} - credGetter.FromFile = mockCredGetter - tc.mockCredFunc(mockCredGetter, repoKeySelector) - if tc.resticCmdEnvFunc != nil { - resticCmdEnvFunc = tc.resticCmdEnvFunc - } - if tc.resticTempCACertFileFunc != nil { - resticTempCACertFileFunc = tc.resticTempCACertFileFunc - } - provider, err := NewResticUploaderProvider(repoIdentifier, bsl, credGetter, repoKeySelector, log) - tc.checkFunc(t, provider, err) - }) - } -} - -func TestParseUploaderConfig(t *testing.T) { - rp := &resticProvider{} - - testCases := []struct { - name string - uploaderConfig map[string]string - expectedFlags []string - }{ - { - name: "SparseFilesEnabled", - uploaderConfig: map[string]string{ - "WriteSparseFiles": "true", - }, - expectedFlags: []string{"--sparse"}, - }, - { - name: "SparseFilesDisabled", - uploaderConfig: map[string]string{ - "writeSparseFiles": "false", - }, - expectedFlags: []string{}, - }, - { - name: "RestoreConcorrency", - uploaderConfig: map[string]string{ - "Parallel": "5", - }, - expectedFlags: []string{}, - }, - } - - for _, testCase := range testCases { - t.Run(testCase.name, func(t *testing.T) { - result, err := rp.parseRestoreExtraFlags(testCase.uploaderConfig) - if err != nil { - t.Errorf("Test case %s failed with error: %v", testCase.name, err) - return - } - - if !reflect.DeepEqual(result, testCase.expectedFlags) { - t.Errorf("Test case %s failed. Expected: %v, Got: %v", testCase.name, testCase.expectedFlags, result) - } - }) - } -} diff --git a/pkg/uploader/types.go b/pkg/uploader/types.go index f69cbf072..52f8ca5bf 100644 --- a/pkg/uploader/types.go +++ b/pkg/uploader/types.go @@ -22,7 +22,6 @@ import ( ) const ( - ResticType = "restic" KopiaType = "kopia" SnapshotRequesterTag = "snapshot-requester" SnapshotUploaderTag = "snapshot-uploader"