mirrors/velero
1
0
Fork 0
mirror of https://github.com/vmware-tanzu/velero.git synced 2026-01-05 13:05:17 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'main' of https://github.com/qiuming-best/velero into support-restore-sparse

Browse Source
This commit is contained in:
Ming Qiu
2023-11-28 13:53:49 +00:00
parent b57dde1572 85482aefaf
commit 0afaa70e9b
95 changed files with 2439 additions and 570 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
pkg/uploader/kopia/snapshot.go
View File

@@ -18,7 +18,6 @@ package kopia
import (
"context"
"encoding/json"
"fmt"
"math"
"os"
@@ -39,7 +38,7 @@ import (
"github.com/kopia/kopia/snapshot/snapshotfs"
"github.com/pkg/errors"
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
"github.com/vmware-tanzu/velero/pkg/util/uploaderconfig"
"github.com/vmware-tanzu/velero/pkg/kopia"
"github.com/vmware-tanzu/velero/pkg/repository/udmrepo"
@@ -107,21 +106,18 @@ func getDefaultPolicy() *policy.Policy {
}
}
func setupPolicy(ctx context.Context, rep repo.RepositoryWriter, sourceInfo snapshot.SourceInfo, uploaderCfg map[string]string) (*policy.Tree, error) {
func setupPolicy(ctx context.Context, rep repo.RepositoryWriter, sourceInfo snapshot.SourceInfo, uploaderCfg *map[string]string) (*policy.Tree, error) {
// some internal operations from Kopia code retrieves policies from repo directly, so we need to persist the policy to repo
backupCfg := velerov1api.BackupConfig{}
// currently, we only have one uploader config in one uploader config so we can just loop through it
for configItem, jsonConfig := range uploaderCfg {
err := json.Unmarshal([]byte(jsonConfig), &backupCfg)
if err != nil {
return nil, errors.Wrapf(err, "failed to parse %s uploader config", configItem)
}
break
}
curPolicy := getDefaultPolicy()
if backupCfg.ParallelFilesUpload > 0 {
curPolicy.UploadPolicy.MaxParallelFileReads = newOptionalInt(backupCfg.ParallelFilesUpload)
if uploaderCfg != nil {
uploaderConfig, err := uploaderconfig.GetBackupConfig(uploaderCfg)
if err != nil {
return nil, errors.Wrap(err, "failed to get uploader config")
}
if uploaderConfig.ParallelFilesUpload > 0 {
curPolicy.UploadPolicy.MaxParallelFileReads = newOptionalInt(uploaderConfig.ParallelFilesUpload)
}
}
err := setPolicyFunc(ctx, rep, sourceInfo, curPolicy)
@@ -145,7 +141,7 @@ func setupPolicy(ctx context.Context, rep repo.RepositoryWriter, sourceInfo snap
// Backup backup specific sourcePath and update progress
func Backup(ctx context.Context, fsUploader SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string,
forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) {
forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg *map[string]string, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) {
if fsUploader == nil {
return nil, false, errors.New("get empty kopia uploader")
}
@@ -241,7 +237,7 @@ func SnapshotSource(
forceFull bool,
parentSnapshot string,
snapshotTags map[string]string,
uploaderCfg map[string]string,
uploaderCfg *map[string]string,
log logrus.FieldLogger,
description string,
) (string, int64, error) {
@@ -373,7 +369,7 @@ func findPreviousSnapshotManifest(ctx context.Context, rep repo.Repository, sour
}
// Restore restore specific sourcePath with given snapshotID and update progress
func Restore(ctx context.Context, rep repo.RepositoryWriter, progress *Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string,
func Restore(ctx context.Context, rep repo.RepositoryWriter, progress *Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg *map[string]string,
log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) {
log.Info("Start to restore...")
@@ -396,23 +392,20 @@ func Restore(ctx context.Context, rep repo.RepositoryWriter, progress *Progress,
return 0, 0, errors.Wrapf(err, "Unable to resolve path %v", dest)
}
restoreCfg := velerov1api.RestoreConfig{}
// currently, we only have one uploader config in map so we can just loop through it
for configItem, jsonConfig := range uploaderCfg {
err := json.Unmarshal([]byte(jsonConfig), &restoreCfg)
if err != nil {
return 0, 0, errors.Wrapf(err, "failed to parse %s uploader config", configItem)
}
break
}
fsOutput := &restore.FilesystemOutput{
TargetPath: path,
OverwriteDirectories: true,
OverwriteFiles: true,
OverwriteSymlinks: true,
IgnorePermissionErrors: true,
WriteSparseFiles: restoreCfg.WriteSparseFiles,
}
if uploaderCfg != nil {
restoreCfg, err := uploaderconfig.GetRestoreConfig(uploaderCfg)
if err != nil {
return 0, 0, errors.Wrap(err, "failed to get uploader config")
}
fsOutput.WriteSparseFiles = restoreCfg.WriteSparseFiles
}
log.Debugf("Restore filesystem output %v", fsOutput)

16
pkg/uploader/kopia/snapshot_test.go
View File

@@ -96,7 +96,7 @@ func TestSnapshotSource(t *testing.T) {
testCases := []struct {
name string
args []mockArgs
uploaderCfg map[string]string
uploaderCfg *map[string]string
notError bool
}{
{
@@ -152,7 +152,7 @@ func TestSnapshotSource(t *testing.T) {
notError: false,
},
{
name: "set policy with ParallelFilesUpload",
name: "set policy with parallel files upload",
args: []mockArgs{
{methodName: "LoadSnapshot", returns: []interface{}{manifest, nil}},
{methodName: "SaveSnapshot", returns: []interface{}{manifest.ID, nil}},
@@ -162,8 +162,10 @@ func TestSnapshotSource(t *testing.T) {
{methodName: "Upload", returns: []interface{}{manifest, nil}},
{methodName: "Flush", returns: []interface{}{nil}},
},
uploaderCfg: map[string]string{"ParallelFilesUpload": "10"},
notError: true,
uploaderCfg: &map[string]string{
"ParallelFilesUpload": "10",
},
notError: true,
},
{
name: "failed to upload snapshot",
@@ -645,9 +647,9 @@ func TestBackup(t *testing.T) {
var snapshotInfo *uploader.SnapshotInfo
var err error
if tc.isEmptyUploader {
snapshotInfo, isSnapshotEmpty, err = Backup(context.Background(), nil, s.repoWriterMock, tc.sourcePath, "", tc.forceFull, tc.parentSnapshot, tc.volMode, make(map[string]string), tc.tags, &logrus.Logger{})
snapshotInfo, isSnapshotEmpty, err = Backup(context.Background(), nil, s.repoWriterMock, tc.sourcePath, "", tc.forceFull, tc.parentSnapshot, tc.volMode, &map[string]string{}, tc.tags, &logrus.Logger{})
} else {
snapshotInfo, isSnapshotEmpty, err = Backup(context.Background(), s.uploderMock, s.repoWriterMock, tc.sourcePath, "", tc.forceFull, tc.parentSnapshot, tc.volMode, make(map[string]string), tc.tags, &logrus.Logger{})
snapshotInfo, isSnapshotEmpty, err = Backup(context.Background(), s.uploderMock, s.repoWriterMock, tc.sourcePath, "", tc.forceFull, tc.parentSnapshot, tc.volMode, &map[string]string{}, tc.tags, &logrus.Logger{})
}
// Check if the returned error matches the expected error
if tc.expectedError != nil {
@@ -786,7 +788,7 @@ func TestRestore(t *testing.T) {
repoWriterMock.On("OpenObject", mock.Anything, mock.Anything).Return(em, nil)
progress := new(Progress)
bytesRestored, fileCount, err := Restore(context.Background(), repoWriterMock, progress, tc.snapshotID, tc.dest, tc.volMode, make(map[string]string), logrus.New(), nil)
bytesRestored, fileCount, err := Restore(context.Background(), repoWriterMock, progress, tc.snapshotID, tc.dest, tc.volMode, &map[string]string{}, logrus.New(), nil)
// Check if the returned error matches the expected error
if tc.expectedError != nil {

4
pkg/uploader/provider/kopia.go
View File

@@ -119,7 +119,7 @@ func (kp *kopiaProvider) RunBackup(
forceFull bool,
parentSnapshot string,
volMode uploader.PersistentVolumeMode,
uploaderCfg map[string]string,
uploaderCfg *map[string]string,
updater uploader.ProgressUpdater) (string, bool, error) {
if updater == nil {
return "", false, errors.New("Need to initial backup progress updater first")
@@ -204,7 +204,7 @@ func (kp *kopiaProvider) RunRestore(
snapshotID string,
volumePath string,
volMode uploader.PersistentVolumeMode,
uploaderCfg map[string]string,
uploaderCfg *map[string]string,
updater uploader.ProgressUpdater) error {
log := kp.log.WithFields(logrus.Fields{
"snapshotID": snapshotID,

22
pkg/uploader/provider/kopia_test.go
View File

@@ -68,34 +68,34 @@ func TestRunBackup(t *testing.T) {
testCases := []struct {
name string
hookBackupFunc func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error)
hookBackupFunc func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg *map[string]string, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error)
volMode uploader.PersistentVolumeMode
notError bool
}{
{
name: "success to backup",
hookBackupFunc: func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) {
hookBackupFunc: func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg *map[string]string, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) {
return &uploader.SnapshotInfo{}, false, nil
},
notError: true,
},
{
name: "get error to backup",
hookBackupFunc: func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) {
hookBackupFunc: func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg *map[string]string, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) {
return &uploader.SnapshotInfo{}, false, errors.New("failed to backup")
},
notError: false,
},
{
name: "got empty snapshot",
hookBackupFunc: func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) {
hookBackupFunc: func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg *map[string]string, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) {
return nil, true, errors.New("snapshot is empty")
},
notError: false,
},
{
name: "success to backup block mode volume",
hookBackupFunc: func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) {
hookBackupFunc: func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg *map[string]string, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) {
return &uploader.SnapshotInfo{}, false, nil
},
volMode: uploader.PersistentVolumeBlock,
@@ -108,7 +108,7 @@ func TestRunBackup(t *testing.T) {
tc.volMode = uploader.PersistentVolumeFilesystem
}
BackupFunc = tc.hookBackupFunc
_, _, err := kp.RunBackup(context.Background(), "var", "", nil, false, "", tc.volMode, map[string]string{}, &updater)
_, _, err := kp.RunBackup(context.Background(), "var", "", nil, false, "", tc.volMode, &map[string]string{}, &updater)
if tc.notError {
assert.NoError(t, err)
} else {
@@ -125,27 +125,27 @@ func TestRunRestore(t *testing.T) {
testCases := []struct {
name string
hookRestoreFunc func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error)
hookRestoreFunc func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg *map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error)
notError bool
volMode uploader.PersistentVolumeMode
}{
{
name: "normal restore",
hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) {
hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg *map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) {
return 0, 0, nil
},
notError: true,
},
{
name: "failed to restore",
hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) {
hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg *map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) {
return 0, 0, errors.New("failed to restore")
},
notError: false,
},
{
name: "normal block mode restore",
hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) {
hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg *map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) {
return 0, 0, nil
},
volMode: uploader.PersistentVolumeBlock,
@@ -159,7 +159,7 @@ func TestRunRestore(t *testing.T) {
tc.volMode = uploader.PersistentVolumeFilesystem
}
RestoreFunc = tc.hookRestoreFunc
err := kp.RunRestore(context.Background(), "", "/var", tc.volMode, map[string]string{}, &updater)
err := kp.RunRestore(context.Background(), "", "/var", tc.volMode, &map[string]string{}, &updater)
if tc.notError {
assert.NoError(t, err)
} else {

14
pkg/uploader/provider/mocks/Provider.go
View File

@@ -30,28 +30,28 @@ func (_m *Provider) Close(ctx context.Context) error {
}
// RunBackup provides a mock function with given fields: ctx, path, realSource, tags, forceFull, parentSnapshot, volMode, uploaderCfg, updater
func (_m *Provider) RunBackup(ctx context.Context, path string, realSource string, tags map[string]string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, updater uploader.ProgressUpdater) (string, bool, error) {
func (_m *Provider) RunBackup(ctx context.Context, path string, realSource string, tags map[string]string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg *map[string]string, updater uploader.ProgressUpdater) (string, bool, error) {
ret := _m.Called(ctx, path, realSource, tags, forceFull, parentSnapshot, volMode, uploaderCfg, updater)
var r0 string
var r1 bool
var r2 error
if rf, ok := ret.Get(0).(func(context.Context, string, string, map[string]string, bool, string, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) (string, bool, error)); ok {
if rf, ok := ret.Get(0).(func(context.Context, string, string, map[string]string, bool, string, uploader.PersistentVolumeMode, *map[string]string, uploader.ProgressUpdater) (string, bool, error)); ok {
return rf(ctx, path, realSource, tags, forceFull, parentSnapshot, volMode, uploaderCfg, updater)
}
if rf, ok := ret.Get(0).(func(context.Context, string, string, map[string]string, bool, string, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) string); ok {
if rf, ok := ret.Get(0).(func(context.Context, string, string, map[string]string, bool, string, uploader.PersistentVolumeMode, *map[string]string, uploader.ProgressUpdater) string); ok {
r0 = rf(ctx, path, realSource, tags, forceFull, parentSnapshot, volMode, uploaderCfg, updater)
} else {
r0 = ret.Get(0).(string)
}
if rf, ok := ret.Get(1).(func(context.Context, string, string, map[string]string, bool, string, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) bool); ok {
if rf, ok := ret.Get(1).(func(context.Context, string, string, map[string]string, bool, string, uploader.PersistentVolumeMode, *map[string]string, uploader.ProgressUpdater) bool); ok {
r1 = rf(ctx, path, realSource, tags, forceFull, parentSnapshot, volMode, uploaderCfg, updater)
} else {
r1 = ret.Get(1).(bool)
}
if rf, ok := ret.Get(2).(func(context.Context, string, string, map[string]string, bool, string, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) error); ok {
if rf, ok := ret.Get(2).(func(context.Context, string, string, map[string]string, bool, string, uploader.PersistentVolumeMode, *map[string]string, uploader.ProgressUpdater) error); ok {
r2 = rf(ctx, path, realSource, tags, forceFull, parentSnapshot, volMode, uploaderCfg, updater)
} else {
r2 = ret.Error(2)
@@ -61,11 +61,11 @@ func (_m *Provider) RunBackup(ctx context.Context, path string, realSource strin
}
// RunRestore provides a mock function with given fields: ctx, snapshotID, volumePath, volMode, uploaderConfig, updater
func (_m *Provider) RunRestore(ctx context.Context, snapshotID string, volumePath string, volMode uploader.PersistentVolumeMode, uploaderConfig map[string]string, updater uploader.ProgressUpdater) error {
func (_m *Provider) RunRestore(ctx context.Context, snapshotID string, volumePath string, volMode uploader.PersistentVolumeMode, uploaderConfig *map[string]string, updater uploader.ProgressUpdater) error {
ret := _m.Called(ctx, snapshotID, volumePath, volMode, uploaderConfig, updater)
var r0 error
if rf, ok := ret.Get(0).(func(context.Context, string, string, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) error); ok {
if rf, ok := ret.Get(0).(func(context.Context, string, string, uploader.PersistentVolumeMode, *map[string]string, uploader.ProgressUpdater) error); ok {
r0 = rf(ctx, snapshotID, volumePath, volMode, uploaderConfig, updater)
} else {
r0 = ret.Error(0)

4
pkg/uploader/provider/provider.go
View File

@@ -49,7 +49,7 @@ type Provider interface {
forceFull bool,
parentSnapshot string,
volMode uploader.PersistentVolumeMode,
uploaderCfg map[string]string,
uploaderCfg *map[string]string,
updater uploader.ProgressUpdater) (string, bool, error)
// RunRestore which will do restore for one specific volume with given snapshot id and return error
// updater is used for updating backup progress which implement by third-party
@@ -58,7 +58,7 @@ type Provider interface {
snapshotID string,
volumePath string,
volMode uploader.PersistentVolumeMode,
uploaderConfig map[string]string,
uploaderConfig *map[string]string,
updater uploader.ProgressUpdater) error
// Close which will close related repository
Close(ctx context.Context) error

48
pkg/uploader/provider/restic.go
View File

@@ -18,7 +18,6 @@ package provider
import (
"context"
"encoding/json"
"fmt"
"os"
"strings"
@@ -32,6 +31,7 @@ import (
"github.com/vmware-tanzu/velero/pkg/restic"
"github.com/vmware-tanzu/velero/pkg/uploader"
"github.com/vmware-tanzu/velero/pkg/util/filesystem"
"github.com/vmware-tanzu/velero/pkg/util/uploaderconfig"
)
// resticBackupCMDFunc and resticRestoreCMDFunc are mainly used to make testing more convenient
@@ -123,7 +123,7 @@ func (rp *resticProvider) RunBackup(
forceFull bool,
parentSnapshot string,
volMode uploader.PersistentVolumeMode,
uploaderCfg map[string]string,
uploaderCfg *map[string]string,
updater uploader.ProgressUpdater) (string, bool, error) {
if updater == nil {
return "", false, errors.New("Need to initial backup progress updater first")
@@ -146,18 +146,15 @@ func (rp *resticProvider) RunBackup(
"parentSnapshot": parentSnapshot,
})
backupCfg := velerov1api.BackupConfig{}
// currently, we only have one uploader config in one uploader config so we can just loop through it
for configItem, jsonConfig := range uploaderCfg {
err := json.Unmarshal([]byte(jsonConfig), &backupCfg)
if uploaderCfg != nil {
uploaderConfig, err := uploaderconfig.GetBackupConfig(uploaderCfg)
if err != nil {
return "", false, errors.Wrapf(err, "failed to parse %s config", configItem)
return "", false, errors.Wrap(err, "failed to get uploader config")
}
break
}
if backupCfg.ParallelFilesUpload > 0 {
log.Warnf("ParallelFilesUpload is set to %d, but restic does not support parallel file uploads. Ignoring.", backupCfg.ParallelFilesUpload)
if uploaderConfig.ParallelFilesUpload > 0 {
log.Warnf("ParallelFilesUpload is set to %d, but restic does not support parallel file uploads. Ignoring.", uploaderConfig.ParallelFilesUpload)
}
}
backupCmd := resticBackupCMDFunc(rp.repoIdentifier, rp.credentialsFile, path, tags)
@@ -201,7 +198,7 @@ func (rp *resticProvider) RunRestore(
snapshotID string,
volumePath string,
volMode uploader.PersistentVolumeMode,
uploaderCfg map[string]string,
uploaderCfg *map[string]string,
updater uploader.ProgressUpdater) error {
if updater == nil {
return errors.New("Need to initial backup progress updater first")
@@ -222,11 +219,13 @@ func (rp *resticProvider) RunRestore(
restoreCmd.ExtraFlags = append(restoreCmd.ExtraFlags, rp.extraFlags...)
}
extraFlags, err := rp.parseRestoreExtraFlags(uploaderCfg)
if err != nil {
return errors.Wrap(err, "failed to parse uploader config")
} else if len(extraFlags) != 0 {
restoreCmd.ExtraFlags = append(restoreCmd.ExtraFlags, extraFlags...)
if uploaderCfg != nil {
extraFlags, err := rp.parseRestoreExtraFlags(uploaderCfg)
if err != nil {
return errors.Wrap(err, "failed to parse uploader config")
} else if len(extraFlags) != 0 {
restoreCmd.ExtraFlags = append(restoreCmd.ExtraFlags, extraFlags...)
}
}
stdout, stderr, err := restic.RunRestore(restoreCmd, log, updater)
@@ -235,19 +234,14 @@ func (rp *resticProvider) RunRestore(
return err
}
func (rp *resticProvider) parseRestoreExtraFlags(uploaderCfg map[string]string) ([]string, error) {
func (rp *resticProvider) parseRestoreExtraFlags(uploaderCfg *map[string]string) ([]string, error) {
extraFlags := []string{}
restoreCfg := velerov1api.RestoreConfig{}
// currently, we only have one uploader config in map so we can just loop through it
for configItem, jsonConfig := range uploaderCfg {
err := json.Unmarshal([]byte(jsonConfig), &restoreCfg)
if err != nil {
return extraFlags, errors.Wrapf(err, "failed to parse %s uploader config", configItem)
}
break
uploaderConfig, err := uploaderconfig.GetRestoreConfig(uploaderCfg)
if err != nil {
return extraFlags, errors.Wrap(err, "failed to get uploader config")
}
if restoreCfg.WriteSparseFiles {
if uploaderConfig.WriteSparseFiles {
extraFlags = append(extraFlags, "--sparse")
}
return extraFlags, nil

19
pkg/uploader/provider/restic_test.go
View File

@@ -38,7 +38,6 @@ import (
"github.com/vmware-tanzu/velero/pkg/uploader"
"github.com/vmware-tanzu/velero/pkg/util"
"github.com/vmware-tanzu/velero/pkg/util/filesystem"
"github.com/vmware-tanzu/velero/pkg/util/uploaderconfig"
)
func TestResticRunBackup(t *testing.T) {
@@ -151,9 +150,9 @@ func TestResticRunBackup(t *testing.T) {
}
if !tc.nilUpdater {
updater := FakeBackupProgressUpdater{PodVolumeBackup: &velerov1api.PodVolumeBackup{}, Log: tc.rp.log, Ctx: context.Background(), Cli: fake.NewClientBuilder().WithScheme(util.VeleroScheme).Build()}
_, _, err = tc.rp.RunBackup(context.Background(), "var", "", map[string]string{}, false, parentSnapshot, tc.volMode, map[string]string{}, &updater)
_, _, err = tc.rp.RunBackup(context.Background(), "var", "", map[string]string{}, false, parentSnapshot, tc.volMode, &map[string]string{}, &updater)
} else {
_, _, err = tc.rp.RunBackup(context.Background(), "var", "", map[string]string{}, false, parentSnapshot, tc.volMode, map[string]string{}, nil)
_, _, err = tc.rp.RunBackup(context.Background(), "var", "", map[string]string{}, false, parentSnapshot, tc.volMode, &map[string]string{}, nil)
}
tc.rp.log.Infof("test name %v error %v", tc.name, err)
@@ -224,9 +223,9 @@ func TestResticRunRestore(t *testing.T) {
var err error
if !tc.nilUpdater {
updater := FakeBackupProgressUpdater{PodVolumeBackup: &velerov1api.PodVolumeBackup{}, Log: tc.rp.log, Ctx: context.Background(), Cli: fake.NewClientBuilder().WithScheme(util.VeleroScheme).Build()}
err = tc.rp.RunRestore(context.Background(), "", "var", tc.volMode, map[string]string{}, &updater)
err = tc.rp.RunRestore(context.Background(), "", "var", tc.volMode, &map[string]string{}, &updater)
} else {
err = tc.rp.RunRestore(context.Background(), "", "var", tc.volMode, map[string]string{}, nil)
err = tc.rp.RunRestore(context.Background(), "", "var", tc.volMode, &map[string]string{}, nil)
}
tc.rp.log.Infof("test name %v error %v", tc.name, err)
@@ -418,20 +417,20 @@ func TestParseUploaderConfig(t *testing.T) {
testCases := []struct {
name string
uploaderConfig map[string]string
uploaderConfig *map[string]string
expectedFlags []string
}{
{
name: "SparseFilesEnabled",
uploaderConfig: map[string]string{
uploaderconfig.PodVolumeRestores: `{"WriteSparseFiles": true}`,
uploaderConfig: &map[string]string{
"WriteSparseFiles": "true",
},
expectedFlags: []string{"--sparse"},
},
{
name: "SparseFilesDisabled",
uploaderConfig: map[string]string{
uploaderconfig.PodVolumeRestores: `{"WriteSparseFiles": false}`,
uploaderConfig: &map[string]string{
"writeSparseFiles": "false",
},
expectedFlags: []string{},
},