Merge pull request #9676 from Lyndon-Li/remove-restic-for-repo
Some checks failed
Run the E2E test on kind / get-go-version (push) Failing after 1m5s
Run the E2E test on kind / build (push) Has been skipped
Run the E2E test on kind / setup-test-matrix (push) Successful in 3s
Run the E2E test on kind / run-e2e-test (push) Has been skipped
Main CI / get-go-version (push) Successful in 14s
Main CI / Build (push) Failing after 29s
Close stale issues and PRs / stale (push) Successful in 13s
Trivy Nightly Scan / Trivy nightly scan (velero, main) (push) Failing after 1m35s
Trivy Nightly Scan / Trivy nightly scan (velero-plugin-for-aws, main) (push) Failing after 1m10s
Trivy Nightly Scan / Trivy nightly scan (velero-plugin-for-gcp, main) (push) Failing after 1m13s
Trivy Nightly Scan / Trivy nightly scan (velero-plugin-for-microsoft-azure, main) (push) Failing after 1m16s

Remove restic for repo
This commit is contained in:
lyndon-li
2026-04-08 14:21:17 +08:00
committed by GitHub
16 changed files with 33 additions and 849 deletions

View File

@@ -0,0 +1 @@
Fix issue #9470, remove restic from repository

View File

@@ -69,9 +69,7 @@ spec:
- ""
type: string
resticIdentifier:
description: |-
ResticIdentifier is the full restic-compatible string for identifying
this repository. This field is only used when RepositoryType is "restic".
description: Deprecated
type: string
volumeNamespace:
description: |-

File diff suppressed because one or more lines are too long

View File

@@ -35,8 +35,7 @@ type BackupRepositorySpec struct {
// +optional
RepositoryType string `json:"repositoryType"`
// ResticIdentifier is the full restic-compatible string for identifying
// this repository. This field is only used when RepositoryType is "restic".
// Deprecated
// +optional
ResticIdentifier string `json:"resticIdentifier,omitempty"`
@@ -58,8 +57,7 @@ const (
BackupRepositoryPhaseReady BackupRepositoryPhase = "Ready"
BackupRepositoryPhaseNotReady BackupRepositoryPhase = "NotReady"
BackupRepositoryTypeRestic string = "restic"
BackupRepositoryTypeKopia string = "kopia"
BackupRepositoryTypeKopia string = "kopia"
)
// BackupRepositoryStatus is the current status of a BackupRepository.

View File

@@ -43,7 +43,6 @@ import (
"github.com/vmware-tanzu/velero/pkg/constant"
"github.com/vmware-tanzu/velero/pkg/label"
"github.com/vmware-tanzu/velero/pkg/metrics"
repoconfig "github.com/vmware-tanzu/velero/pkg/repository/config"
"github.com/vmware-tanzu/velero/pkg/repository/maintenance"
repomanager "github.com/vmware-tanzu/velero/pkg/repository/manager"
"github.com/vmware-tanzu/velero/pkg/util/kube"
@@ -238,6 +237,10 @@ func (r *BackupRepoReconciler) Reconcile(ctx context.Context, req ctrl.Request)
return ctrl.Result{}, err
}
if backupRepo.Spec.RepositoryType != velerov1api.BackupRepositoryTypeKopia {
return ctrl.Result{}, nil
}
bsl, bslErr := r.getBSL(ctx, backupRepo)
if bslErr != nil {
log.WithError(bslErr).Error("Fail to get BSL for BackupRepository. Skip reconciling.")
@@ -245,7 +248,7 @@ func (r *BackupRepoReconciler) Reconcile(ctx context.Context, req ctrl.Request)
}
if backupRepo.Status.Phase == "" || backupRepo.Status.Phase == velerov1api.BackupRepositoryPhaseNew {
if err := r.initializeRepo(ctx, backupRepo, bsl, log); err != nil {
if err := r.initializeRepo(ctx, backupRepo, log); err != nil {
log.WithError(err).Error("error initialize repository")
return ctrl.Result{}, errors.WithStack(err)
}
@@ -263,7 +266,7 @@ func (r *BackupRepoReconciler) Reconcile(ctx context.Context, req ctrl.Request)
switch backupRepo.Status.Phase {
case velerov1api.BackupRepositoryPhaseNotReady:
ready, err := r.checkNotReadyRepo(ctx, backupRepo, bsl, log)
ready, err := r.checkNotReadyRepo(ctx, backupRepo, log)
if err != nil {
return ctrl.Result{}, err
} else if !ready {
@@ -311,35 +314,9 @@ func (r *BackupRepoReconciler) getBSL(ctx context.Context, req *velerov1api.Back
return loc, nil
}
func (r *BackupRepoReconciler) getIdentifierByBSL(bsl *velerov1api.BackupStorageLocation, req *velerov1api.BackupRepository) (string, error) {
repoIdentifier, err := repoconfig.GetRepoIdentifier(bsl, req.Spec.VolumeNamespace)
if err != nil {
return "", errors.Wrapf(err, "error to get identifier for repo %s", req.Name)
}
return repoIdentifier, nil
}
func (r *BackupRepoReconciler) initializeRepo(ctx context.Context, req *velerov1api.BackupRepository, bsl *velerov1api.BackupStorageLocation, log logrus.FieldLogger) error {
func (r *BackupRepoReconciler) initializeRepo(ctx context.Context, req *velerov1api.BackupRepository, log logrus.FieldLogger) error {
log.WithField("repoConfig", r.backupRepoConfig).Info("Initializing backup repository")
var repoIdentifier string
// Only get restic identifier for restic repositories
if req.Spec.RepositoryType == "" || req.Spec.RepositoryType == velerov1api.BackupRepositoryTypeRestic {
var err error
repoIdentifier, err = r.getIdentifierByBSL(bsl, req)
if err != nil {
return r.patchBackupRepository(ctx, req, func(rr *velerov1api.BackupRepository) {
rr.Status.Message = err.Error()
rr.Status.Phase = velerov1api.BackupRepositoryPhaseNotReady
if rr.Spec.MaintenanceFrequency.Duration <= 0 {
rr.Spec.MaintenanceFrequency = metav1.Duration{Duration: r.getRepositoryMaintenanceFrequency(req)}
}
})
}
}
config, err := getBackupRepositoryConfig(ctx, r, r.backupRepoConfig, r.namespace, req.Name, req.Spec.RepositoryType, log)
if err != nil {
log.WithError(err).Warn("Failed to get repo config, repo config is ignored")
@@ -349,11 +326,6 @@ func (r *BackupRepoReconciler) initializeRepo(ctx context.Context, req *velerov1
// defaulting - if the patch fails, return an error so the item is returned to the queue
if err := r.patchBackupRepository(ctx, req, func(rr *velerov1api.BackupRepository) {
// Only set ResticIdentifier for restic repositories
if rr.Spec.RepositoryType == "" || rr.Spec.RepositoryType == velerov1api.BackupRepositoryTypeRestic {
rr.Spec.ResticIdentifier = repoIdentifier
}
if rr.Spec.MaintenanceFrequency.Duration <= 0 {
rr.Spec.MaintenanceFrequency = metav1.Duration{Duration: r.getRepositoryMaintenanceFrequency(req)}
}
@@ -576,25 +548,9 @@ func dueForMaintenance(req *velerov1api.BackupRepository, now time.Time) bool {
return req.Status.LastMaintenanceTime == nil || req.Status.LastMaintenanceTime.Add(req.Spec.MaintenanceFrequency.Duration).Before(now)
}
func (r *BackupRepoReconciler) checkNotReadyRepo(ctx context.Context, req *velerov1api.BackupRepository, bsl *velerov1api.BackupStorageLocation, log logrus.FieldLogger) (bool, error) {
func (r *BackupRepoReconciler) checkNotReadyRepo(ctx context.Context, req *velerov1api.BackupRepository, log logrus.FieldLogger) (bool, error) {
log.Info("Checking backup repository for readiness")
// Only check and update restic identifier for restic repositories
if req.Spec.RepositoryType == "" || req.Spec.RepositoryType == velerov1api.BackupRepositoryTypeRestic {
repoIdentifier, err := r.getIdentifierByBSL(bsl, req)
if err != nil {
return false, r.patchBackupRepository(ctx, req, repoNotReady(err.Error()))
}
if repoIdentifier != req.Spec.ResticIdentifier {
if err := r.patchBackupRepository(ctx, req, func(rr *velerov1api.BackupRepository) {
rr.Spec.ResticIdentifier = repoIdentifier
}); err != nil {
return false, err
}
}
}
// we need to ensure it (first check, if check fails, attempt to init)
// because we don't know if it's been successfully initialized yet.
if err := ensureRepo(req, r.repositoryManager); err != nil {

View File

@@ -98,32 +98,6 @@ func TestPatchBackupRepository(t *testing.T) {
}
func TestCheckNotReadyRepo(t *testing.T) {
// Test for restic repository
t.Run("restic repository", func(t *testing.T) {
rr := mockBackupRepositoryCR()
rr.Spec.BackupStorageLocation = "default"
rr.Spec.ResticIdentifier = "fake-identifier"
rr.Spec.VolumeNamespace = "volume-ns-1"
rr.Spec.RepositoryType = velerov1api.BackupRepositoryTypeRestic
reconciler := mockBackupRepoReconciler(t, "PrepareRepo", rr, nil)
err := reconciler.Client.Create(t.Context(), rr)
require.NoError(t, err)
location := velerov1api.BackupStorageLocation{
Spec: velerov1api.BackupStorageLocationSpec{
Config: map[string]string{"resticRepoPrefix": "s3:test.amazonaws.com/bucket/restic"},
},
ObjectMeta: metav1.ObjectMeta{
Namespace: velerov1api.DefaultNamespace,
Name: rr.Spec.BackupStorageLocation,
},
}
_, err = reconciler.checkNotReadyRepo(t.Context(), rr, &location, reconciler.logger)
require.NoError(t, err)
assert.Equal(t, velerov1api.BackupRepositoryPhaseReady, rr.Status.Phase)
assert.Equal(t, "s3:test.amazonaws.com/bucket/restic/volume-ns-1", rr.Spec.ResticIdentifier)
})
// Test for kopia repository
t.Run("kopia repository", func(t *testing.T) {
rr := mockBackupRepositoryCR()
@@ -133,48 +107,13 @@ func TestCheckNotReadyRepo(t *testing.T) {
reconciler := mockBackupRepoReconciler(t, "PrepareRepo", rr, nil)
err := reconciler.Client.Create(t.Context(), rr)
require.NoError(t, err)
location := velerov1api.BackupStorageLocation{
Spec: velerov1api.BackupStorageLocationSpec{
Config: map[string]string{"resticRepoPrefix": "s3:test.amazonaws.com/bucket/restic"},
},
ObjectMeta: metav1.ObjectMeta{
Namespace: velerov1api.DefaultNamespace,
Name: rr.Spec.BackupStorageLocation,
},
}
_, err = reconciler.checkNotReadyRepo(t.Context(), rr, &location, reconciler.logger)
_, err = reconciler.checkNotReadyRepo(t.Context(), rr, reconciler.logger)
require.NoError(t, err)
assert.Equal(t, velerov1api.BackupRepositoryPhaseReady, rr.Status.Phase)
// ResticIdentifier should remain empty for kopia
assert.Empty(t, rr.Spec.ResticIdentifier)
})
// Test for empty repository type (defaults to restic)
t.Run("empty repository type", func(t *testing.T) {
rr := mockBackupRepositoryCR()
rr.Spec.BackupStorageLocation = "default"
rr.Spec.ResticIdentifier = "fake-identifier"
rr.Spec.VolumeNamespace = "volume-ns-1"
// Deliberately leave RepositoryType empty
reconciler := mockBackupRepoReconciler(t, "PrepareRepo", rr, nil)
err := reconciler.Client.Create(t.Context(), rr)
require.NoError(t, err)
location := velerov1api.BackupStorageLocation{
Spec: velerov1api.BackupStorageLocationSpec{
Config: map[string]string{"resticRepoPrefix": "s3:test.amazonaws.com/bucket/restic"},
},
ObjectMeta: metav1.ObjectMeta{
Namespace: velerov1api.DefaultNamespace,
Name: rr.Spec.BackupStorageLocation,
},
}
_, err = reconciler.checkNotReadyRepo(t.Context(), rr, &location, reconciler.logger)
require.NoError(t, err)
assert.Equal(t, velerov1api.BackupRepositoryPhaseReady, rr.Status.Phase)
assert.Equal(t, "s3:test.amazonaws.com/bucket/restic/volume-ns-1", rr.Spec.ResticIdentifier)
})
}
func startMaintenanceJobFail(client.Client, context.Context, *velerov1api.BackupRepository, string, logrus.Level, *logging.FormatFlag, logrus.FieldLogger) (string, error) {
@@ -463,17 +402,8 @@ func TestInitializeRepo(t *testing.T) {
reconciler := mockBackupRepoReconciler(t, "PrepareRepo", rr, nil)
err := reconciler.Client.Create(t.Context(), rr)
require.NoError(t, err)
location := velerov1api.BackupStorageLocation{
Spec: velerov1api.BackupStorageLocationSpec{
Config: map[string]string{"resticRepoPrefix": "s3:test.amazonaws.com/bucket/restic"},
},
ObjectMeta: metav1.ObjectMeta{
Namespace: velerov1api.DefaultNamespace,
Name: rr.Spec.BackupStorageLocation,
},
}
err = reconciler.initializeRepo(t.Context(), rr, &location, reconciler.logger)
err = reconciler.initializeRepo(t.Context(), rr, reconciler.logger)
require.NoError(t, err)
assert.Equal(t, velerov1api.BackupRepositoryPhaseReady, rr.Status.Phase)
}
@@ -1494,7 +1424,7 @@ func TestDeleteOldMaintenanceJobWithConfigMap(t *testing.T) {
MaintenanceFrequency: metav1.Duration{Duration: testMaintenanceFrequency},
BackupStorageLocation: "default",
VolumeNamespace: "test-ns",
RepositoryType: "restic",
RepositoryType: "kopia",
},
Status: velerov1api.BackupRepositoryStatus{
Phase: velerov1api.BackupRepositoryPhaseReady,
@@ -1531,7 +1461,7 @@ func TestDeleteOldMaintenanceJobWithConfigMap(t *testing.T) {
MaintenanceFrequency: metav1.Duration{Duration: testMaintenanceFrequency},
BackupStorageLocation: "default",
VolumeNamespace: "test-ns",
RepositoryType: "restic",
RepositoryType: "kopia",
},
Status: velerov1api.BackupRepositoryStatus{
Phase: velerov1api.BackupRepositoryPhaseReady,
@@ -1550,8 +1480,8 @@ func TestDeleteOldMaintenanceJobWithConfigMap(t *testing.T) {
Name: "repo-maintenance-job-config",
},
Data: map[string]string{
"global": `{"keepLatestMaintenanceJobs": 5}`,
"test-ns-default-restic": `{"keepLatestMaintenanceJobs": 2}`,
"global": `{"keepLatestMaintenanceJobs": 5}`,
"test-ns-default-kopia": `{"keepLatestMaintenanceJobs": 2}`,
},
},
},
@@ -1605,58 +1535,6 @@ func TestInitializeRepoWithRepositoryTypes(t *testing.T) {
corev1api.AddToScheme(scheme)
velerov1api.AddToScheme(scheme)
// Test for restic repository
t.Run("restic repository", func(t *testing.T) {
rr := mockBackupRepositoryCR()
rr.Spec.BackupStorageLocation = "default"
rr.Spec.VolumeNamespace = "volume-ns-1"
rr.Spec.RepositoryType = velerov1api.BackupRepositoryTypeRestic
location := &velerov1api.BackupStorageLocation{
ObjectMeta: metav1.ObjectMeta{
Namespace: velerov1api.DefaultNamespace,
Name: "default",
},
Spec: velerov1api.BackupStorageLocationSpec{
Provider: "aws",
StorageType: velerov1api.StorageType{
ObjectStorage: &velerov1api.ObjectStorageLocation{
Bucket: "test-bucket",
Prefix: "test-prefix",
},
},
Config: map[string]string{
"region": "us-east-1",
},
},
}
fakeClient := clientFake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(rr, location).Build()
mgr := &repomokes.Manager{}
mgr.On("PrepareRepo", rr).Return(nil)
reconciler := NewBackupRepoReconciler(
velerov1api.DefaultNamespace,
velerotest.NewLogger(),
fakeClient,
mgr,
testMaintenanceFrequency,
"",
"",
logrus.InfoLevel,
nil,
nil,
)
err := reconciler.initializeRepo(t.Context(), rr, location, reconciler.logger)
require.NoError(t, err)
// Verify ResticIdentifier is set for restic
assert.NotEmpty(t, rr.Spec.ResticIdentifier)
assert.Contains(t, rr.Spec.ResticIdentifier, "volume-ns-1")
assert.Equal(t, velerov1api.BackupRepositoryPhaseReady, rr.Status.Phase)
})
// Test for kopia repository
t.Run("kopia repository", func(t *testing.T) {
rr := mockBackupRepositoryCR()
@@ -1700,65 +1578,13 @@ func TestInitializeRepoWithRepositoryTypes(t *testing.T) {
nil,
)
err := reconciler.initializeRepo(t.Context(), rr, location, reconciler.logger)
err := reconciler.initializeRepo(t.Context(), rr, reconciler.logger)
require.NoError(t, err)
// Verify ResticIdentifier is NOT set for kopia
assert.Empty(t, rr.Spec.ResticIdentifier)
assert.Equal(t, velerov1api.BackupRepositoryPhaseReady, rr.Status.Phase)
})
// Test for empty repository type (defaults to restic)
t.Run("empty repository type", func(t *testing.T) {
rr := mockBackupRepositoryCR()
rr.Spec.BackupStorageLocation = "default"
rr.Spec.VolumeNamespace = "volume-ns-1"
// Leave RepositoryType empty
location := &velerov1api.BackupStorageLocation{
ObjectMeta: metav1.ObjectMeta{
Namespace: velerov1api.DefaultNamespace,
Name: "default",
},
Spec: velerov1api.BackupStorageLocationSpec{
Provider: "aws",
StorageType: velerov1api.StorageType{
ObjectStorage: &velerov1api.ObjectStorageLocation{
Bucket: "test-bucket",
Prefix: "test-prefix",
},
},
Config: map[string]string{
"region": "us-east-1",
},
},
}
fakeClient := clientFake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(rr, location).Build()
mgr := &repomokes.Manager{}
mgr.On("PrepareRepo", rr).Return(nil)
reconciler := NewBackupRepoReconciler(
velerov1api.DefaultNamespace,
velerotest.NewLogger(),
fakeClient,
mgr,
testMaintenanceFrequency,
"",
"",
logrus.InfoLevel,
nil,
nil,
)
err := reconciler.initializeRepo(t.Context(), rr, location, reconciler.logger)
require.NoError(t, err)
// Verify ResticIdentifier is set when type is empty (defaults to restic)
assert.NotEmpty(t, rr.Spec.ResticIdentifier)
assert.Contains(t, rr.Spec.ResticIdentifier, "volume-ns-1")
assert.Equal(t, velerov1api.BackupRepositoryPhaseReady, rr.Status.Phase)
})
}
func TestRepoMaintenanceMetricsRecording(t *testing.T) {

View File

@@ -251,11 +251,9 @@ func (fs *fileSystemBR) boostRepoConnect(ctx context.Context, repositoryType str
if err := repoProvider.NewUnifiedRepoProvider(*credentialGetter, repositoryType, fs.log).BoostRepoConnect(ctx, repoProvider.RepoParam{BackupLocation: fs.backupLocation, BackupRepo: fs.backupRepo, CacheDir: cacheDir}); err != nil {
return err
}
} else {
if err := repoProvider.NewResticRepositoryProvider(*credentialGetter, filesystem.NewFileSystem(), fs.log).BoostRepoConnect(ctx, repoProvider.RepoParam{BackupLocation: fs.backupLocation, BackupRepo: fs.backupRepo}); err != nil {
return err
}
return nil
}
return nil
return errors.Errorf("error getting provider for repo %s", repositoryType)
}

View File

@@ -305,11 +305,6 @@ func (b *backupper) BackupPodVolumes(backup *velerov1api.Backup, pod *corev1api.
}
}
repoIdentifier := ""
if repositoryType == velerov1api.BackupRepositoryTypeRestic {
repoIdentifier = repo.Spec.ResticIdentifier
}
for _, volumeName := range volumesToBackup {
volume, ok := podVolumes[volumeName]
if !ok {
@@ -366,7 +361,7 @@ func (b *backupper) BackupPodVolumes(backup *velerov1api.Backup, pod *corev1api.
continue
}
volumeBackup := newPodVolumeBackup(backup, pod, volume, repoIdentifier, b.uploaderType, pvc)
volumeBackup := newPodVolumeBackup(backup, pod, volume, "", b.uploaderType, pvc)
// the PVB must be added into the indexer before creating it in API server otherwise unexpected behavior may happen:
// the PVB may be handled very quickly by the controller and the informer handler will insert the PVB before "b.pvbIndexer.Add(volumeBackup)" runs,
// this causes the PVB inserted by "b.pvbIndexer.Add(volumeBackup)" overrides the PVB in the indexer while the PVB inserted by "b.pvbIndexer.Add(volumeBackup)"

View File

@@ -166,11 +166,6 @@ func (r *restorer) RestorePodVolumes(data RestoreData, tracker *volume.RestoreVo
podVolumes[podVolume.Name] = podVolume
}
repoIdentifier := ""
if repositoryType == velerov1api.BackupRepositoryTypeRestic {
repoIdentifier = repo.Spec.ResticIdentifier
}
for volume, backupInfo := range volumesToRestore {
volumeObj, ok := podVolumes[volume]
var pvc *corev1api.PersistentVolumeClaim
@@ -185,7 +180,7 @@ func (r *restorer) RestorePodVolumes(data RestoreData, tracker *volume.RestoreVo
}
}
volumeRestore := newPodVolumeRestore(data.Restore, data.Pod, data.BackupLocation, volume, backupInfo.snapshotID, backupInfo.snapshotSize, repoIdentifier, backupInfo.uploaderType, data.SourceNamespace, pvc)
volumeRestore := newPodVolumeRestore(data.Restore, data.Pod, data.BackupLocation, volume, backupInfo.snapshotID, backupInfo.snapshotSize, "", backupInfo.uploaderType, data.SourceNamespace, pvc)
if err := veleroclient.CreateRetryGenerateName(r.crClient, r.ctx, volumeRestore); err != nil {
errs = append(errs, errors.WithStack(err))
continue

View File

@@ -167,7 +167,7 @@ func getUploaderTypeOrDefault(uploaderType string) string {
return uploader.KopiaType
}
// getRepositoryType returns the hardcode repositoryType
// getRepositoryType returns the hardcode repositoryType.
// TODO: In future, when we have multiple implementations of Unified Repo (besides Kopia), we will add the repositoryType to BSL,
// because by then, we are not able to hardcode the repositoryType to BackupRepositoryTypeKopia for Unified Repo.
func getRepositoryType() string {

View File

@@ -17,14 +17,7 @@ limitations under the License.
package config
import (
"fmt"
"path"
"strings"
"github.com/pkg/errors"
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
"github.com/vmware-tanzu/velero/pkg/persistence"
)
type BackendType string
@@ -40,56 +33,6 @@ const (
CredentialsFileKey = "credentialsFile"
)
// this func is assigned to a package-level variable so it can be
// replaced when unit-testing
var getAWSBucketRegion = GetAWSBucketRegion
// getRepoPrefix returns the prefix of the value of the --repo flag for
// restic commands, i.e. everything except the "/<repo-name>".
func getRepoPrefix(location *velerov1api.BackupStorageLocation) (string, error) {
var bucket, prefix string
if location.Spec.ObjectStorage != nil {
layout := persistence.NewObjectStoreLayout(location.Spec.ObjectStorage.Prefix)
bucket = location.Spec.ObjectStorage.Bucket
prefix = layout.GetResticDir()
}
backendType := GetBackendType(location.Spec.Provider, location.Spec.Config)
if repoPrefix := location.Spec.Config["resticRepoPrefix"]; repoPrefix != "" {
return repoPrefix, nil
}
switch backendType {
case AWSBackend:
var url string
// non-AWS, S3-compatible object store
if s3Url := location.Spec.Config["s3Url"]; s3Url != "" {
url = strings.TrimSuffix(s3Url, "/")
} else {
var err error
region := location.Spec.Config["region"]
if region == "" {
region, err = getAWSBucketRegion(bucket, location.Spec.Config)
}
if err != nil {
return "", errors.Wrapf(err, "failed to detect the region via bucket: %s", bucket)
}
url = fmt.Sprintf("s3-%s.amazonaws.com", region)
}
return fmt.Sprintf("s3:%s/%s", url, path.Join(bucket, prefix)), nil
case AzureBackend:
return fmt.Sprintf("azure:%s:/%s", bucket, prefix), nil
case GCPBackend:
return fmt.Sprintf("gs:%s:/%s", bucket, prefix), nil
}
return "", errors.Errorf("invalid backend type %s, provider %s", backendType, location.Spec.Provider)
}
// GetBackendType returns a backend type that is known by Velero.
// If the provider doesn't indicate a known backend type, but the endpoint is
// specified, Velero regards it as a S3 compatible object store and return AWSBackend as the type.
@@ -111,14 +54,3 @@ func GetBackendType(provider string, config map[string]string) BackendType {
func IsBackendTypeValid(backendType BackendType) bool {
return (backendType == AWSBackend || backendType == AzureBackend || backendType == GCPBackend || backendType == FSBackend)
}
// GetRepoIdentifier returns the string to be used as the value of the --repo flag in
// restic commands for the given repository.
func GetRepoIdentifier(location *velerov1api.BackupStorageLocation, name string) (string, error) {
prefix, err := getRepoPrefix(location)
if err != nil {
return "", err
}
return fmt.Sprintf("%s/%s", strings.TrimSuffix(prefix, "/"), name), nil
}

View File

@@ -1,261 +0,0 @@
/*
Copyright 2018, 2019 the Velero contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package config
import (
"testing"
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
)
func TestGetRepoIdentifier(t *testing.T) {
testCases := []struct {
name string
bsl *velerov1api.BackupStorageLocation
repoName string
getAWSBucketRegion func(s string, config map[string]string) (string, error)
expected string
expectedErr string
}{
{
name: "error is returned if BSL uses unsupported provider and resticRepoPrefix is not set",
bsl: &velerov1api.BackupStorageLocation{
Spec: velerov1api.BackupStorageLocationSpec{
Provider: "unsupported-provider",
StorageType: velerov1api.StorageType{
ObjectStorage: &velerov1api.ObjectStorageLocation{
Bucket: "bucket-2",
Prefix: "prefix-2",
},
},
},
},
repoName: "repo-1",
expectedErr: "invalid backend type velero.io/unsupported-provider, provider unsupported-provider",
},
{
name: "resticRepoPrefix in BSL config is used if set",
bsl: &velerov1api.BackupStorageLocation{
Spec: velerov1api.BackupStorageLocationSpec{
Provider: "custom-repo-identifier",
Config: map[string]string{
"resticRepoPrefix": "custom:prefix:/restic",
},
StorageType: velerov1api.StorageType{
ObjectStorage: &velerov1api.ObjectStorageLocation{
Bucket: "bucket",
Prefix: "prefix",
},
},
},
},
repoName: "repo-1",
expected: "custom:prefix:/restic/repo-1",
},
{
name: "s3Url in BSL config is used",
bsl: &velerov1api.BackupStorageLocation{
Spec: velerov1api.BackupStorageLocationSpec{
Provider: "custom-repo-identifier",
Config: map[string]string{
"s3Url": "s3Url",
},
StorageType: velerov1api.StorageType{
ObjectStorage: &velerov1api.ObjectStorageLocation{
Bucket: "bucket",
Prefix: "prefix",
},
},
},
},
repoName: "repo-1",
expected: "s3:s3Url/bucket/prefix/restic/repo-1",
},
{
name: "s3.amazonaws.com URL format is used if region cannot be determined for AWS BSL",
bsl: &velerov1api.BackupStorageLocation{
Spec: velerov1api.BackupStorageLocationSpec{
Provider: "aws",
StorageType: velerov1api.StorageType{
ObjectStorage: &velerov1api.ObjectStorageLocation{
Bucket: "bucket",
},
},
},
},
repoName: "repo-1",
getAWSBucketRegion: func(s string, config map[string]string) (string, error) {
return "", errors.New("no region found")
},
expected: "",
expectedErr: "failed to detect the region via bucket: bucket: no region found",
},
{
name: "s3.s3-<region>.amazonaws.com URL format is used if region can be determined for AWS BSL",
bsl: &velerov1api.BackupStorageLocation{
Spec: velerov1api.BackupStorageLocationSpec{
Provider: "aws",
StorageType: velerov1api.StorageType{
ObjectStorage: &velerov1api.ObjectStorageLocation{
Bucket: "bucket",
},
},
},
},
repoName: "repo-1",
getAWSBucketRegion: func(string, map[string]string) (string, error) {
return "eu-west-1", nil
},
expected: "s3:s3-eu-west-1.amazonaws.com/bucket/restic/repo-1",
},
{
name: "prefix is included in repo identifier if set for AWS BSL",
bsl: &velerov1api.BackupStorageLocation{
Spec: velerov1api.BackupStorageLocationSpec{
Provider: "aws",
StorageType: velerov1api.StorageType{
ObjectStorage: &velerov1api.ObjectStorageLocation{
Bucket: "bucket",
Prefix: "prefix",
},
},
},
},
repoName: "repo-1",
getAWSBucketRegion: func(s string, config map[string]string) (string, error) {
return "eu-west-1", nil
},
expected: "s3:s3-eu-west-1.amazonaws.com/bucket/prefix/restic/repo-1",
},
{
name: "s3Url is used in repo identifier if set for AWS BSL",
bsl: &velerov1api.BackupStorageLocation{
Spec: velerov1api.BackupStorageLocationSpec{
Provider: "aws",
Config: map[string]string{
"s3Url": "alternate-url",
},
StorageType: velerov1api.StorageType{
ObjectStorage: &velerov1api.ObjectStorageLocation{
Bucket: "bucket",
Prefix: "prefix",
},
},
},
},
repoName: "repo-1",
getAWSBucketRegion: func(s string, config map[string]string) (string, error) {
return "eu-west-1", nil
},
expected: "s3:alternate-url/bucket/prefix/restic/repo-1",
},
{
name: "region is used in repo identifier if set for AWS BSL",
bsl: &velerov1api.BackupStorageLocation{
Spec: velerov1api.BackupStorageLocationSpec{
Provider: "aws",
Config: map[string]string{
"region": "us-west-1",
},
StorageType: velerov1api.StorageType{
ObjectStorage: &velerov1api.ObjectStorageLocation{
Bucket: "bucket",
Prefix: "prefix",
},
},
},
},
repoName: "aws-repo",
getAWSBucketRegion: func(s string, config map[string]string) (string, error) {
return "eu-west-1", nil
},
expected: "s3:s3-us-west-1.amazonaws.com/bucket/prefix/restic/aws-repo",
},
{
name: "trailing slash in s3Url is not included in repo identifier for AWS BSL",
bsl: &velerov1api.BackupStorageLocation{
Spec: velerov1api.BackupStorageLocationSpec{
Provider: "aws",
Config: map[string]string{
"s3Url": "alternate-url-with-trailing-slash/",
},
StorageType: velerov1api.StorageType{
ObjectStorage: &velerov1api.ObjectStorageLocation{
Bucket: "bucket",
Prefix: "prefix",
},
},
},
},
repoName: "aws-repo",
getAWSBucketRegion: func(s string, config map[string]string) (string, error) {
return "eu-west-1", nil
},
expected: "s3:alternate-url-with-trailing-slash/bucket/prefix/restic/aws-repo",
},
{
name: "repo identifier includes bucket and prefix for Azure BSL",
bsl: &velerov1api.BackupStorageLocation{
Spec: velerov1api.BackupStorageLocationSpec{
Provider: "azure",
StorageType: velerov1api.StorageType{
ObjectStorage: &velerov1api.ObjectStorageLocation{
Bucket: "azure-bucket",
Prefix: "azure-prefix",
},
},
},
},
repoName: "azure-repo",
expected: "azure:azure-bucket:/azure-prefix/restic/azure-repo",
},
{
name: "repo identifier includes bucket and prefix for GCP BSL",
bsl: &velerov1api.BackupStorageLocation{
Spec: velerov1api.BackupStorageLocationSpec{
Provider: "gcp",
StorageType: velerov1api.StorageType{
ObjectStorage: &velerov1api.ObjectStorageLocation{
Bucket: "gcp-bucket",
Prefix: "gcp-prefix",
},
},
},
},
repoName: "gcp-repo",
expected: "gs:gcp-bucket:/gcp-prefix/restic/gcp-repo",
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
getAWSBucketRegion = tc.getAWSBucketRegion
id, err := GetRepoIdentifier(tc.bsl, tc.repoName)
assert.Equal(t, tc.expected, id)
if tc.expectedErr == "" {
assert.NoError(t, err)
} else {
require.EqualError(t, err, tc.expectedErr)
assert.Empty(t, id)
}
})
}
}

View File

@@ -109,10 +109,6 @@ func NewManager(
log: log,
}
mgr.providers[velerov1api.BackupRepositoryTypeRestic] = provider.NewResticRepositoryProvider(credentials.CredentialGetter{
FromFile: credentialFileStore,
FromSecret: credentialSecretStore,
}, mgr.fileSystem, mgr.log)
mgr.providers[velerov1api.BackupRepositoryTypeKopia] = provider.NewUnifiedRepoProvider(credentials.CredentialGetter{
FromFile: credentialFileStore,
FromSecret: credentialSecretStore,
@@ -275,8 +271,6 @@ func (m *manager) ClientSideCacheLimit(repo *velerov1api.BackupRepository) (int6
func (m *manager) getRepositoryProvider(repo *velerov1api.BackupRepository) (provider.Provider, error) {
switch repo.Spec.RepositoryType {
case "", velerov1api.BackupRepositoryTypeRestic:
return m.providers[velerov1api.BackupRepositoryTypeRestic], nil
case velerov1api.BackupRepositoryTypeKopia:
return m.providers[velerov1api.BackupRepositoryTypeKopia], nil
default:

View File

@@ -32,15 +32,13 @@ func TestGetRepositoryProvider(t *testing.T) {
repo := &velerov1.BackupRepository{}
// empty repository type
provider, err := mgr.getRepositoryProvider(repo)
require.NoError(t, err)
assert.NotNil(t, provider)
_, err := mgr.getRepositoryProvider(repo)
require.Error(t, err)
// valid repository type
repo.Spec.RepositoryType = velerov1.BackupRepositoryTypeRestic
provider, err = mgr.getRepositoryProvider(repo)
require.NoError(t, err)
assert.NotNil(t, provider)
// invalid repository type
repo.Spec.RepositoryType = "restic"
_, err = mgr.getRepositoryProvider(repo)
require.Error(t, err)
// invalid repository type
repo.Spec.RepositoryType = "unknown"
@@ -61,6 +59,6 @@ func TestGetRepositoryConfigProvider(t *testing.T) {
assert.NotNil(t, provider)
// invalid repository type
_, err = mgr.getRepositoryProvider(velerov1.BackupRepositoryTypeRestic)
_, err = mgr.getRepositoryProvider("restic")
require.Error(t, err)
}

View File

@@ -1,99 +0,0 @@
/*
Copyright the Velero contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package provider
import (
"context"
"strings"
"time"
"github.com/sirupsen/logrus"
"github.com/vmware-tanzu/velero/internal/credentials"
"github.com/vmware-tanzu/velero/pkg/repository/restic"
"github.com/vmware-tanzu/velero/pkg/util/filesystem"
)
func NewResticRepositoryProvider(credGetter credentials.CredentialGetter, fs filesystem.Interface, log logrus.FieldLogger) Provider {
return &resticRepositoryProvider{
svc: restic.NewRepositoryService(credGetter, fs, log),
}
}
type resticRepositoryProvider struct {
svc *restic.RepositoryService
}
func (r *resticRepositoryProvider) InitRepo(ctx context.Context, param RepoParam) error {
return r.svc.InitRepo(param.BackupLocation, param.BackupRepo)
}
func (r *resticRepositoryProvider) ConnectToRepo(ctx context.Context, param RepoParam) error {
return r.svc.ConnectToRepo(param.BackupLocation, param.BackupRepo)
}
func (r *resticRepositoryProvider) PrepareRepo(ctx context.Context, param RepoParam) error {
if err := r.ConnectToRepo(ctx, param); err != nil {
// If the repository has not yet been initialized, the error message will always include
// the following string. This is the only scenario where we should try to initialize it.
// Other errors (e.g. "already locked") should be returned as-is since the repository
// does already exist, but it can't be connected to.
if strings.Contains(err.Error(), "Is there a repository at the following location?") {
return r.InitRepo(ctx, param)
}
return err
}
return nil
}
func (r *resticRepositoryProvider) BoostRepoConnect(ctx context.Context, param RepoParam) error {
return nil
}
func (r *resticRepositoryProvider) PruneRepo(ctx context.Context, param RepoParam) error {
return r.svc.PruneRepo(param.BackupLocation, param.BackupRepo)
}
func (r *resticRepositoryProvider) EnsureUnlockRepo(ctx context.Context, param RepoParam) error {
return r.svc.UnlockRepo(param.BackupLocation, param.BackupRepo)
}
func (r *resticRepositoryProvider) Forget(ctx context.Context, snapshotID string, param RepoParam) error {
return r.svc.Forget(param.BackupLocation, param.BackupRepo, snapshotID)
}
func (r *resticRepositoryProvider) BatchForget(ctx context.Context, snapshotIDs []string, param RepoParam) []error {
errs := []error{}
for _, snapshot := range snapshotIDs {
err := r.Forget(ctx, snapshot, param)
if err != nil {
errs = append(errs, err)
}
}
return errs
}
func (r *resticRepositoryProvider) DefaultMaintenanceFrequency() time.Duration {
return r.svc.DefaultMaintenanceFrequency()
}
func (r *resticRepositoryProvider) ClientSideCacheLimit(repoOption map[string]string) int64 {
return 0
}

View File

@@ -1,147 +0,0 @@
/*
Copyright the Velero contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package restic
import (
"os"
"time"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/vmware-tanzu/velero/internal/credentials"
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
repokey "github.com/vmware-tanzu/velero/pkg/repository/keys"
"github.com/vmware-tanzu/velero/pkg/restic"
veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec"
"github.com/vmware-tanzu/velero/pkg/util/filesystem"
)
func NewRepositoryService(credGetter credentials.CredentialGetter, fs filesystem.Interface, log logrus.FieldLogger) *RepositoryService {
return &RepositoryService{
credGetter: credGetter,
fileSystem: fs,
log: log,
}
}
type RepositoryService struct {
credGetter credentials.CredentialGetter
fileSystem filesystem.Interface
log logrus.FieldLogger
}
func (r *RepositoryService) InitRepo(bsl *velerov1api.BackupStorageLocation, repo *velerov1api.BackupRepository) error {
return r.exec(restic.InitCommand(repo.Spec.ResticIdentifier), bsl)
}
func (r *RepositoryService) ConnectToRepo(bsl *velerov1api.BackupStorageLocation, repo *velerov1api.BackupRepository) error {
snapshotsCmd := restic.SnapshotsCommand(repo.Spec.ResticIdentifier)
// use the '--latest=1' flag to minimize the amount of data fetched since
// we're just validating that the repo exists and can be authenticated
// to.
// "--last" is replaced by "--latest=1" in restic v0.12.1
snapshotsCmd.ExtraFlags = append(snapshotsCmd.ExtraFlags, "--latest=1")
return r.exec(snapshotsCmd, bsl)
}
func (r *RepositoryService) PruneRepo(bsl *velerov1api.BackupStorageLocation, repo *velerov1api.BackupRepository) error {
return r.exec(restic.PruneCommand(repo.Spec.ResticIdentifier), bsl)
}
func (r *RepositoryService) UnlockRepo(bsl *velerov1api.BackupStorageLocation, repo *velerov1api.BackupRepository) error {
return r.exec(restic.UnlockCommand(repo.Spec.ResticIdentifier), bsl)
}
func (r *RepositoryService) Forget(bsl *velerov1api.BackupStorageLocation, repo *velerov1api.BackupRepository, snapshotID string) error {
return r.exec(restic.ForgetCommand(repo.Spec.ResticIdentifier, snapshotID), bsl)
}
func (r *RepositoryService) DefaultMaintenanceFrequency() time.Duration {
return restic.DefaultMaintenanceFrequency
}
func (r *RepositoryService) exec(cmd *restic.Command, bsl *velerov1api.BackupStorageLocation) error {
file, err := r.credGetter.FromFile.Path(repokey.RepoKeySelector())
if err != nil {
return err
}
// ignore error since there's nothing we can do and it's a temp file.
defer os.Remove(file)
cmd.PasswordFile = file
// if there's a caCert on the ObjectStorage, write it to disk so that it can be passed to restic
var caCertFile string
if bsl.Spec.ObjectStorage != nil {
var caCertData []byte
// Try CACertRef first (new method), then fall back to CACert (deprecated)
if bsl.Spec.ObjectStorage.CACertRef != nil {
caCertString, err := r.credGetter.FromSecret.Get(bsl.Spec.ObjectStorage.CACertRef)
if err != nil {
return errors.Wrap(err, "error getting CA certificate from secret")
}
caCertData = []byte(caCertString)
} else if bsl.Spec.ObjectStorage.CACert != nil {
caCertData = bsl.Spec.ObjectStorage.CACert
}
if caCertData != nil {
caCertFile, err = restic.TempCACertFile(caCertData, bsl.Name, r.fileSystem)
if err != nil {
return errors.Wrap(err, "error creating temp cacert file")
}
// ignore error since there's nothing we can do and it's a temp file.
defer os.Remove(caCertFile)
}
}
cmd.CACertFile = caCertFile
// CmdEnv uses credGetter.FromFile (not FromSecret) to get cloud provider credentials.
// FromFile materializes the BSL's Credential secret to a file path that cloud SDKs
// can read (e.g., AWS_SHARED_CREDENTIALS_FILE). This is different from caCertRef above,
// which uses FromSecret to read the CA certificate data directly into memory, then
// writes it to a temp file because restic CLI only accepts file paths (--cacert flag).
env, err := restic.CmdEnv(bsl, r.credGetter.FromFile)
if err != nil {
return err
}
cmd.Env = env
// #4820: restrieve insecureSkipTLSVerify from BSL configuration for
// AWS plugin. If nothing is return, that means insecureSkipTLSVerify
// is not enable for Restic command.
skipTLSRet := restic.GetInsecureSkipTLSVerifyFromBSL(bsl, r.log)
if len(skipTLSRet) > 0 {
cmd.ExtraFlags = append(cmd.ExtraFlags, skipTLSRet)
}
stdout, stderr, err := veleroexec.RunCommandWithLog(cmd.Cmd(), r.log)
r.log.WithFields(logrus.Fields{
"repository": cmd.RepoName(),
"command": cmd.String(),
"stdout": stdout,
"stderr": stderr,
}).Debugf("Ran restic command")
if err != nil {
return errors.Wrapf(err, "error running command=%s, stdout=%s, stderr=%s", cmd.String(), stdout, stderr)
}
return nil
}