Merge branch 'main' into kopia-repo-snapshot-operations

This commit is contained in:
Lyndon-Li
2026-05-21 13:27:40 +08:00
76 changed files with 3191 additions and 1008 deletions
+27 -13
View File
@@ -227,33 +227,43 @@ func TestExecute(t *testing.T) {
pvcMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&tc.pvc)
require.NoError(t, err)
var reconcileErrCh chan error
if tc.pvc != nil && !tc.failVSCreate && !tc.skipVSReadyUpdate {
reconcileErrCh = make(chan error, 1)
go func() {
var vsList snapshotv1api.VolumeSnapshotList
err := wait.PollUntilContextTimeout(t.Context(), 1*time.Second, 10*time.Second, true, func(ctx context.Context) (bool, error) {
err = pvcBIA.crClient.List(ctx, &vsList, &crclient.ListOptions{Namespace: tc.pvc.Namespace})
require.NoError(t, err)
if err != nil || len(vsList.Items) == 0 {
if err := pvcBIA.crClient.List(ctx, &vsList, &crclient.ListOptions{Namespace: tc.pvc.Namespace}); err != nil {
return false, err
}
if len(vsList.Items) == 0 {
return false, nil
}
return true, nil
})
require.NoError(t, err)
if err != nil {
reconcileErrCh <- err
return
}
vscName := "testVSC"
handleName := "testHandle"
vsc := builder.ForVolumeSnapshotContent("testVSC").Status(&snapshotv1api.VolumeSnapshotContentStatus{SnapshotHandle: &handleName}).Result()
err = pvcBIA.crClient.Create(t.Context(), vsc)
if err != nil {
reconcileErrCh <- err
return
}
readyToUse := true
vsList.Items[0].Status = &snapshotv1api.VolumeSnapshotStatus{
BoundVolumeSnapshotContentName: &vscName,
ReadyToUse: &readyToUse,
}
err = pvcBIA.crClient.Update(t.Context(), &vsList.Items[0])
require.NoError(t, err)
handleName := "testHandle"
vsc := builder.ForVolumeSnapshotContent("testVSC").Status(&snapshotv1api.VolumeSnapshotContentStatus{SnapshotHandle: &handleName}).Result()
err = pvcBIA.crClient.Create(t.Context(), vsc)
require.NoError(t, err)
reconcileErrCh <- err
}()
}
@@ -274,12 +284,16 @@ func TestExecute(t *testing.T) {
require.NoError(t, err)
}
if reconcileErrCh != nil {
require.NoError(t, <-reconcileErrCh)
}
if tc.expectedDataUpload != nil {
dataUploadList := new(velerov2alpha1.DataUploadList)
err := crClient.List(t.Context(), dataUploadList, &crclient.ListOptions{LabelSelector: labels.SelectorFromSet(map[string]string{velerov1api.BackupNameLabel: tc.backup.Name})})
require.NoError(t, err)
require.Len(t, dataUploadList.Items, 1)
require.True(t, cmp.Equal(tc.expectedDataUpload, &dataUploadList.Items[0], cmpopts.IgnoreFields(velerov2alpha1.DataUpload{}, "ResourceVersion", "Name", "Spec.CSISnapshot.VolumeSnapshot")))
require.Empty(t, cmp.Diff(tc.expectedDataUpload, &dataUploadList.Items[0], cmpopts.IgnoreFields(velerov2alpha1.DataUpload{}, "TypeMeta", "ResourceVersion", "Name", "Spec.CSISnapshot.VolumeSnapshot")))
}
if tc.expectedPVC != nil {
@@ -488,7 +502,7 @@ func TestCancel(t *testing.T) {
err = crClient.Get(t.Context(), crclient.ObjectKey{Namespace: tc.dataUpload.Namespace, Name: tc.dataUpload.Name}, du)
require.NoError(t, err)
require.True(t, cmp.Equal(tc.expectedDataUpload, *du, cmpopts.IgnoreFields(velerov2alpha1.DataUpload{}, "ResourceVersion")))
require.Empty(t, cmp.Diff(tc.expectedDataUpload, *du, cmpopts.IgnoreFields(velerov2alpha1.DataUpload{}, "TypeMeta", "ResourceVersion")))
}
})
}
+30 -11
View File
@@ -167,15 +167,15 @@ func NewKubernetesBackupper(
}, nil
}
// getNamespaceIncludesExcludesAndArgoCDNamespaces returns an IncludesExcludes list containing which namespaces to
// include and exclude from the backup and a list of namespaces managed by ArgoCD.
func getNamespaceIncludesExcludesAndArgoCDNamespaces(backup *velerov1api.Backup, kbClient kbclient.Client) (*collections.NamespaceIncludesExcludes, []string, error) {
// getNamespaceIncludesExcludes returns an IncludesExcludes list containing which namespaces to
// include and exclude from the backup.
func getNamespaceIncludesExcludes(backup *velerov1api.Backup, kbClient kbclient.Client) (*collections.NamespaceIncludesExcludes, error) {
nsList := corev1api.NamespaceList{}
activeNamespaces := []string{}
nsManagedByArgoCD := []string{}
if err := kbClient.List(context.Background(), &nsList); err != nil {
return nil, nsManagedByArgoCD, err
return nil, err
}
activeNamespaces := []string{}
for _, ns := range nsList.Items {
activeNamespaces = append(activeNamespaces, ns.Name)
}
@@ -188,10 +188,20 @@ func getNamespaceIncludesExcludesAndArgoCDNamespaces(backup *velerov1api.Backup,
// Expand wildcards if needed
if err := includesExcludes.ExpandIncludesExcludes(); err != nil {
return nil, []string{}, err
return nil, err
}
// Check for ArgoCD managed namespaces in the namespaces that will be included
return includesExcludes, nil
}
// getArgoCDManagedNamespaces returns a list of namespaces managed by ArgoCD that should be included in the backup.
func getArgoCDManagedNamespaces(kbClient kbclient.Client, includesExcludes *collections.NamespaceIncludesExcludes) ([]string, error) {
nsList := corev1api.NamespaceList{}
if err := kbClient.List(context.Background(), &nsList); err != nil {
return nil, err
}
nsManagedByArgoCD := []string{}
for _, ns := range nsList.Items {
nsLabels := ns.GetLabels()
if len(nsLabels[ArgoCDManagedByNamespaceLabel]) > 0 && includesExcludes.ShouldInclude(ns.Name) {
@@ -199,7 +209,7 @@ func getNamespaceIncludesExcludesAndArgoCDNamespaces(backup *velerov1api.Backup,
}
}
return includesExcludes, nsManagedByArgoCD, nil
return nsManagedByArgoCD, nil
}
func getResourceHooks(hookSpecs []velerov1api.BackupResourceHookSpec, discoveryHelper discovery.Helper) ([]hook.ResourceHook, error) {
@@ -274,13 +284,18 @@ func (kb *kubernetesBackupper) BackupWithResolvers(
return errors.WithStack(err)
}
var err error
var nsManagedByArgoCD []string
backupRequest.NamespaceIncludesExcludes, nsManagedByArgoCD, err = getNamespaceIncludesExcludesAndArgoCDNamespaces(backupRequest.Backup, kb.kbClient)
backupRequest.NamespaceIncludesExcludes, err = getNamespaceIncludesExcludes(backupRequest.Backup, kb.kbClient)
if err != nil {
log.WithError(err).Errorf("error getting namespace includes/excludes")
return err
}
nsManagedByArgoCD, err := getArgoCDManagedNamespaces(kb.kbClient, backupRequest.NamespaceIncludesExcludes)
if err != nil {
log.WithError(err).Errorf("error getting ArgoCD managed namespaces")
return err
}
if backupRequest.NamespaceIncludesExcludes.IsWildcardExpanded() {
expandedIncludes := backupRequest.NamespaceIncludesExcludes.GetIncludes()
expandedExcludes := backupRequest.NamespaceIncludesExcludes.GetExcludes()
@@ -292,6 +307,10 @@ func (kb *kubernetesBackupper) BackupWithResolvers(
return err
}
if len(wildcardResult) == 0 {
log.Warnf("no namespaces matched the resolution of wildcard patterns ")
}
log.WithFields(logrus.Fields{
"expandedIncludes": expandedIncludes,
"expandedExcludes": expandedExcludes,
+171
View File
@@ -0,0 +1,171 @@
// Code generated by mockery; DO NOT EDIT.
// github.com/vektra/mockery
// template: testify
package mocks
import (
"context"
mock "github.com/stretchr/testify/mock"
"github.com/vmware-tanzu/velero/pkg/cbtservice"
)
// NewService creates a new instance of Service. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func NewService(t interface {
mock.TestingT
Cleanup(func())
}) *Service {
mock := &Service{}
mock.Mock.Test(t)
t.Cleanup(func() { mock.AssertExpectations(t) })
return mock
}
// Service is an autogenerated mock type for the Service type
type Service struct {
mock.Mock
}
type Service_Expecter struct {
mock *mock.Mock
}
func (_m *Service) EXPECT() *Service_Expecter {
return &Service_Expecter{mock: &_m.Mock}
}
// GetAllocatedBlocks provides a mock function for the type Service
func (_mock *Service) GetAllocatedBlocks(ctx context.Context, snapshot string, record func([]cbtservice.Range) error) error {
ret := _mock.Called(ctx, snapshot, record)
if len(ret) == 0 {
panic("no return value specified for GetAllocatedBlocks")
}
var r0 error
if returnFunc, ok := ret.Get(0).(func(context.Context, string, func([]cbtservice.Range) error) error); ok {
r0 = returnFunc(ctx, snapshot, record)
} else {
r0 = ret.Error(0)
}
return r0
}
// Service_GetAllocatedBlocks_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetAllocatedBlocks'
type Service_GetAllocatedBlocks_Call struct {
*mock.Call
}
// GetAllocatedBlocks is a helper method to define mock.On call
// - ctx context.Context
// - snapshot string
// - record func([]cbtservice.Range) error
func (_e *Service_Expecter) GetAllocatedBlocks(ctx interface{}, snapshot interface{}, record interface{}) *Service_GetAllocatedBlocks_Call {
return &Service_GetAllocatedBlocks_Call{Call: _e.mock.On("GetAllocatedBlocks", ctx, snapshot, record)}
}
func (_c *Service_GetAllocatedBlocks_Call) Run(run func(ctx context.Context, snapshot string, record func([]cbtservice.Range) error)) *Service_GetAllocatedBlocks_Call {
_c.Call.Run(func(args mock.Arguments) {
var arg0 context.Context
if args[0] != nil {
arg0 = args[0].(context.Context)
}
var arg1 string
if args[1] != nil {
arg1 = args[1].(string)
}
var arg2 func([]cbtservice.Range) error
if args[2] != nil {
arg2 = args[2].(func([]cbtservice.Range) error)
}
run(
arg0,
arg1,
arg2,
)
})
return _c
}
func (_c *Service_GetAllocatedBlocks_Call) Return(err error) *Service_GetAllocatedBlocks_Call {
_c.Call.Return(err)
return _c
}
func (_c *Service_GetAllocatedBlocks_Call) RunAndReturn(run func(ctx context.Context, snapshot string, record func([]cbtservice.Range) error) error) *Service_GetAllocatedBlocks_Call {
_c.Call.Return(run)
return _c
}
// GetChangedBlocks provides a mock function for the type Service
func (_mock *Service) GetChangedBlocks(ctx context.Context, snapshot string, changeID string, record func([]cbtservice.Range) error) error {
ret := _mock.Called(ctx, snapshot, changeID, record)
if len(ret) == 0 {
panic("no return value specified for GetChangedBlocks")
}
var r0 error
if returnFunc, ok := ret.Get(0).(func(context.Context, string, string, func([]cbtservice.Range) error) error); ok {
r0 = returnFunc(ctx, snapshot, changeID, record)
} else {
r0 = ret.Error(0)
}
return r0
}
// Service_GetChangedBlocks_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetChangedBlocks'
type Service_GetChangedBlocks_Call struct {
*mock.Call
}
// GetChangedBlocks is a helper method to define mock.On call
// - ctx context.Context
// - snapshot string
// - changeID string
// - record func([]cbtservice.Range) error
func (_e *Service_Expecter) GetChangedBlocks(ctx interface{}, snapshot interface{}, changeID interface{}, record interface{}) *Service_GetChangedBlocks_Call {
return &Service_GetChangedBlocks_Call{Call: _e.mock.On("GetChangedBlocks", ctx, snapshot, changeID, record)}
}
func (_c *Service_GetChangedBlocks_Call) Run(run func(ctx context.Context, snapshot string, changeID string, record func([]cbtservice.Range) error)) *Service_GetChangedBlocks_Call {
_c.Call.Run(func(args mock.Arguments) {
var arg0 context.Context
if args[0] != nil {
arg0 = args[0].(context.Context)
}
var arg1 string
if args[1] != nil {
arg1 = args[1].(string)
}
var arg2 string
if args[2] != nil {
arg2 = args[2].(string)
}
var arg3 func([]cbtservice.Range) error
if args[3] != nil {
arg3 = args[3].(func([]cbtservice.Range) error)
}
run(
arg0,
arg1,
arg2,
arg3,
)
})
return _c
}
func (_c *Service_GetChangedBlocks_Call) Return(err error) *Service_GetChangedBlocks_Call {
_c.Call.Return(err)
return _c
}
func (_c *Service_GetChangedBlocks_Call) RunAndReturn(run func(ctx context.Context, snapshot string, changeID string, record func([]cbtservice.Range) error) error) *Service_GetChangedBlocks_Call {
_c.Call.Return(run)
return _c
}
+2 -2
View File
@@ -20,8 +20,8 @@ import "context"
// Range defines the range of a change
type Range struct {
Offset int64
Length int64
Offset uint64
Length uint64
}
// SourceInfo is the information provided to the uploader, the uploader calls CBT service with this information
-48
View File
@@ -37,7 +37,6 @@ import (
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/client-go/kubernetes"
cacheutil "k8s.io/client-go/tools/cache"
@@ -430,10 +429,6 @@ func (s *nodeAgentServer) run() {
s.logger.WithError(err).Fatal("Unable to create the pod volume restore controller")
}
if err := controller.InitLegacyPodVolumeRestoreReconciler(s.mgr.GetClient(), s.mgr, s.kubeClient, s.dataPathMgr, s.namespace, s.config.resourceTimeout, s.logger); err != nil {
s.logger.WithError(err).Fatal("Unable to create the legacy pod volume restore controller")
}
dataUploadReconciler := controller.NewDataUploadReconciler(
s.mgr.GetClient(),
s.mgr,
@@ -509,8 +504,6 @@ func (s *nodeAgentServer) run() {
if err := pvrReconciler.AttemptPVRResume(s.ctx, s.logger.WithField("node", s.nodeName), s.namespace); err != nil {
s.logger.WithError(errors.WithStack(err)).Error("Failed to attempt PVR resume")
}
s.markLegacyPVRsFailed(s.mgr.GetClient())
}()
s.logger.Info("Controllers starting...")
@@ -604,47 +597,6 @@ func (s *nodeAgentServer) validatePodVolumesHostPath(client kubernetes.Interface
return nil
}
func (s *nodeAgentServer) markLegacyPVRsFailed(client ctrlclient.Client) {
pvrs := &velerov1api.PodVolumeRestoreList{}
if err := client.List(s.ctx, pvrs, &ctrlclient.ListOptions{Namespace: s.namespace}); err != nil {
s.logger.WithError(errors.WithStack(err)).Error("failed to list podvolumerestores")
return
}
for i, pvr := range pvrs.Items {
if !controller.IsLegacyPVR(&pvr) {
continue
}
if pvr.Status.Phase != velerov1api.PodVolumeRestorePhaseInProgress {
s.logger.Debugf("the status of podvolumerestore %q is %q, skip", pvr.GetName(), pvr.Status.Phase)
continue
}
pod := &corev1api.Pod{}
if err := client.Get(s.ctx, types.NamespacedName{
Namespace: pvr.Spec.Pod.Namespace,
Name: pvr.Spec.Pod.Name,
}, pod); err != nil {
s.logger.WithError(errors.WithStack(err)).Errorf("failed to get pod \"%s/%s\" of podvolumerestore %q",
pvr.Spec.Pod.Namespace, pvr.Spec.Pod.Name, pvr.GetName())
continue
}
if pod.Spec.NodeName != s.nodeName {
s.logger.Debugf("the node of pod referenced by podvolumerestore %q is %q, not %q, skip", pvr.GetName(), pod.Spec.NodeName, s.nodeName)
continue
}
if err := controller.UpdatePVRStatusToFailed(s.ctx, client, &pvrs.Items[i], errors.New("cannot survive from node-agent restart"),
fmt.Sprintf("get a legacy podvolumerestore with status %q during the server starting, mark it as %q", velerov1api.PodVolumeRestorePhaseInProgress, velerov1api.PodVolumeRestorePhaseFailed),
time.Now(), s.logger); err != nil {
s.logger.WithError(errors.WithStack(err)).Errorf("failed to patch podvolumerestore %q", pvr.GetName())
continue
}
s.logger.WithField("podvolumerestore", pvr.GetName()).Warn(pvr.Status.Message)
}
}
var getConfigsFunc = nodeagent.GetConfigs
func (s *nodeAgentServer) getDataPathConfigs() error {
+2 -2
View File
@@ -1164,8 +1164,8 @@ func markPodVolumeRestoresCancel(ctx context.Context, client ctrlclient.Client,
for i := range pvrs.Items {
pvr := pvrs.Items[i]
if controller.IsLegacyPVR(&pvr) {
log.WithField("PVR", pvr.GetName()).Warn("Found a legacy PVR during velero server restart, cannot stop it")
if _, err := uploader.ValidateUploaderType(pvr.Spec.UploaderType); err != nil {
log.WithField("PVR", pvr.Name).Warnf("invalid uploader type %s, skip marking cancel for this PVR", pvr.Spec.UploaderType)
continue
}
+5
View File
@@ -132,6 +132,11 @@ operations can also be performed as 'velero backup get' and 'velero schedule cre
// init and add the klog flags
klog.InitFlags(flag.CommandLine)
// Opt into the new klog behavior so that -stderrthreshold is honored even
// when -logtostderr=true (the default).
// Ref: kubernetes/klog#212, kubernetes/klog#432
flag.CommandLine.Set("legacy_stderr_threshold_behavior", "false") //nolint:errcheck // flag is registered by klog.InitFlags above
flag.CommandLine.Set("stderrthreshold", "INFO") //nolint:errcheck // flag is registered by klog.InitFlags above
c.PersistentFlags().AddGoFlagSet(flag.CommandLine)
return c
+7
View File
@@ -570,6 +570,13 @@ func (b *backupReconciler) prepareBackupRequest(ctx context.Context, backup *vel
}
}
// Empty IncludedNamespaces means "include all namespaces". Normalize
// to ["*"] so that downstream wildcard expansion does not collapse
// an empty-includes + wildcard-excludes combination into "back up nothing".
if len(request.Spec.IncludedNamespaces) == 0 {
request.Spec.IncludedNamespaces = []string{"*"}
}
// validate the included/excluded namespaces
for _, err := range collections.ValidateNamespaceIncludesExcludes(request.Spec.IncludedNamespaces, request.Spec.ExcludedNamespaces) {
request.Status.ValidationErrors = append(request.Status.ValidationErrors, fmt.Sprintf("Invalid included/excluded namespace lists: %v", err))
+49 -1
View File
@@ -27,6 +27,7 @@ import (
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
@@ -320,6 +321,34 @@ func TestBackupLocationLabel(t *testing.T) {
}
}
func TestPrepareBackupRequest_EmptyIncludedNamespacesNormalizedToWildcard(t *testing.T) {
formatFlag := logging.FormatText
logger := logging.DefaultLogger(logrus.DebugLevel, formatFlag)
apiServer := velerotest.NewAPIServer(t)
discoveryHelper, err := discovery.NewHelper(apiServer.DiscoveryClient, logger)
require.NoError(t, err)
backupLocation := builder.ForBackupStorageLocation("velero", "loc-1").Result()
fakeClient := velerotest.NewFakeControllerRuntimeClient(t, backupLocation)
c := &backupReconciler{
discoveryHelper: discoveryHelper,
kbClient: fakeClient,
defaultBackupLocation: backupLocation.Name,
clock: &clock.RealClock{},
formatFlag: formatFlag,
}
backup := defaultBackup().Result()
backup.Spec.IncludedNamespaces = nil
res := c.prepareBackupRequest(ctx, backup, logger)
defer res.WorkerPool.Stop()
assert.Equal(t, []string{"*"}, res.Spec.IncludedNamespaces)
}
func Test_prepareBackupRequest_BackupStorageLocation(t *testing.T) {
var (
defaultBackupTTL = metav1.Duration{Duration: 24 * 30 * time.Hour}
@@ -709,6 +738,7 @@ func TestProcessBackupCompletions(t *testing.T) {
},
Spec: velerov1api.BackupSpec{
StorageLocation: defaultBackupLocation.Name,
IncludedNamespaces: []string{"*"},
DefaultVolumesToFsBackup: boolptr.True(),
SnapshotMoveData: boolptr.False(),
ExcludedClusterScopedResources: autoExcludeClusterScopedResources,
@@ -748,6 +778,7 @@ func TestProcessBackupCompletions(t *testing.T) {
},
Spec: velerov1api.BackupSpec{
StorageLocation: "alt-loc",
IncludedNamespaces: []string{"*"},
DefaultVolumesToFsBackup: boolptr.False(),
SnapshotMoveData: boolptr.False(),
ExcludedClusterScopedResources: autoExcludeClusterScopedResources,
@@ -791,6 +822,7 @@ func TestProcessBackupCompletions(t *testing.T) {
},
Spec: velerov1api.BackupSpec{
StorageLocation: "read-write",
IncludedNamespaces: []string{"*"},
DefaultVolumesToFsBackup: boolptr.True(),
SnapshotMoveData: boolptr.False(),
ExcludedClusterScopedResources: autoExcludeClusterScopedResources,
@@ -831,6 +863,7 @@ func TestProcessBackupCompletions(t *testing.T) {
Spec: velerov1api.BackupSpec{
TTL: metav1.Duration{Duration: 10 * time.Minute},
StorageLocation: defaultBackupLocation.Name,
IncludedNamespaces: []string{"*"},
DefaultVolumesToFsBackup: boolptr.False(),
SnapshotMoveData: boolptr.False(),
ExcludedClusterScopedResources: autoExcludeClusterScopedResources,
@@ -871,6 +904,7 @@ func TestProcessBackupCompletions(t *testing.T) {
},
Spec: velerov1api.BackupSpec{
StorageLocation: defaultBackupLocation.Name,
IncludedNamespaces: []string{"*"},
DefaultVolumesToFsBackup: boolptr.True(),
SnapshotMoveData: boolptr.False(),
ExcludedClusterScopedResources: autoExcludeClusterScopedResources,
@@ -912,6 +946,7 @@ func TestProcessBackupCompletions(t *testing.T) {
},
Spec: velerov1api.BackupSpec{
StorageLocation: defaultBackupLocation.Name,
IncludedNamespaces: []string{"*"},
DefaultVolumesToFsBackup: boolptr.False(),
SnapshotMoveData: boolptr.False(),
ExcludedClusterScopedResources: autoExcludeClusterScopedResources,
@@ -953,6 +988,7 @@ func TestProcessBackupCompletions(t *testing.T) {
},
Spec: velerov1api.BackupSpec{
StorageLocation: defaultBackupLocation.Name,
IncludedNamespaces: []string{"*"},
DefaultVolumesToFsBackup: boolptr.True(),
SnapshotMoveData: boolptr.False(),
ExcludedClusterScopedResources: autoExcludeClusterScopedResources,
@@ -994,6 +1030,7 @@ func TestProcessBackupCompletions(t *testing.T) {
},
Spec: velerov1api.BackupSpec{
StorageLocation: defaultBackupLocation.Name,
IncludedNamespaces: []string{"*"},
DefaultVolumesToFsBackup: boolptr.True(),
SnapshotMoveData: boolptr.False(),
ExcludedClusterScopedResources: autoExcludeClusterScopedResources,
@@ -1035,6 +1072,7 @@ func TestProcessBackupCompletions(t *testing.T) {
},
Spec: velerov1api.BackupSpec{
StorageLocation: defaultBackupLocation.Name,
IncludedNamespaces: []string{"*"},
DefaultVolumesToFsBackup: boolptr.False(),
SnapshotMoveData: boolptr.False(),
ExcludedClusterScopedResources: autoExcludeClusterScopedResources,
@@ -1077,6 +1115,7 @@ func TestProcessBackupCompletions(t *testing.T) {
},
Spec: velerov1api.BackupSpec{
StorageLocation: defaultBackupLocation.Name,
IncludedNamespaces: []string{"*"},
DefaultVolumesToFsBackup: boolptr.True(),
SnapshotMoveData: boolptr.False(),
ExcludedClusterScopedResources: autoExcludeClusterScopedResources,
@@ -1119,6 +1158,7 @@ func TestProcessBackupCompletions(t *testing.T) {
},
Spec: velerov1api.BackupSpec{
StorageLocation: defaultBackupLocation.Name,
IncludedNamespaces: []string{"*"},
DefaultVolumesToFsBackup: boolptr.True(),
SnapshotMoveData: boolptr.False(),
ExcludedClusterScopedResources: autoExcludeClusterScopedResources,
@@ -1161,6 +1201,7 @@ func TestProcessBackupCompletions(t *testing.T) {
},
Spec: velerov1api.BackupSpec{
StorageLocation: defaultBackupLocation.Name,
IncludedNamespaces: []string{"*"},
DefaultVolumesToFsBackup: boolptr.False(),
SnapshotMoveData: boolptr.True(),
ExcludedClusterScopedResources: autoExcludeClusterScopedResources,
@@ -1204,6 +1245,7 @@ func TestProcessBackupCompletions(t *testing.T) {
},
Spec: velerov1api.BackupSpec{
StorageLocation: defaultBackupLocation.Name,
IncludedNamespaces: []string{"*"},
DefaultVolumesToFsBackup: boolptr.False(),
SnapshotMoveData: boolptr.False(),
ExcludedClusterScopedResources: autoExcludeClusterScopedResources,
@@ -1247,6 +1289,7 @@ func TestProcessBackupCompletions(t *testing.T) {
},
Spec: velerov1api.BackupSpec{
StorageLocation: defaultBackupLocation.Name,
IncludedNamespaces: []string{"*"},
DefaultVolumesToFsBackup: boolptr.False(),
SnapshotMoveData: boolptr.False(),
ExcludedClusterScopedResources: autoExcludeClusterScopedResources,
@@ -1290,6 +1333,7 @@ func TestProcessBackupCompletions(t *testing.T) {
},
Spec: velerov1api.BackupSpec{
StorageLocation: defaultBackupLocation.Name,
IncludedNamespaces: []string{"*"},
DefaultVolumesToFsBackup: boolptr.False(),
SnapshotMoveData: boolptr.True(),
ExcludedClusterScopedResources: autoExcludeClusterScopedResources,
@@ -1334,6 +1378,7 @@ func TestProcessBackupCompletions(t *testing.T) {
},
Spec: velerov1api.BackupSpec{
StorageLocation: defaultBackupLocation.Name,
IncludedNamespaces: []string{"*"},
DefaultVolumesToFsBackup: boolptr.False(),
SnapshotMoveData: boolptr.False(),
ExcludedClusterScopedResources: autoExcludeClusterScopedResources,
@@ -1377,6 +1422,7 @@ func TestProcessBackupCompletions(t *testing.T) {
},
Spec: velerov1api.BackupSpec{
StorageLocation: defaultBackupLocation.Name,
IncludedNamespaces: []string{"*"},
DefaultVolumesToFsBackup: boolptr.False(),
SnapshotMoveData: boolptr.True(),
ExcludedClusterScopedResources: autoExcludeClusterScopedResources,
@@ -1424,6 +1470,7 @@ func TestProcessBackupCompletions(t *testing.T) {
},
Spec: velerov1api.BackupSpec{
StorageLocation: defaultBackupLocation.Name,
IncludedNamespaces: []string{"*"},
DefaultVolumesToFsBackup: boolptr.False(),
SnapshotMoveData: boolptr.True(),
IncludedClusterScopedResources: []string{"storageclasses"},
@@ -1473,6 +1520,7 @@ func TestProcessBackupCompletions(t *testing.T) {
},
Spec: velerov1api.BackupSpec{
StorageLocation: defaultBackupLocation.Name,
IncludedNamespaces: []string{"*"},
DefaultVolumesToFsBackup: boolptr.False(),
SnapshotMoveData: boolptr.True(),
IncludedClusterScopedResources: []string{"storageclasses"},
@@ -1609,7 +1657,7 @@ func TestProcessBackupCompletions(t *testing.T) {
err = c.kbClient.Get(t.Context(), kbclient.ObjectKey{Namespace: test.backup.Namespace, Name: test.backup.Name}, res)
require.NoError(t, err)
res.ResourceVersion = ""
assert.Equal(t, test.expectedResult, res)
assert.Empty(t, cmp.Diff(test.expectedResult, res, cmpopts.IgnoreFields(velerov1api.Backup{}, "TypeMeta")))
// reset defaultBackupLocation resourceVersion
defaultBackupLocation.ObjectMeta.ResourceVersion = ""
})
@@ -603,7 +603,7 @@ func (r *PodVolumeRestoreReconciler) closeDataPath(ctx context.Context, pvrName
func (r *PodVolumeRestoreReconciler) SetupWithManager(mgr ctrl.Manager) error {
gp := kube.NewGenericEventPredicate(func(object client.Object) bool {
pvr := object.(*velerov1api.PodVolumeRestore)
if IsLegacyPVR(pvr) {
if _, err := uploader.ValidateUploaderType(pvr.Spec.UploaderType); err != nil {
return false
}
@@ -628,7 +628,8 @@ func (r *PodVolumeRestoreReconciler) SetupWithManager(mgr ctrl.Manager) error {
pred := kube.NewAllEventPredicate(func(obj client.Object) bool {
pvr := obj.(*velerov1api.PodVolumeRestore)
return !IsLegacyPVR(pvr)
_, err := uploader.ValidateUploaderType(pvr.Spec.UploaderType)
return err == nil
})
return ctrl.NewControllerManagedBy(mgr).
@@ -678,7 +679,7 @@ func (r *PodVolumeRestoreReconciler) findPVRForTargetPod(ctx context.Context, po
requests := []reconcile.Request{}
for _, item := range list.Items {
if IsLegacyPVR(&item) {
if _, err := uploader.ValidateUploaderType(item.Spec.UploaderType); err != nil {
continue
}
@@ -708,6 +709,11 @@ func (r *PodVolumeRestoreReconciler) findPVRForRestorePod(ctx context.Context, p
"PVR": pvr.Name,
})
if _, err := uploader.ValidateUploaderType(pvr.Spec.UploaderType); err != nil {
log.WithField("uploaderType", pvr.Spec.UploaderType).Debug("skip PVR with invalid uploader type")
return []reconcile.Request{}
}
if pvr.Status.Phase != velerov1api.PodVolumeRestorePhaseAccepted {
return []reconcile.Request{}
}
@@ -1029,7 +1035,7 @@ func (r *PodVolumeRestoreReconciler) AttemptPVRResume(ctx context.Context, logge
for i := range pvrs.Items {
pvr := &pvrs.Items[i]
if IsLegacyPVR(pvr) {
if _, err := uploader.ValidateUploaderType(pvr.Spec.UploaderType); err != nil {
continue
}
@@ -1,364 +0,0 @@
/*
Copyright The Velero Contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controller
import (
"context"
"fmt"
"os"
"path/filepath"
"time"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
corev1api "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes"
clocks "k8s.io/utils/clock"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/builder"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/handler"
"sigs.k8s.io/controller-runtime/pkg/manager"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"github.com/vmware-tanzu/velero/internal/credentials"
veleroapishared "github.com/vmware-tanzu/velero/pkg/apis/velero/shared"
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
"github.com/vmware-tanzu/velero/pkg/datapath"
"github.com/vmware-tanzu/velero/pkg/exposer"
"github.com/vmware-tanzu/velero/pkg/podvolume"
"github.com/vmware-tanzu/velero/pkg/repository"
"github.com/vmware-tanzu/velero/pkg/restorehelper"
"github.com/vmware-tanzu/velero/pkg/uploader"
"github.com/vmware-tanzu/velero/pkg/util/boolptr"
"github.com/vmware-tanzu/velero/pkg/util/filesystem"
"github.com/vmware-tanzu/velero/pkg/util/kube"
)
func InitLegacyPodVolumeRestoreReconciler(client client.Client, mgr manager.Manager, kubeClient kubernetes.Interface, dataPathMgr *datapath.Manager, namespace string,
resourceTimeout time.Duration, logger logrus.FieldLogger) error {
log := logger.WithField("controller", "PodVolumeRestoreLegacy")
credentialFileStore, err := credentials.NewNamespacedFileStore(client, namespace, credentials.DefaultStoreDirectory(), filesystem.NewFileSystem())
if err != nil {
return errors.Wrapf(err, "error creating credentials file store")
}
credSecretStore, err := credentials.NewNamespacedSecretStore(client, namespace)
if err != nil {
return errors.Wrapf(err, "error creating secret file store")
}
credentialGetter := &credentials.CredentialGetter{FromFile: credentialFileStore, FromSecret: credSecretStore}
ensurer := repository.NewEnsurer(client, log, resourceTimeout)
reconciler := &PodVolumeRestoreReconcilerLegacy{
Client: client,
kubeClient: kubeClient,
logger: log,
repositoryEnsurer: ensurer,
credentialGetter: credentialGetter,
fileSystem: filesystem.NewFileSystem(),
clock: &clocks.RealClock{},
dataPathMgr: dataPathMgr,
}
if err = reconciler.SetupWithManager(mgr); err != nil {
return errors.Wrapf(err, "error setup controller manager")
}
return nil
}
type PodVolumeRestoreReconcilerLegacy struct {
client.Client
kubeClient kubernetes.Interface
logger logrus.FieldLogger
repositoryEnsurer *repository.Ensurer
credentialGetter *credentials.CredentialGetter
fileSystem filesystem.Interface
clock clocks.WithTickerAndDelayedExecution
dataPathMgr *datapath.Manager
}
// +kubebuilder:rbac:groups=velero.io,resources=podvolumerestores,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=velero.io,resources=podvolumerestores/status,verbs=get;update;patch
// +kubebuilder:rbac:groups="",resources=pods,verbs=get
// +kubebuilder:rbac:groups="",resources=persistentvolumes,verbs=get
// +kubebuilder:rbac:groups="",resources=persistentvolumerclaims,verbs=get
func (c *PodVolumeRestoreReconcilerLegacy) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
log := c.logger.WithField("PodVolumeRestore", req.NamespacedName.String())
log.Info("Reconciling PVR by legacy controller")
pvr := &velerov1api.PodVolumeRestore{}
if err := c.Get(ctx, types.NamespacedName{Namespace: req.Namespace, Name: req.Name}, pvr); err != nil {
if apierrors.IsNotFound(err) {
log.Warn("PodVolumeRestore not found, skip")
return ctrl.Result{}, nil
}
log.WithError(err).Error("Unable to get the PodVolumeRestore")
return ctrl.Result{}, err
}
log = log.WithField("pod", fmt.Sprintf("%s/%s", pvr.Spec.Pod.Namespace, pvr.Spec.Pod.Name))
if len(pvr.OwnerReferences) == 1 {
log = log.WithField("restore", fmt.Sprintf("%s/%s", pvr.Namespace, pvr.OwnerReferences[0].Name))
}
shouldProcess, pod, err := shouldProcess(ctx, c.Client, log, pvr)
if err != nil {
return ctrl.Result{}, err
}
if !shouldProcess {
return ctrl.Result{}, nil
}
initContainerIndex := getInitContainerIndex(pod)
if initContainerIndex > 0 {
log.Warnf(`Init containers before the %s container may cause issues
if they interfere with volumes being restored: %s index %d`, restorehelper.WaitInitContainer, restorehelper.WaitInitContainer, initContainerIndex)
}
log.Info("Restore starting")
callbacks := datapath.Callbacks{
OnCompleted: c.OnDataPathCompleted,
OnFailed: c.OnDataPathFailed,
OnCancelled: c.OnDataPathCancelled,
OnProgress: c.OnDataPathProgress,
}
fsRestore, err := c.dataPathMgr.CreateFileSystemBR(pvr.Name, pVBRRequestor, ctx, c.Client, pvr.Namespace, callbacks, log)
if err != nil {
if err == datapath.ConcurrentLimitExceed {
return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, nil
} else {
return c.errorOut(ctx, pvr, err, "error to create data path", log)
}
}
original := pvr.DeepCopy()
pvr.Status.Phase = velerov1api.PodVolumeRestorePhaseInProgress
pvr.Status.StartTimestamp = &metav1.Time{Time: c.clock.Now()}
if err = c.Patch(ctx, pvr, client.MergeFrom(original)); err != nil {
c.closeDataPath(ctx, pvr.Name)
return c.errorOut(ctx, pvr, err, "error to update status to in progress", log)
}
volumePath, err := exposer.GetPodVolumeHostPath(ctx, pod, pvr.Spec.Volume, c.kubeClient, c.fileSystem, log)
if err != nil {
c.closeDataPath(ctx, pvr.Name)
return c.errorOut(ctx, pvr, err, "error exposing host path for pod volume", log)
}
log.WithField("path", volumePath.ByPath).Debugf("Found host path")
if err := fsRestore.Init(ctx, &datapath.FSBRInitParam{
BSLName: pvr.Spec.BackupStorageLocation,
SourceNamespace: pvr.Spec.SourceNamespace,
UploaderType: pvr.Spec.UploaderType,
RepositoryType: podvolume.GetPvrRepositoryType(pvr),
RepoIdentifier: pvr.Spec.RepoIdentifier,
RepositoryEnsurer: c.repositoryEnsurer,
CredentialGetter: c.credentialGetter,
}); err != nil {
c.closeDataPath(ctx, pvr.Name)
return c.errorOut(ctx, pvr, err, "error to initialize data path", log)
}
if err := fsRestore.StartRestore(pvr.Spec.SnapshotID, volumePath, pvr.Spec.UploaderSettings); err != nil {
c.closeDataPath(ctx, pvr.Name)
return c.errorOut(ctx, pvr, err, "error starting data path restore", log)
}
log.WithField("path", volumePath.ByPath).Info("Async fs restore data path started")
return ctrl.Result{}, nil
}
func (c *PodVolumeRestoreReconcilerLegacy) errorOut(ctx context.Context, pvr *velerov1api.PodVolumeRestore, err error, msg string, log logrus.FieldLogger) (ctrl.Result, error) {
_ = UpdatePVRStatusToFailed(ctx, c.Client, pvr, err, msg, c.clock.Now(), log)
return ctrl.Result{}, err
}
func (c *PodVolumeRestoreReconcilerLegacy) SetupWithManager(mgr ctrl.Manager) error {
// The pod may not being scheduled at the point when its PVRs are initially reconciled.
// By watching the pods, we can trigger the PVR reconciliation again once the pod is finally scheduled on the node.
pred := kube.NewAllEventPredicate(func(obj client.Object) bool {
pvr := obj.(*velerov1api.PodVolumeRestore)
return IsLegacyPVR(pvr)
})
return ctrl.NewControllerManagedBy(mgr).Named("podvolumerestorelegacy").
For(&velerov1api.PodVolumeRestore{}, builder.WithPredicates(pred)).
Watches(&corev1api.Pod{}, handler.EnqueueRequestsFromMapFunc(c.findVolumeRestoresForPod)).
Complete(c)
}
func (c *PodVolumeRestoreReconcilerLegacy) findVolumeRestoresForPod(ctx context.Context, pod client.Object) []reconcile.Request {
list := &velerov1api.PodVolumeRestoreList{}
options := &client.ListOptions{
LabelSelector: labels.Set(map[string]string{
velerov1api.PodUIDLabel: string(pod.GetUID()),
}).AsSelector(),
}
if err := c.Client.List(context.TODO(), list, options); err != nil {
c.logger.WithField("pod", fmt.Sprintf("%s/%s", pod.GetNamespace(), pod.GetName())).WithError(err).
Error("unable to list PodVolumeRestores")
return []reconcile.Request{}
}
requests := []reconcile.Request{}
for _, item := range list.Items {
if !IsLegacyPVR(&item) {
continue
}
requests = append(requests, reconcile.Request{
NamespacedName: types.NamespacedName{
Namespace: item.GetNamespace(),
Name: item.GetName(),
},
})
}
return requests
}
func (c *PodVolumeRestoreReconcilerLegacy) OnDataPathCompleted(ctx context.Context, namespace string, pvrName string, result datapath.Result) {
defer c.dataPathMgr.RemoveAsyncBR(pvrName)
log := c.logger.WithField("pvr", pvrName)
log.WithField("PVR", pvrName).Info("Async fs restore data path completed")
var pvr velerov1api.PodVolumeRestore
if err := c.Client.Get(ctx, types.NamespacedName{Name: pvrName, Namespace: namespace}, &pvr); err != nil {
log.WithError(err).Warn("Failed to get PVR on completion")
return
}
volumePath := result.Restore.Target.ByPath
if volumePath == "" {
_, _ = c.errorOut(ctx, &pvr, errors.New("path is empty"), "invalid restore target", log)
return
}
// Remove the .velero directory from the restored volume (it may contain done files from previous restores
// of this volume, which we don't want to carry over). If this fails for any reason, log and continue, since
// this is non-essential cleanup (the done files are named based on restore UID and the init container looks
// for the one specific to the restore being executed).
if err := os.RemoveAll(filepath.Join(volumePath, ".velero")); err != nil {
log.WithError(err).Warnf("error removing .velero directory from directory %s", volumePath)
}
var restoreUID types.UID
for _, owner := range pvr.OwnerReferences {
if boolptr.IsSetToTrue(owner.Controller) {
restoreUID = owner.UID
break
}
}
// Create the .velero directory within the volume dir so we can write a done file
// for this restore.
if err := os.MkdirAll(filepath.Join(volumePath, ".velero"), 0755); err != nil {
_, _ = c.errorOut(ctx, &pvr, err, "error creating .velero directory for done file", log)
return
}
// Write a done file with name=<restore-uid> into the just-created .velero dir
// within the volume. The velero init container on the pod is waiting
// for this file to exist in each restored volume before completing.
if err := os.WriteFile(filepath.Join(volumePath, ".velero", string(restoreUID)), nil, 0644); err != nil { //nolint:gosec // Internal usage. No need to check.
_, _ = c.errorOut(ctx, &pvr, err, "error writing done file", log)
return
}
original := pvr.DeepCopy()
pvr.Status.Phase = velerov1api.PodVolumeRestorePhaseCompleted
pvr.Status.CompletionTimestamp = &metav1.Time{Time: c.clock.Now()}
if err := c.Patch(ctx, &pvr, client.MergeFrom(original)); err != nil {
log.WithError(err).Error("error updating PodVolumeRestore status")
}
log.Info("Restore completed")
}
func (c *PodVolumeRestoreReconcilerLegacy) OnDataPathFailed(ctx context.Context, namespace string, pvrName string, err error) {
defer c.dataPathMgr.RemoveAsyncBR(pvrName)
log := c.logger.WithField("pvr", pvrName)
log.WithError(err).Error("Async fs restore data path failed")
var pvr velerov1api.PodVolumeRestore
if getErr := c.Client.Get(ctx, types.NamespacedName{Name: pvrName, Namespace: namespace}, &pvr); getErr != nil {
log.WithError(getErr).Warn("Failed to get PVR on failure")
} else {
_, _ = c.errorOut(ctx, &pvr, err, "data path restore failed", log)
}
}
func (c *PodVolumeRestoreReconcilerLegacy) OnDataPathCancelled(ctx context.Context, namespace string, pvrName string) {
defer c.dataPathMgr.RemoveAsyncBR(pvrName)
log := c.logger.WithField("pvr", pvrName)
log.Warn("Async fs restore data path canceled")
var pvr velerov1api.PodVolumeRestore
if getErr := c.Client.Get(ctx, types.NamespacedName{Name: pvrName, Namespace: namespace}, &pvr); getErr != nil {
log.WithError(getErr).Warn("Failed to get PVR on cancel")
} else {
_, _ = c.errorOut(ctx, &pvr, errors.New("PVR is canceled"), "data path restore canceled", log)
}
}
func (c *PodVolumeRestoreReconcilerLegacy) OnDataPathProgress(ctx context.Context, namespace string, pvrName string, progress *uploader.Progress) {
log := c.logger.WithField("pvr", pvrName)
var pvr velerov1api.PodVolumeRestore
if err := c.Client.Get(ctx, types.NamespacedName{Name: pvrName, Namespace: namespace}, &pvr); err != nil {
log.WithError(err).Warn("Failed to get PVB on progress")
return
}
original := pvr.DeepCopy()
pvr.Status.Progress = veleroapishared.DataMoveOperationProgress{TotalBytes: progress.TotalBytes, BytesDone: progress.BytesDone}
if err := c.Client.Patch(ctx, &pvr, client.MergeFrom(original)); err != nil {
log.WithError(err).Error("Failed to update progress")
}
}
func (c *PodVolumeRestoreReconcilerLegacy) closeDataPath(ctx context.Context, pvbName string) {
fsRestore := c.dataPathMgr.GetAsyncBR(pvbName)
if fsRestore != nil {
fsRestore.Close(ctx)
}
c.dataPathMgr.RemoveAsyncBR(pvbName)
}
func IsLegacyPVR(pvr *velerov1api.PodVolumeRestore) bool {
return pvr.Spec.UploaderType == "restic"
}
@@ -1,93 +0,0 @@
/*
Copyright The Velero Contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controller
import (
"testing"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
corev1api "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
)
func TestFindVolumeRestoresForPodLegacy(t *testing.T) {
pod := &corev1api.Pod{}
pod.UID = "uid"
scheme := runtime.NewScheme()
scheme.AddKnownTypes(velerov1api.SchemeGroupVersion, &velerov1api.PodVolumeRestore{}, &velerov1api.PodVolumeRestoreList{})
clientBuilder := fake.NewClientBuilder().WithScheme(scheme)
// no matching PVR
reconciler := &PodVolumeRestoreReconcilerLegacy{
Client: clientBuilder.Build(),
logger: logrus.New(),
}
requests := reconciler.findVolumeRestoresForPod(t.Context(), pod)
assert.Empty(t, requests)
// contain one matching PVR
reconciler.Client = clientBuilder.WithLists(&velerov1api.PodVolumeRestoreList{
Items: []velerov1api.PodVolumeRestore{
{
ObjectMeta: metav1.ObjectMeta{
Name: "pvr1",
Labels: map[string]string{
velerov1api.PodUIDLabel: string(pod.GetUID()),
},
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "pvr2",
Labels: map[string]string{
velerov1api.PodUIDLabel: "non-matching-uid",
},
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "pvr3",
Labels: map[string]string{
velerov1api.PodUIDLabel: string(pod.GetUID()),
},
},
Spec: velerov1api.PodVolumeRestoreSpec{
UploaderType: "kopia",
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "pvr4",
Labels: map[string]string{
velerov1api.PodUIDLabel: string(pod.GetUID()),
},
},
Spec: velerov1api.PodVolumeRestoreSpec{
UploaderType: "restic",
},
},
},
}).Build()
requests = reconciler.findVolumeRestoresForPod(t.Context(), pod)
assert.Len(t, requests, 1)
}
@@ -506,18 +506,17 @@ func TestFindPVRForTargetPod(t *testing.T) {
scheme := runtime.NewScheme()
scheme.AddKnownTypes(velerov1api.SchemeGroupVersion, &velerov1api.PodVolumeRestore{}, &velerov1api.PodVolumeRestoreList{})
clientBuilder := fake.NewClientBuilder().WithScheme(scheme)
// no matching PVR
reconciler := &PodVolumeRestoreReconciler{
client: clientBuilder.Build(),
client: fake.NewClientBuilder().WithScheme(scheme).Build(),
logger: logrus.New(),
}
requests := reconciler.findPVRForTargetPod(t.Context(), pod)
assert.Empty(t, requests)
// contain one matching PVR
reconciler.client = clientBuilder.WithLists(&velerov1api.PodVolumeRestoreList{
reconciler.client = fake.NewClientBuilder().WithScheme(scheme).WithLists(&velerov1api.PodVolumeRestoreList{
Items: []velerov1api.PodVolumeRestore{
{
ObjectMeta: metav1.ObjectMeta{
@@ -526,6 +525,7 @@ func TestFindPVRForTargetPod(t *testing.T) {
velerov1api.PodUIDLabel: string(pod.GetUID()),
},
},
Spec: velerov1api.PodVolumeRestoreSpec{UploaderType: uploader.KopiaType},
},
{
ObjectMeta: metav1.ObjectMeta{
@@ -688,6 +688,7 @@ func TestPodVolumeRestoreReconcile(t *testing.T) {
mockClose bool
needExclusiveUpdateError error
constrained bool
preserveEmptyUploader bool
expected *velerov1api.PodVolumeRestore
expectDeleted bool
expectCancelRecord bool
@@ -939,6 +940,13 @@ func TestPodVolumeRestoreReconcile(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if !test.preserveEmptyUploader && test.pvr != nil && test.pvr.Spec.UploaderType == "" {
test.pvr.Spec.UploaderType = uploader.KopiaType
}
if !test.preserveEmptyUploader && test.expected != nil && test.expected.Spec.UploaderType == "" {
test.expected.Spec.UploaderType = uploader.KopiaType
}
objs := []runtime.Object{daemonSet, node}
ctlObj := []client.Object{}
@@ -1396,7 +1404,7 @@ func TestFindPVBForRestorePod(t *testing.T) {
}{
{
name: "find pvr for pod",
pvr: pvrBuilder().Phase(velerov1api.PodVolumeRestorePhaseAccepted).Result(),
pvr: pvrBuilder().UploaderType(uploader.KopiaType).Phase(velerov1api.PodVolumeRestorePhaseAccepted).Result(),
pod: builder.ForPod(velerov1api.DefaultNamespace, pvrName).Labels(map[string]string{velerov1api.PVRLabel: pvrName}).Status(corev1api.PodStatus{Phase: corev1api.PodRunning}).Result(),
checkFunc: func(pvr *velerov1api.PodVolumeRestore, requests []reconcile.Request) {
// Assert that the function returns a single request
@@ -1407,7 +1415,7 @@ func TestFindPVBForRestorePod(t *testing.T) {
},
}, {
name: "no selected label found for pod",
pvr: pvrBuilder().Phase(velerov1api.PodVolumeRestorePhaseAccepted).Result(),
pvr: pvrBuilder().UploaderType(uploader.KopiaType).Phase(velerov1api.PodVolumeRestorePhaseAccepted).Result(),
pod: builder.ForPod(velerov1api.DefaultNamespace, pvrName).Result(),
checkFunc: func(pvr *velerov1api.PodVolumeRestore, requests []reconcile.Request) {
// Assert that the function returns a single request
@@ -1415,7 +1423,7 @@ func TestFindPVBForRestorePod(t *testing.T) {
},
}, {
name: "no matched pod",
pvr: pvrBuilder().Phase(velerov1api.PodVolumeRestorePhaseAccepted).Result(),
pvr: pvrBuilder().UploaderType(uploader.KopiaType).Phase(velerov1api.PodVolumeRestorePhaseAccepted).Result(),
pod: builder.ForPod(velerov1api.DefaultNamespace, pvrName).Labels(map[string]string{velerov1api.PVRLabel: "non-existing-pvr"}).Result(),
checkFunc: func(pvr *velerov1api.PodVolumeRestore, requests []reconcile.Request) {
assert.Empty(t, requests)
@@ -1423,12 +1431,20 @@ func TestFindPVBForRestorePod(t *testing.T) {
},
{
name: "pvr not accept",
pvr: pvrBuilder().Phase(velerov1api.PodVolumeRestorePhaseInProgress).Result(),
pvr: pvrBuilder().UploaderType(uploader.KopiaType).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Result(),
pod: builder.ForPod(velerov1api.DefaultNamespace, pvrName).Labels(map[string]string{velerov1api.PVRLabel: pvrName}).Result(),
checkFunc: func(pvr *velerov1api.PodVolumeRestore, requests []reconcile.Request) {
assert.Empty(t, requests)
},
},
{
name: "invalid uploader type",
pvr: pvrBuilder().UploaderType("restic").Phase(velerov1api.PodVolumeRestorePhaseAccepted).Result(),
pod: builder.ForPod(velerov1api.DefaultNamespace, pvrName).Labels(map[string]string{velerov1api.PVRLabel: pvrName}).Status(corev1api.PodStatus{Phase: corev1api.PodRunning}).Result(),
checkFunc: func(pvr *velerov1api.PodVolumeRestore, requests []reconcile.Request) {
assert.Empty(t, requests)
},
},
}
for _, test := range tests {
ctx := t.Context()
@@ -1613,32 +1629,32 @@ func TestAttemptPVRResume(t *testing.T) {
}{
{
name: "Other pvr",
pvr: pvrBuilder().Phase(velerov1api.PodVolumeRestorePhasePrepared).Result(),
pvr: pvrBuilder().UploaderType(uploader.KopiaType).Phase(velerov1api.PodVolumeRestorePhasePrepared).Result(),
},
{
name: "Other pvr",
pvr: pvrBuilder().Phase(velerov1api.PodVolumeRestorePhaseAccepted).Result(),
pvr: pvrBuilder().UploaderType(uploader.KopiaType).Phase(velerov1api.PodVolumeRestorePhaseAccepted).Result(),
},
{
name: "InProgress pvr, not the current node",
pvr: pvrBuilder().Phase(velerov1api.PodVolumeRestorePhaseInProgress).Result(),
pvr: pvrBuilder().UploaderType(uploader.KopiaType).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Result(),
inProgressPvrs: []string{pvrName},
},
{
name: "InProgress pvr, no resume error",
pvr: pvrBuilder().Phase(velerov1api.PodVolumeRestorePhaseInProgress).Node("node-1").Result(),
pvr: pvrBuilder().UploaderType(uploader.KopiaType).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Node("node-1").Result(),
inProgressPvrs: []string{pvrName},
},
{
name: "InProgress pvr, resume error, cancel error",
pvr: pvrBuilder().Phase(velerov1api.PodVolumeRestorePhaseInProgress).Node("node-1").Result(),
pvr: pvrBuilder().UploaderType(uploader.KopiaType).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Node("node-1").Result(),
resumeErr: errors.New("fake-resume-error"),
needErrs: []bool{false, false, true, false, false, false},
inProgressPvrs: []string{pvrName},
},
{
name: "InProgress pvr, resume error, cancel succeed",
pvr: pvrBuilder().Phase(velerov1api.PodVolumeRestorePhaseInProgress).Node("node-1").Result(),
pvr: pvrBuilder().UploaderType(uploader.KopiaType).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Node("node-1").Result(),
resumeErr: errors.New("fake-resume-error"),
cancelledPvrs: []string{pvrName},
inProgressPvrs: []string{pvrName},
@@ -1646,7 +1662,7 @@ func TestAttemptPVRResume(t *testing.T) {
{
name: "Error",
needErrs: []bool{false, false, false, false, false, true},
pvr: pvrBuilder().Phase(velerov1api.PodVolumeRestorePhasePrepared).Result(),
pvr: pvrBuilder().UploaderType(uploader.KopiaType).Phase(velerov1api.PodVolumeRestorePhasePrepared).Result(),
expectedError: "error to list PVRs: List error",
},
}
+2 -1
View File
@@ -198,7 +198,8 @@ func DaemonSet(namespace string, opts ...podTemplateOption) *appsv1api.DaemonSet
Secret: &corev1api.SecretVolumeSource{
// read-only for Owner, Group, Public
DefaultMode: ptr.To(int32(0444)),
SecretName: "cloud-credentials",
// #nosec G101 -- This is a reference to a Secret resource name, not a credential
SecretName: "cloud-credentials",
},
},
},
+2 -1
View File
@@ -454,7 +454,8 @@ func Deployment(namespace string, opts ...podTemplateOption) *appsv1api.Deployme
Secret: &corev1api.SecretVolumeSource{
// read-only for Owner, Group, Public
DefaultMode: ptr.To(int32(0444)),
SecretName: "cloud-credentials",
// #nosec G101 -- This is a reference to a Secret resource name, not a credential
SecretName: "cloud-credentials",
},
},
},
@@ -196,3 +196,21 @@ func (l *logrusAdapter) Name() string {
func (l *logrusAdapter) StandardWriter(opts *hclog.StandardLoggerOptions) io.Writer {
panic("not implemented")
}
// GetLevel returns the current level
func (l *logrusAdapter) GetLevel() hclog.Level {
switch l.level {
case logrus.TraceLevel:
return hclog.Trace
case logrus.DebugLevel:
return hclog.Debug
case logrus.InfoLevel:
return hclog.Info
case logrus.WarnLevel:
return hclog.Warn
case logrus.ErrorLevel:
return hclog.Error
default:
return hclog.NoLevel
}
}
+1 -1
View File
@@ -140,7 +140,7 @@ func NewServer() Server {
func (s *server) BindFlags(flags *pflag.FlagSet) Server {
s.flagSet = flags
s.config.BindFlags(flags)
s.flagSet.ParseErrorsWhitelist.UnknownFlags = true // Velero.io word list : ignore
s.flagSet.ParseErrorsAllowlist.UnknownFlags = true // Velero.io word list : ignore
return s
}
+64 -30
View File
@@ -29,6 +29,7 @@ import (
corev1api "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
ctrlcache "sigs.k8s.io/controller-runtime/pkg/cache"
ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
ctrlfake "sigs.k8s.io/controller-runtime/pkg/client/fake"
@@ -40,9 +41,61 @@ import (
"github.com/vmware-tanzu/velero/pkg/builder"
"github.com/vmware-tanzu/velero/pkg/repository"
velerotest "github.com/vmware-tanzu/velero/pkg/test"
"github.com/vmware-tanzu/velero/pkg/util/kube"
)
type fakeInformerRegistration struct{}
func (fakeInformerRegistration) HasSynced() bool { return true }
func (fakeInformerRegistration) HasSyncedChecker() cache.DoneChecker {
return fakeDoneChecker{name: "fakeInformerRegistration"}
}
type fakeDoneChecker struct {
name string
}
func (f fakeDoneChecker) Name() string { return f.name }
func (f fakeDoneChecker) Done() <-chan struct{} {
ch := make(chan struct{})
close(ch)
return ch
}
type fakeInformer struct {
handler cache.ResourceEventHandler
}
func (f *fakeInformer) AddEventHandler(handler cache.ResourceEventHandler) (cache.ResourceEventHandlerRegistration, error) {
f.handler = handler
return fakeInformerRegistration{}, nil
}
func (f *fakeInformer) AddEventHandlerWithResyncPeriod(handler cache.ResourceEventHandler, _ time.Duration) (cache.ResourceEventHandlerRegistration, error) {
return f.AddEventHandler(handler)
}
func (f *fakeInformer) AddEventHandlerWithOptions(handler cache.ResourceEventHandler, _ cache.HandlerOptions) (cache.ResourceEventHandlerRegistration, error) {
return f.AddEventHandler(handler)
}
func (f *fakeInformer) RemoveEventHandler(_ cache.ResourceEventHandlerRegistration) error {
return nil
}
func (f *fakeInformer) AddIndexers(_ cache.Indexers) error { return nil }
func (f *fakeInformer) HasSynced() bool { return true }
func (f *fakeInformer) HasSyncedChecker() cache.DoneChecker {
return fakeDoneChecker{name: "fakeInformer"}
}
func (f *fakeInformer) IsStopped() bool { return false }
var _ ctrlcache.Informer = (*fakeInformer)(nil)
func TestIsHostPathVolume(t *testing.T) {
// hostPath pod volume
vol := &corev1api.Volume{
@@ -557,27 +610,16 @@ func TestBackupPodVolumes(t *testing.T) {
objList = append(objList, test.kubeClientObj...)
fakeCtrlClient := fakeClientBuilder.WithRuntimeObjects(objList...).Build()
fakeCRWatchClient := velerotest.NewFakeControllerRuntimeWatchClient(t, test.kubeClientObj...)
lw := kube.InternalLW{
Client: fakeCRWatchClient,
Namespace: velerov1api.DefaultNamespace,
ObjectList: new(velerov1api.PodVolumeBackupList),
}
pvbInformer := cache.NewSharedIndexInformer(&lw, &velerov1api.PodVolumeBackup{}, 0, cache.Indexers{})
go pvbInformer.Run(ctx.Done())
require.True(t, cache.WaitForCacheSync(ctx.Done(), pvbInformer.HasSynced))
// This test validates creation-time behavior only, so we don't need
// informer sync/watch to be running.
pvbInformer := cache.NewSharedIndexInformer(&cache.ListWatch{}, &velerov1api.PodVolumeBackup{}, 0, cache.Indexers{})
ensurer := repository.NewEnsurer(fakeCtrlClient, velerotest.NewLogger(), time.Millisecond)
backupObj := builder.ForBackup(velerov1api.DefaultNamespace, "fake-backup").Result()
backupObj.Spec.StorageLocation = test.bsl
factory := NewBackupperFactory(repository.NewRepoLocker(), ensurer, fakeCtrlClient, pvbInformer, velerotest.NewLogger())
bp, err := factory.NewBackupper(ctx, log, backupObj, test.uploaderType)
require.NoError(t, err)
bp := newBackupper(ctx, log, repository.NewRepoLocker(), ensurer, pvbInformer, fakeCtrlClient, test.uploaderType, backupObj)
if test.mockGetRepositoryType {
funcGetRepositoryType = func() string { return "" }
@@ -587,9 +629,7 @@ func TestBackupPodVolumes(t *testing.T) {
pvbs, _, errs := bp.BackupPodVolumes(backupObj, test.sourcePod, test.volumes, nil, velerotest.NewLogger())
if test.errs == nil {
require.NoError(t, err)
} else {
if test.errs != nil {
for i := 0; i < len(errs); i++ {
require.EqualError(t, errs[i], test.errs[i])
}
@@ -760,17 +800,7 @@ func TestWaitAllPodVolumesProcessed(t *testing.T) {
velerov1api.AddToScheme(scheme)
client := ctrlfake.NewClientBuilder().WithScheme(scheme).WithObjects(objs...).Build()
lw := kube.InternalLW{
Client: client,
Namespace: velerov1api.DefaultNamespace,
ObjectList: new(velerov1api.PodVolumeBackupList),
}
informer := cache.NewSharedIndexInformer(&lw, &velerov1api.PodVolumeBackup{}, 0, cache.Indexers{})
ctx := t.Context()
go informer.Run(ctx.Done())
require.True(t, cache.WaitForCacheSync(ctx.Done(), informer.HasSynced))
informer := &fakeInformer{}
logger := logrus.New()
logHook := &logHook{}
@@ -787,9 +817,13 @@ func TestWaitAllPodVolumesProcessed(t *testing.T) {
err := client.Get(t.Context(), ctrlclient.ObjectKey{Namespace: c.pvb.Namespace, Name: c.pvb.Name}, pvb)
require.NoError(t, err)
oldPVB := pvb.DeepCopy()
pvb.Status = *c.statusToBeUpdated
err = client.Update(t.Context(), pvb)
require.NoError(t, err)
require.NotNil(t, informer.handler)
informer.handler.OnUpdate(oldPVB, pvb)
}
pvbs := backuper.WaitAllPodVolumesProcessed(logger)
+4 -19
View File
@@ -25,7 +25,6 @@ import (
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
appsv1api "k8s.io/api/apps/v1"
corev1api "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -39,7 +38,6 @@ import (
"github.com/vmware-tanzu/velero/pkg/builder"
"github.com/vmware-tanzu/velero/pkg/repository"
velerotest "github.com/vmware-tanzu/velero/pkg/test"
"github.com/vmware-tanzu/velero/pkg/util/kube"
)
func TestGetVolumesRepositoryType(t *testing.T) {
@@ -359,27 +357,14 @@ func TestRestorePodVolumes(t *testing.T) {
fakeKubeClient := kubefake.NewSimpleClientset(test.kubeClientObj...)
var kubeClient kubernetes.Interface = fakeKubeClient
fakeCRWatchClient := velerotest.NewFakeControllerRuntimeWatchClient(t, test.kubeClientObj...)
lw := kube.InternalLW{
Client: fakeCRWatchClient,
Namespace: velerov1api.DefaultNamespace,
ObjectList: new(velerov1api.PodVolumeRestoreList),
}
pvrInformer := cache.NewSharedIndexInformer(&lw, &velerov1api.PodVolumeBackup{}, 0, cache.Indexers{})
go pvrInformer.Run(ctx.Done())
require.True(t, cache.WaitForCacheSync(ctx.Done(), pvrInformer.HasSynced))
// This test verifies restore behavior itself, not informer sync/watch.
pvrInformer := cache.NewSharedIndexInformer(&cache.ListWatch{}, &velerov1api.PodVolumeRestore{}, 0, cache.Indexers{})
ensurer := repository.NewEnsurer(fakeCRClient, velerotest.NewLogger(), time.Millisecond)
restoreObj := builder.ForRestore(velerov1api.DefaultNamespace, "fake-restore").Result()
factory := NewRestorerFactory(repository.NewRepoLocker(), ensurer, kubeClient,
fakeCRClient, pvrInformer, velerotest.NewLogger())
rs, err := factory.NewRestorer(ctx, restoreObj)
require.NoError(t, err)
rs := newRestorer(ctx, repository.NewRepoLocker(), ensurer, pvrInformer, kubeClient, fakeCRClient, restoreObj, velerotest.NewLogger())
go func() {
if test.ctx != nil {
@@ -388,7 +373,7 @@ func TestRestorePodVolumes(t *testing.T) {
} else if test.retPVRs != nil {
time.Sleep(time.Second)
for _, pvr := range test.retPVRs {
rs.(*restorer).results[resultsKey(test.restoredPod.Namespace, test.restoredPod.Name)] <- pvr
rs.results[resultsKey(test.restoredPod.Namespace, test.restoredPod.Name)] <- pvr
}
}
}()
+4 -1
View File
@@ -19,6 +19,9 @@ package repository
import (
"fmt"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/stretchr/testify/assert"
"testing"
@@ -153,7 +156,7 @@ func TestGetBackupRepository(t *testing.T) {
if backupRepo != nil && tc.expected != nil {
backupRepo.ResourceVersion = tc.expected.ResourceVersion
require.Equal(t, *tc.expected, *backupRepo)
require.Empty(t, cmp.Diff(*tc.expected, *backupRepo, cmpopts.IgnoreFields(velerov1api.BackupRepository{}, "TypeMeta")))
} else {
require.Equal(t, tc.expected, backupRepo)
}
+2 -2
View File
@@ -359,7 +359,7 @@ func TestCancel(t *testing.T) {
err = pvcRIA.crClient.Get(t.Context(), crclient.ObjectKey{Namespace: tc.dataDownload.Namespace, Name: tc.dataDownload.Name}, resultDataDownload)
require.NoError(t, err)
require.True(t, cmp.Equal(tc.expectedDataDownload, *resultDataDownload, cmpopts.IgnoreFields(velerov2alpha1.DataDownload{}, "ResourceVersion", "Name")))
require.Empty(t, cmp.Diff(tc.expectedDataDownload, *resultDataDownload, cmpopts.IgnoreFields(velerov2alpha1.DataDownload{}, "TypeMeta", "ResourceVersion", "Name")))
})
}
}
@@ -523,7 +523,7 @@ func TestExecute(t *testing.T) {
LabelSelector: labels.SelectorFromSet(tc.expectedDataDownload.Labels),
})
require.NoError(t, err)
require.True(t, cmp.Equal(tc.expectedDataDownload, &dataDownloadList.Items[0], cmpopts.IgnoreFields(velerov2alpha1.DataDownload{}, "ResourceVersion", "Name")))
require.Empty(t, cmp.Diff(tc.expectedDataDownload, &dataDownloadList.Items[0], cmpopts.IgnoreFields(velerov2alpha1.DataDownload{}, "TypeMeta", "ResourceVersion", "Name")))
}
})
}
+2 -2
View File
@@ -117,7 +117,7 @@ func deleteHealthCheckNodePort(service *corev1api.Service) error {
continue
}
fields := new(map[string]any)
if err := json.Unmarshal(entry.FieldsV1.Raw, fields); err != nil {
if err := json.Unmarshal(entry.FieldsV1.GetRawBytes(), fields); err != nil {
return errors.WithStack(err)
}
@@ -222,7 +222,7 @@ func deleteNodePorts(service *corev1api.Service) error {
continue
}
fields := new(map[string]any)
if err := json.Unmarshal(entry.FieldsV1.Raw, fields); err != nil {
if err := json.Unmarshal(entry.FieldsV1.GetRawBytes(), fields); err != nil {
return errors.WithStack(err)
}
+1 -1
View File
@@ -107,7 +107,7 @@ func AssertDeepEqual(t *testing.T, expected, actual any) bool {
}
if !equality.Semantic.DeepEqual(expected, actual) {
s := diff.ObjectDiff(expected, actual)
s := diff.Diff(expected, actual)
return assert.Fail(t, fmt.Sprintf("Objects not equal:\n\n%s", s))
}
+88 -26
View File
@@ -16,40 +16,102 @@ limitations under the License.
package cbt
import "github.com/vmware-tanzu/velero/pkg/cbtservice"
import (
"math/bits"
// Bitmap defines the methods to store and iterate the CBT bitmap
type Bitmap interface {
// Set sets bits within the provided range
Set(cbtservice.Range)
"github.com/RoaringBitmap/roaring"
// SetFull sets all bits to the bitmap
SetFull()
"github.com/vmware-tanzu/velero/pkg/uploader/cbt/types"
)
// Snapshot returns snapshot of the bitmap
SourceID() string
const (
InvalidOffset64 = ^uint64(0)
)
// ChangeID returns the changeID of the bitmap
ChangeID() string
// Iterator returns the iterator for the CBT Bitmap
Iterator() Iterator
type bitmapImpl struct {
bitmap *roaring.Bitmap
blockSize uint
blockSizeLog int
length uint64
snapshot string
changeID string
volumeID string
}
// Iterator defines the methods to iterate the CBT bitmap and query the associated information
type Iterator interface {
// ChangeID returns the changeID of the bitmap
ChangeID() string
type bitmapIterator struct {
bitmapImpl
iterator roaring.IntPeekable
}
// Snapshot returns snapshot of the bitmap
Snapshot() string
func NewBitmap(blockSize uint, length uint64, snapshot string, changeID string, volumeID string) types.Bitmap {
return &bitmapImpl{
bitmap: roaring.New(),
blockSize: blockSize,
blockSizeLog: bits.Len(blockSize) - 1,
length: length,
snapshot: snapshot,
changeID: changeID,
volumeID: volumeID,
}
}
// BlockSize returns the granularity of the bitmap
BlockSize() int
func (c *bitmapImpl) Set(offset, length uint64) {
if offset >= c.length {
return
}
// Count returns the toal number of count in the bitmap
Count() uint64
if offset+length > c.length {
length = c.length - offset
}
// Next returns the offset of the next set block and whether it comes to the end of the iteration
Next() (int64, bool)
start := offset >> c.blockSizeLog
end := (offset + length + uint64(c.blockSize) - 1) >> c.blockSizeLog
c.bitmap.AddRange(start, end)
}
func (c *bitmapImpl) SetFull() {
start := uint64(0)
end := (c.length + uint64(c.blockSize) - 1) >> c.blockSizeLog
c.bitmap.AddRange(start, end)
}
func (c *bitmapImpl) Snapshot() string {
return c.snapshot
}
func (c *bitmapImpl) ChangeID() string {
return c.changeID
}
func (c *bitmapImpl) VolumeID() string {
return c.volumeID
}
func (c *bitmapImpl) Iterator() types.Iterator {
if c.bitmap == nil {
return nil
}
return &bitmapIterator{
bitmapImpl: *c,
iterator: c.bitmap.Iterator(),
}
}
func (c *bitmapIterator) Next() (uint64, bool) {
if !c.iterator.HasNext() {
return InvalidOffset64, false
}
return uint64(c.iterator.Next()) << c.blockSizeLog, true
}
func (c *bitmapIterator) Count() uint64 {
return c.bitmap.GetCardinality()
}
func (c *bitmapIterator) BlockSize() uint {
return c.blockSize
}
+256
View File
@@ -0,0 +1,256 @@
/*
Copyright The Velero Contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cbt
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestBitmapProperties(t *testing.T) {
b := NewBitmap(1024*1024, 10000*1024*1024, "snap-1", "change-1", "vol-1")
assert.Equal(t, "snap-1", b.Snapshot())
assert.Equal(t, "change-1", b.ChangeID())
assert.Equal(t, "vol-1", b.VolumeID())
}
func TestBitmapSet(t *testing.T) {
const mb = 1024 * 1024
const gb = 1024 * 1024 * 1024
tests := []struct {
name string
blockSize uint
totalLength uint64
setCalls []struct{ offset, length uint64 }
expectedCount uint64
expectedNext []uint64
}{
{
name: "set single block within bounds",
blockSize: mb,
totalLength: 10 * gb,
setCalls: []struct{ offset, length uint64 }{
{0, 1000},
},
expectedCount: 1,
expectedNext: []uint64{0},
},
{
name: "set exactly one block",
blockSize: mb,
totalLength: 10 * gb,
setCalls: []struct{ offset, length uint64 }{
{0, mb},
},
expectedCount: 1,
expectedNext: []uint64{0},
},
{
name: "set overlapping two blocks",
blockSize: mb,
totalLength: 10 * gb,
setCalls: []struct{ offset, length uint64 }{
{mb - 1, 2},
},
expectedCount: 2,
expectedNext: []uint64{0, mb},
},
{
name: "set multiple non-contiguous blocks",
blockSize: mb,
totalLength: 20 * gb,
setCalls: []struct{ offset, length uint64 }{
{0, 100},
{2 * mb, 100},
},
expectedCount: 2,
expectedNext: []uint64{0, 2 * mb},
},
{
name: "set completely out of bounds (offset >= length)",
blockSize: mb,
totalLength: 10 * gb,
setCalls: []struct{ offset, length uint64 }{
{10 * gb, 100},
{15 * gb, 100},
},
expectedCount: 0,
expectedNext: []uint64{},
},
{
name: "set partially out of bounds (truncated)",
blockSize: mb,
totalLength: 10 * gb,
setCalls: []struct{ offset, length uint64 }{
{10*gb - mb/2, mb}, // Starts in the last block, length pushes it out of bounds
},
expectedCount: 1, // Only the last block should be set
expectedNext: []uint64{10*gb - mb},
},
{
name: "set spanning entire length",
blockSize: mb,
totalLength: 3 * mb, // 3 blocks: 0-1MB, 1MB-2MB, 2MB-3MB
setCalls: []struct{ offset, length uint64 }{
{0, 3 * mb},
},
expectedCount: 3,
expectedNext: []uint64{0, mb, 2 * mb},
},
{
name: "set large contiguous range",
blockSize: mb,
totalLength: 100 * gb,
setCalls: []struct{ offset, length uint64 }{
{10 * mb, 5 * mb}, // Starts at 10MB, spans 5 full blocks
},
expectedCount: 5,
expectedNext: []uint64{10 * mb, 11 * mb, 12 * mb, 13 * mb, 14 * mb},
},
{
name: "set empty length",
blockSize: mb,
totalLength: 10 * gb,
setCalls: []struct{ offset, length uint64 }{
{mb, 0},
},
expectedCount: 0,
expectedNext: []uint64{},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
b := NewBitmap(tt.blockSize, tt.totalLength, "snap-1", "change-1", "vol-1")
for _, call := range tt.setCalls {
b.Set(call.offset, call.length)
}
iter := b.Iterator()
require.NotNil(t, iter)
assert.Equal(t, tt.expectedCount, iter.Count())
var actualNext []uint64
for {
offset, hasNext := iter.Next()
if !hasNext {
assert.Equal(t, InvalidOffset64, offset)
break
}
actualNext = append(actualNext, offset)
}
if len(tt.expectedNext) == 0 {
assert.Empty(t, actualNext)
} else {
assert.Equal(t, tt.expectedNext, actualNext)
}
})
}
}
func TestBitmapSetFull(t *testing.T) {
const mb = 1024 * 1024
// Total length 3MB, blockSize 1MB. This means 3 blocks total:
// block 0: 0 - 1MB
// block 1: 1MB - 2MB
// block 2: 2MB - 3MB
b := NewBitmap(mb, 3*mb, "snap-1", "change-1", "vol-1")
b.SetFull()
iter := b.Iterator()
require.NotNil(t, iter)
assert.Equal(t, uint64(3), iter.Count())
expectedOffsets := []uint64{0, mb, 2 * mb}
var actualOffsets []uint64
for {
offset, hasNext := iter.Next()
if !hasNext {
break
}
actualOffsets = append(actualOffsets, offset)
}
assert.Equal(t, expectedOffsets, actualOffsets)
}
func TestBitmapIterator(t *testing.T) {
const mb = 1024 * 1024
const gb = 1024 * 1024 * 1024
b := NewBitmap(mb, 10*gb, "snap-1", "change-1", "vol-1")
// Set multiple ranges to test contiguous iteration
b.Set(mb, 100) // Block 1
b.Set(3*mb, 5*mb) // Blocks 3, 4, 5, 6, 7
b.Set(10*gb-mb, mb) // Last block
iter := b.Iterator()
require.NotNil(t, iter)
// Test iterator properties
assert.Equal(t, "snap-1", iter.Snapshot())
assert.Equal(t, "change-1", iter.ChangeID())
assert.Equal(t, "vol-1", iter.VolumeID())
assert.Equal(t, uint(mb), iter.BlockSize())
assert.Equal(t, uint64(7), iter.Count()) // 1 + 5 + 1 = 7 blocks
expectedOffsets := []uint64{
mb,
3 * mb, 4 * mb, 5 * mb, 6 * mb, 7 * mb,
10*gb - mb,
}
// Test iteration
var actualOffsets []uint64
for {
offset, hasNext := iter.Next()
if !hasNext {
assert.Equal(t, InvalidOffset64, offset)
break
}
actualOffsets = append(actualOffsets, offset)
}
assert.Equal(t, expectedOffsets, actualOffsets)
// Test end of iteration multiple times to ensure it stays exhausted
offset, hasNext := iter.Next()
assert.False(t, hasNext)
assert.Equal(t, InvalidOffset64, offset)
offset, hasNext = iter.Next()
assert.False(t, hasNext)
assert.Equal(t, InvalidOffset64, offset)
}
func TestBitmapIteratorNilBitmap(t *testing.T) {
// Directly create bitmapImpl with a nil roaring.Bitmap to test safety
b := &bitmapImpl{
bitmap: nil,
}
iter := b.Iterator()
assert.Nil(t, iter)
}
+31 -19
View File
@@ -19,31 +19,43 @@ package cbt
import (
"context"
"github.com/pkg/errors"
"github.com/vmware-tanzu/velero/pkg/cbtservice"
"github.com/vmware-tanzu/velero/pkg/uploader/cbt/types"
)
// SetBitmapOrFull translates the allocated/changed blocks from CBT service to the given bitmap or set the bitmap to full when error happens
func SetBitmapOrFull(ctx context.Context, service cbtservice.Service, bitmap Bitmap) error {
var err error
func SetBitmapOrFull(ctx context.Context, service cbtservice.Service, bitmap types.Bitmap) (err error) {
defer func() {
if err != nil {
bitmap.SetFull()
}
}()
if service == nil {
return errors.New("CBT service is absent")
}
if bitmap.Snapshot() == "" {
return errors.New("invalid snapshot")
}
if bitmap.ChangeID() == "" {
err = setFromAllocatedBlocks(ctx, service, bitmap)
} else {
err = setFromChangedBlocks(ctx, service, bitmap)
return errors.Wrapf(service.GetAllocatedBlocks(ctx, bitmap.Snapshot(), func(blocks []cbtservice.Range) error {
for _, b := range blocks {
bitmap.Set(b.Offset, b.Length)
}
return nil
}), "error getting allocated blocks from CBT service")
}
if err != nil {
bitmap.SetFull()
}
return errors.Wrapf(service.GetChangedBlocks(ctx, bitmap.Snapshot(), bitmap.ChangeID(), func(blocks []cbtservice.Range) error {
for _, b := range blocks {
bitmap.Set(b.Offset, b.Length)
}
return err
}
// TODO implement in following PRs
func setFromAllocatedBlocks(_ context.Context, _ cbtservice.Service, _ Bitmap) error {
return nil
}
// TODO implement in following PRs
func setFromChangedBlocks(_ context.Context, _ cbtservice.Service, _ Bitmap) error {
return nil
return nil
}), "error getting changed blocks from CBT service")
}
+142
View File
@@ -0,0 +1,142 @@
/*
Copyright The Velero Contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cbt
import (
"context"
"errors"
"testing"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"github.com/vmware-tanzu/velero/pkg/cbtservice"
cbtservicemocks "github.com/vmware-tanzu/velero/pkg/cbtservice/mocks"
cbtmocks "github.com/vmware-tanzu/velero/pkg/uploader/cbt/types/mocks"
)
func TestSetBitmapOrFull(t *testing.T) {
tests := []struct {
name string
nilService bool
setupMocks func(*cbtservicemocks.Service, *cbtmocks.Bitmap)
expectedErrStr string
}{
{
name: "nil service",
nilService: true,
setupMocks: func(svc *cbtservicemocks.Service, bmp *cbtmocks.Bitmap) {
bmp.On("SetFull").Return()
},
expectedErrStr: "CBT service is absent",
},
{
name: "invalid snapshot",
setupMocks: func(svc *cbtservicemocks.Service, bmp *cbtmocks.Bitmap) {
bmp.On("Snapshot").Return("")
bmp.On("SetFull").Return()
},
expectedErrStr: "invalid snapshot",
},
{
name: "allocated blocks success",
setupMocks: func(svc *cbtservicemocks.Service, bmp *cbtmocks.Bitmap) {
bmp.On("Snapshot").Return("snap-1")
bmp.On("ChangeID").Return("")
svc.On("GetAllocatedBlocks", mock.Anything, "snap-1", mock.Anything).Run(func(args mock.Arguments) {
record := args.Get(2).(func([]cbtservice.Range) error)
record([]cbtservice.Range{
{Offset: 0, Length: 4096},
{Offset: 8192, Length: 4096},
})
}).Return(nil)
bmp.On("Set", uint64(0), uint64(4096)).Return()
bmp.On("Set", uint64(8192), uint64(4096)).Return()
},
},
{
name: "allocated blocks error",
setupMocks: func(svc *cbtservicemocks.Service, bmp *cbtmocks.Bitmap) {
bmp.On("Snapshot").Return("snap-1")
bmp.On("ChangeID").Return("")
svc.On("GetAllocatedBlocks", mock.Anything, "snap-1", mock.Anything).Return(errors.New("mock alloc error"))
bmp.On("SetFull").Return()
},
expectedErrStr: "error getting allocated blocks from CBT service: mock alloc error",
},
{
name: "changed blocks success",
setupMocks: func(svc *cbtservicemocks.Service, bmp *cbtmocks.Bitmap) {
bmp.On("Snapshot").Return("snap-1")
bmp.On("ChangeID").Return("change-1")
svc.On("GetChangedBlocks", mock.Anything, "snap-1", "change-1", mock.Anything).Run(func(args mock.Arguments) {
record := args.Get(3).(func([]cbtservice.Range) error)
record([]cbtservice.Range{
{Offset: 4096, Length: 4096},
})
}).Return(nil)
bmp.On("Set", uint64(4096), uint64(4096)).Return()
},
},
{
name: "changed blocks error",
setupMocks: func(svc *cbtservicemocks.Service, bmp *cbtmocks.Bitmap) {
bmp.On("Snapshot").Return("snap-1")
bmp.On("ChangeID").Return("change-1")
svc.On("GetChangedBlocks", mock.Anything, "snap-1", "change-1", mock.Anything).Return(errors.New("mock changed error"))
bmp.On("SetFull").Return()
},
expectedErrStr: "error getting changed blocks from CBT service: mock changed error",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
svcMock := new(cbtservicemocks.Service)
bmpMock := new(cbtmocks.Bitmap)
if tt.setupMocks != nil {
tt.setupMocks(svcMock, bmpMock)
}
var svc cbtservice.Service
if !tt.nilService {
svc = svcMock
}
err := SetBitmapOrFull(context.Background(), svc, bmpMock)
if tt.expectedErrStr != "" {
require.Error(t, err)
require.EqualError(t, err, tt.expectedErrStr)
} else {
require.NoError(t, err)
}
if !tt.nilService {
svcMock.AssertExpectations(t)
}
bmpMock.AssertExpectations(t)
})
}
}
+294
View File
@@ -0,0 +1,294 @@
// Code generated by mockery; DO NOT EDIT.
// github.com/vektra/mockery
// template: testify
package mocks
import (
mock "github.com/stretchr/testify/mock"
"github.com/vmware-tanzu/velero/pkg/uploader/cbt/types"
)
// NewBitmap creates a new instance of Bitmap. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func NewBitmap(t interface {
mock.TestingT
Cleanup(func())
}) *Bitmap {
mock := &Bitmap{}
mock.Mock.Test(t)
t.Cleanup(func() { mock.AssertExpectations(t) })
return mock
}
// Bitmap is an autogenerated mock type for the Bitmap type
type Bitmap struct {
mock.Mock
}
type Bitmap_Expecter struct {
mock *mock.Mock
}
func (_m *Bitmap) EXPECT() *Bitmap_Expecter {
return &Bitmap_Expecter{mock: &_m.Mock}
}
// ChangeID provides a mock function for the type Bitmap
func (_mock *Bitmap) ChangeID() string {
ret := _mock.Called()
if len(ret) == 0 {
panic("no return value specified for ChangeID")
}
var r0 string
if returnFunc, ok := ret.Get(0).(func() string); ok {
r0 = returnFunc()
} else {
r0 = ret.Get(0).(string)
}
return r0
}
// Bitmap_ChangeID_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ChangeID'
type Bitmap_ChangeID_Call struct {
*mock.Call
}
// ChangeID is a helper method to define mock.On call
func (_e *Bitmap_Expecter) ChangeID() *Bitmap_ChangeID_Call {
return &Bitmap_ChangeID_Call{Call: _e.mock.On("ChangeID")}
}
func (_c *Bitmap_ChangeID_Call) Run(run func()) *Bitmap_ChangeID_Call {
_c.Call.Run(func(args mock.Arguments) {
run()
})
return _c
}
func (_c *Bitmap_ChangeID_Call) Return(s string) *Bitmap_ChangeID_Call {
_c.Call.Return(s)
return _c
}
func (_c *Bitmap_ChangeID_Call) RunAndReturn(run func() string) *Bitmap_ChangeID_Call {
_c.Call.Return(run)
return _c
}
// Iterator provides a mock function for the type Bitmap
func (_mock *Bitmap) Iterator() types.Iterator {
ret := _mock.Called()
if len(ret) == 0 {
panic("no return value specified for Iterator")
}
var r0 types.Iterator
if returnFunc, ok := ret.Get(0).(func() types.Iterator); ok {
r0 = returnFunc()
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(types.Iterator)
}
}
return r0
}
// Bitmap_Iterator_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Iterator'
type Bitmap_Iterator_Call struct {
*mock.Call
}
// Iterator is a helper method to define mock.On call
func (_e *Bitmap_Expecter) Iterator() *Bitmap_Iterator_Call {
return &Bitmap_Iterator_Call{Call: _e.mock.On("Iterator")}
}
func (_c *Bitmap_Iterator_Call) Run(run func()) *Bitmap_Iterator_Call {
_c.Call.Run(func(args mock.Arguments) {
run()
})
return _c
}
func (_c *Bitmap_Iterator_Call) Return(iterator types.Iterator) *Bitmap_Iterator_Call {
_c.Call.Return(iterator)
return _c
}
func (_c *Bitmap_Iterator_Call) RunAndReturn(run func() types.Iterator) *Bitmap_Iterator_Call {
_c.Call.Return(run)
return _c
}
// Set provides a mock function for the type Bitmap
func (_mock *Bitmap) Set(v uint64, v1 uint64) {
_mock.Called(v, v1)
return
}
// Bitmap_Set_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Set'
type Bitmap_Set_Call struct {
*mock.Call
}
// Set is a helper method to define mock.On call
// - v uint64
// - v1 uint64
func (_e *Bitmap_Expecter) Set(v interface{}, v1 interface{}) *Bitmap_Set_Call {
return &Bitmap_Set_Call{Call: _e.mock.On("Set", v, v1)}
}
func (_c *Bitmap_Set_Call) Run(run func(v uint64, v1 uint64)) *Bitmap_Set_Call {
_c.Call.Run(func(args mock.Arguments) {
var arg0 uint64
if args[0] != nil {
arg0 = args[0].(uint64)
}
var arg1 uint64
if args[1] != nil {
arg1 = args[1].(uint64)
}
run(
arg0,
arg1,
)
})
return _c
}
func (_c *Bitmap_Set_Call) Return() *Bitmap_Set_Call {
_c.Call.Return()
return _c
}
func (_c *Bitmap_Set_Call) RunAndReturn(run func(v uint64, v1 uint64)) *Bitmap_Set_Call {
_c.Run(run)
return _c
}
// SetFull provides a mock function for the type Bitmap
func (_mock *Bitmap) SetFull() {
_mock.Called()
return
}
// Bitmap_SetFull_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetFull'
type Bitmap_SetFull_Call struct {
*mock.Call
}
// SetFull is a helper method to define mock.On call
func (_e *Bitmap_Expecter) SetFull() *Bitmap_SetFull_Call {
return &Bitmap_SetFull_Call{Call: _e.mock.On("SetFull")}
}
func (_c *Bitmap_SetFull_Call) Run(run func()) *Bitmap_SetFull_Call {
_c.Call.Run(func(args mock.Arguments) {
run()
})
return _c
}
func (_c *Bitmap_SetFull_Call) Return() *Bitmap_SetFull_Call {
_c.Call.Return()
return _c
}
func (_c *Bitmap_SetFull_Call) RunAndReturn(run func()) *Bitmap_SetFull_Call {
_c.Run(run)
return _c
}
// Snapshot provides a mock function for the type Bitmap
func (_mock *Bitmap) Snapshot() string {
ret := _mock.Called()
if len(ret) == 0 {
panic("no return value specified for Snapshot")
}
var r0 string
if returnFunc, ok := ret.Get(0).(func() string); ok {
r0 = returnFunc()
} else {
r0 = ret.Get(0).(string)
}
return r0
}
// Bitmap_Snapshot_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Snapshot'
type Bitmap_Snapshot_Call struct {
*mock.Call
}
// Snapshot is a helper method to define mock.On call
func (_e *Bitmap_Expecter) Snapshot() *Bitmap_Snapshot_Call {
return &Bitmap_Snapshot_Call{Call: _e.mock.On("Snapshot")}
}
func (_c *Bitmap_Snapshot_Call) Run(run func()) *Bitmap_Snapshot_Call {
_c.Call.Run(func(args mock.Arguments) {
run()
})
return _c
}
func (_c *Bitmap_Snapshot_Call) Return(s string) *Bitmap_Snapshot_Call {
_c.Call.Return(s)
return _c
}
func (_c *Bitmap_Snapshot_Call) RunAndReturn(run func() string) *Bitmap_Snapshot_Call {
_c.Call.Return(run)
return _c
}
// VolumeID provides a mock function for the type Bitmap
func (_mock *Bitmap) VolumeID() string {
ret := _mock.Called()
if len(ret) == 0 {
panic("no return value specified for VolumeID")
}
var r0 string
if returnFunc, ok := ret.Get(0).(func() string); ok {
r0 = returnFunc()
} else {
r0 = ret.Get(0).(string)
}
return r0
}
// Bitmap_VolumeID_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'VolumeID'
type Bitmap_VolumeID_Call struct {
*mock.Call
}
// VolumeID is a helper method to define mock.On call
func (_e *Bitmap_Expecter) VolumeID() *Bitmap_VolumeID_Call {
return &Bitmap_VolumeID_Call{Call: _e.mock.On("VolumeID")}
}
func (_c *Bitmap_VolumeID_Call) Run(run func()) *Bitmap_VolumeID_Call {
_c.Call.Run(func(args mock.Arguments) {
run()
})
return _c
}
func (_c *Bitmap_VolumeID_Call) Return(s string) *Bitmap_VolumeID_Call {
_c.Call.Return(s)
return _c
}
func (_c *Bitmap_VolumeID_Call) RunAndReturn(run func() string) *Bitmap_VolumeID_Call {
_c.Call.Return(run)
return _c
}
+309
View File
@@ -0,0 +1,309 @@
// Code generated by mockery; DO NOT EDIT.
// github.com/vektra/mockery
// template: testify
package mocks
import (
mock "github.com/stretchr/testify/mock"
)
// NewIterator creates a new instance of Iterator. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func NewIterator(t interface {
mock.TestingT
Cleanup(func())
}) *Iterator {
mock := &Iterator{}
mock.Mock.Test(t)
t.Cleanup(func() { mock.AssertExpectations(t) })
return mock
}
// Iterator is an autogenerated mock type for the Iterator type
type Iterator struct {
mock.Mock
}
type Iterator_Expecter struct {
mock *mock.Mock
}
func (_m *Iterator) EXPECT() *Iterator_Expecter {
return &Iterator_Expecter{mock: &_m.Mock}
}
// BlockSize provides a mock function for the type Iterator
func (_mock *Iterator) BlockSize() uint {
ret := _mock.Called()
if len(ret) == 0 {
panic("no return value specified for BlockSize")
}
var r0 uint
if returnFunc, ok := ret.Get(0).(func() uint); ok {
r0 = returnFunc()
} else {
r0 = ret.Get(0).(uint)
}
return r0
}
// Iterator_BlockSize_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'BlockSize'
type Iterator_BlockSize_Call struct {
*mock.Call
}
// BlockSize is a helper method to define mock.On call
func (_e *Iterator_Expecter) BlockSize() *Iterator_BlockSize_Call {
return &Iterator_BlockSize_Call{Call: _e.mock.On("BlockSize")}
}
func (_c *Iterator_BlockSize_Call) Run(run func()) *Iterator_BlockSize_Call {
_c.Call.Run(func(args mock.Arguments) {
run()
})
return _c
}
func (_c *Iterator_BlockSize_Call) Return(v uint) *Iterator_BlockSize_Call {
_c.Call.Return(v)
return _c
}
func (_c *Iterator_BlockSize_Call) RunAndReturn(run func() uint) *Iterator_BlockSize_Call {
_c.Call.Return(run)
return _c
}
// ChangeID provides a mock function for the type Iterator
func (_mock *Iterator) ChangeID() string {
ret := _mock.Called()
if len(ret) == 0 {
panic("no return value specified for ChangeID")
}
var r0 string
if returnFunc, ok := ret.Get(0).(func() string); ok {
r0 = returnFunc()
} else {
r0 = ret.Get(0).(string)
}
return r0
}
// Iterator_ChangeID_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ChangeID'
type Iterator_ChangeID_Call struct {
*mock.Call
}
// ChangeID is a helper method to define mock.On call
func (_e *Iterator_Expecter) ChangeID() *Iterator_ChangeID_Call {
return &Iterator_ChangeID_Call{Call: _e.mock.On("ChangeID")}
}
func (_c *Iterator_ChangeID_Call) Run(run func()) *Iterator_ChangeID_Call {
_c.Call.Run(func(args mock.Arguments) {
run()
})
return _c
}
func (_c *Iterator_ChangeID_Call) Return(s string) *Iterator_ChangeID_Call {
_c.Call.Return(s)
return _c
}
func (_c *Iterator_ChangeID_Call) RunAndReturn(run func() string) *Iterator_ChangeID_Call {
_c.Call.Return(run)
return _c
}
// Count provides a mock function for the type Iterator
func (_mock *Iterator) Count() uint64 {
ret := _mock.Called()
if len(ret) == 0 {
panic("no return value specified for Count")
}
var r0 uint64
if returnFunc, ok := ret.Get(0).(func() uint64); ok {
r0 = returnFunc()
} else {
r0 = ret.Get(0).(uint64)
}
return r0
}
// Iterator_Count_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Count'
type Iterator_Count_Call struct {
*mock.Call
}
// Count is a helper method to define mock.On call
func (_e *Iterator_Expecter) Count() *Iterator_Count_Call {
return &Iterator_Count_Call{Call: _e.mock.On("Count")}
}
func (_c *Iterator_Count_Call) Run(run func()) *Iterator_Count_Call {
_c.Call.Run(func(args mock.Arguments) {
run()
})
return _c
}
func (_c *Iterator_Count_Call) Return(v uint64) *Iterator_Count_Call {
_c.Call.Return(v)
return _c
}
func (_c *Iterator_Count_Call) RunAndReturn(run func() uint64) *Iterator_Count_Call {
_c.Call.Return(run)
return _c
}
// Next provides a mock function for the type Iterator
func (_mock *Iterator) Next() (uint64, bool) {
ret := _mock.Called()
if len(ret) == 0 {
panic("no return value specified for Next")
}
var r0 uint64
var r1 bool
if returnFunc, ok := ret.Get(0).(func() (uint64, bool)); ok {
return returnFunc()
}
if returnFunc, ok := ret.Get(0).(func() uint64); ok {
r0 = returnFunc()
} else {
r0 = ret.Get(0).(uint64)
}
if returnFunc, ok := ret.Get(1).(func() bool); ok {
r1 = returnFunc()
} else {
r1 = ret.Get(1).(bool)
}
return r0, r1
}
// Iterator_Next_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Next'
type Iterator_Next_Call struct {
*mock.Call
}
// Next is a helper method to define mock.On call
func (_e *Iterator_Expecter) Next() *Iterator_Next_Call {
return &Iterator_Next_Call{Call: _e.mock.On("Next")}
}
func (_c *Iterator_Next_Call) Run(run func()) *Iterator_Next_Call {
_c.Call.Run(func(args mock.Arguments) {
run()
})
return _c
}
func (_c *Iterator_Next_Call) Return(v uint64, b bool) *Iterator_Next_Call {
_c.Call.Return(v, b)
return _c
}
func (_c *Iterator_Next_Call) RunAndReturn(run func() (uint64, bool)) *Iterator_Next_Call {
_c.Call.Return(run)
return _c
}
// Snapshot provides a mock function for the type Iterator
func (_mock *Iterator) Snapshot() string {
ret := _mock.Called()
if len(ret) == 0 {
panic("no return value specified for Snapshot")
}
var r0 string
if returnFunc, ok := ret.Get(0).(func() string); ok {
r0 = returnFunc()
} else {
r0 = ret.Get(0).(string)
}
return r0
}
// Iterator_Snapshot_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Snapshot'
type Iterator_Snapshot_Call struct {
*mock.Call
}
// Snapshot is a helper method to define mock.On call
func (_e *Iterator_Expecter) Snapshot() *Iterator_Snapshot_Call {
return &Iterator_Snapshot_Call{Call: _e.mock.On("Snapshot")}
}
func (_c *Iterator_Snapshot_Call) Run(run func()) *Iterator_Snapshot_Call {
_c.Call.Run(func(args mock.Arguments) {
run()
})
return _c
}
func (_c *Iterator_Snapshot_Call) Return(s string) *Iterator_Snapshot_Call {
_c.Call.Return(s)
return _c
}
func (_c *Iterator_Snapshot_Call) RunAndReturn(run func() string) *Iterator_Snapshot_Call {
_c.Call.Return(run)
return _c
}
// VolumeID provides a mock function for the type Iterator
func (_mock *Iterator) VolumeID() string {
ret := _mock.Called()
if len(ret) == 0 {
panic("no return value specified for VolumeID")
}
var r0 string
if returnFunc, ok := ret.Get(0).(func() string); ok {
r0 = returnFunc()
} else {
r0 = ret.Get(0).(string)
}
return r0
}
// Iterator_VolumeID_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'VolumeID'
type Iterator_VolumeID_Call struct {
*mock.Call
}
// VolumeID is a helper method to define mock.On call
func (_e *Iterator_Expecter) VolumeID() *Iterator_VolumeID_Call {
return &Iterator_VolumeID_Call{Call: _e.mock.On("VolumeID")}
}
func (_c *Iterator_VolumeID_Call) Run(run func()) *Iterator_VolumeID_Call {
_c.Call.Run(func(args mock.Arguments) {
run()
})
return _c
}
func (_c *Iterator_VolumeID_Call) Return(s string) *Iterator_VolumeID_Call {
_c.Call.Return(s)
return _c
}
func (_c *Iterator_VolumeID_Call) RunAndReturn(run func() string) *Iterator_VolumeID_Call {
_c.Call.Return(run)
return _c
}
+59
View File
@@ -0,0 +1,59 @@
/*
Copyright The Velero Contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package types
// Bitmap defines the methods to store and iterate the CBT bitmap
type Bitmap interface {
// Set sets bits within the provided range
Set(uint64, uint64)
// SetFull sets all bits to the bitmap
SetFull()
// Snapshot returns snapshot of the bitmap
Snapshot() string
// ChangeID returns the changeID of the bitmap
ChangeID() string
// VolumeID return ID of the volume from which the snapshot is taken
VolumeID() string
// Iterator returns the iterator for the CBT Bitmap
Iterator() Iterator
}
// Iterator defines the methods to iterate the CBT bitmap and query the associated information
type Iterator interface {
// ChangeID returns the changeID of the bitmap
ChangeID() string
// Snapshot returns snapshot of the bitmap
Snapshot() string
// VolumeID return ID of the volume from which the snapshot is taken
VolumeID() string
// BlockSize returns the granularity of the bitmap
BlockSize() uint
// Count returns the total number of count in the bitmap
Count() uint64
// Next returns the offset of the next set block and whether it comes to the end of the iteration
Next() (uint64, bool)
}
@@ -173,7 +173,6 @@ func (nie *NamespaceIncludesExcludes) ExpandIncludesExcludes() error {
}
// ResolveNamespaceList returns a list of all namespaces which will be backed up.
// The second return value indicates whether wildcard expansion was performed.
func (nie *NamespaceIncludesExcludes) ResolveNamespaceList() ([]string, error) {
// Check if this is being called by non-backup processing e.g. backup queue controller
if !nie.wildcardExpanded {
+1 -1
View File
@@ -2004,7 +2004,7 @@ func TestGetVSCForVS(t *testing.T) {
}
if tc.expectedVSC != nil {
require.True(t, cmp.Equal(tc.expectedVSC, vsc, cmpopts.IgnoreFields(snapshotv1api.VolumeSnapshotContent{}, "ResourceVersion")))
require.Empty(t, cmp.Diff(tc.expectedVSC, vsc, cmpopts.IgnoreFields(snapshotv1api.VolumeSnapshotContent{}, "TypeMeta", "ResourceVersion")))
}
})
}
+30 -9
View File
@@ -1,4 +1,4 @@
// Code generated by mockery v2.42.1. DO NOT EDIT.
// Code generated by mockery v2.53.5. DO NOT EDIT.
package mocks
@@ -7,14 +7,10 @@ import (
client "sigs.k8s.io/controller-runtime/pkg/client"
meta "k8s.io/apimachinery/pkg/api/meta"
mock "github.com/stretchr/testify/mock"
meta "k8s.io/apimachinery/pkg/api/meta"
runtime "k8s.io/apimachinery/pkg/runtime"
schema "k8s.io/apimachinery/pkg/runtime/schema"
types "k8s.io/apimachinery/pkg/types"
)
@@ -23,6 +19,31 @@ type Client struct {
mock.Mock
}
// Apply provides a mock function with given fields: ctx, obj, opts
func (_m *Client) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...client.ApplyOption) error {
_va := make([]interface{}, len(opts))
for _i := range opts {
_va[_i] = opts[_i]
}
var _ca []interface{}
_ca = append(_ca, ctx, obj)
_ca = append(_ca, _va...)
ret := _m.Called(_ca...)
if len(ret) == 0 {
panic("no return value specified for Apply")
}
var r0 error
if rf, ok := ret.Get(0).(func(context.Context, runtime.ApplyConfiguration, ...client.ApplyOption) error); ok {
r0 = rf(ctx, obj, opts...)
} else {
r0 = ret.Error(0)
}
return r0
}
// Create provides a mock function with given fields: ctx, obj, opts
func (_m *Client) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error {
_va := make([]interface{}, len(opts))
@@ -229,7 +250,7 @@ func (_m *Client) Patch(ctx context.Context, obj client.Object, patch client.Pat
return r0
}
// RESTMapper provides a mock function with given fields:
// RESTMapper provides a mock function with no fields
func (_m *Client) RESTMapper() meta.RESTMapper {
ret := _m.Called()
@@ -249,7 +270,7 @@ func (_m *Client) RESTMapper() meta.RESTMapper {
return r0
}
// Scheme provides a mock function with given fields:
// Scheme provides a mock function with no fields
func (_m *Client) Scheme() *runtime.Scheme {
ret := _m.Called()
@@ -269,7 +290,7 @@ func (_m *Client) Scheme() *runtime.Scheme {
return r0
}
// Status provides a mock function with given fields:
// Status provides a mock function with no fields
func (_m *Client) Status() client.SubResourceWriter {
ret := _m.Called()
+12 -15
View File
@@ -17,6 +17,7 @@ limitations under the License.
package kube
import (
"sort"
"testing"
"time"
@@ -24,9 +25,7 @@ import (
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/require"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/util/workqueue"
crclient "sigs.k8s.io/controller-runtime/pkg/client"
@@ -91,8 +90,6 @@ func TestPredicate(t *testing.T) {
},
)
require.NoError(t, source.Start(ctx, queue))
// Should not patch a backup storage location object status phase
// if the location's validation frequency is specifically set to zero
require.NoError(t, client.Create(ctx, &velerov1.BackupStorageLocation{
@@ -107,6 +104,8 @@ func TestPredicate(t *testing.T) {
LastValidationTime: &metav1.Time{Time: time.Now()},
},
}))
require.NoError(t, source.Start(ctx, queue))
time.Sleep(2 * time.Second)
require.Equal(t, 0, queue.Len())
@@ -127,24 +126,20 @@ func TestOrder(t *testing.T) {
1*time.Second,
PeriodicalEnqueueSourceOption{
OrderFunc: func(objList crclient.ObjectList) crclient.ObjectList {
locationList := &velerov1.BackupStorageLocationList{}
objArray := make([]runtime.Object, 0)
locationList, ok := objList.(*velerov1.BackupStorageLocationList)
if !ok {
return objList
}
// Generate BSL array.
locations, _ := meta.ExtractList(objList)
// Move default BSL to tail of array.
objArray = append(objArray, locations[1])
objArray = append(objArray, locations[0])
meta.SetList(locationList, objArray)
sort.SliceStable(locationList.Items, func(i, j int) bool {
return locationList.Items[i].Spec.Default && !locationList.Items[j].Spec.Default
})
return locationList
},
},
)
require.NoError(t, source.Start(ctx, queue))
// Should not patch a backup storage location object status phase
// if the location's validation frequency is specifically set to zero
require.NoError(t, client.Create(ctx, &velerov1.BackupStorageLocation{
@@ -172,6 +167,8 @@ func TestOrder(t *testing.T) {
LastValidationTime: &metav1.Time{Time: time.Now()},
},
}))
require.NoError(t, source.Start(ctx, queue))
time.Sleep(2 * time.Second)
first, _ := queue.Get()
+5
View File
@@ -9,6 +9,11 @@ import (
)
func ShouldExpandWildcards(includes []string, excludes []string) bool {
// Empty includes is equivalent to * (match all) - don't expand
if len(includes) == 0 {
return false
}
wildcardFound := false
for _, include := range includes {
// Special case: "*" alone means "match all" - don't expand
+6
View File
@@ -68,6 +68,12 @@ func TestShouldExpandWildcards(t *testing.T) {
excludes: []string{},
expected: false,
},
{
name: "empty includes with wildcard excludes - should not expand",
includes: []string{},
excludes: []string{"ns*"},
expected: false,
},
{
name: "complex wildcard patterns",
includes: []string{"*-prod"},