diff --git a/.github/workflows/e2e-test-kind.yaml b/.github/workflows/e2e-test-kind.yaml index 37bf72092..7dcd51408 100644 --- a/.github/workflows/e2e-test-kind.yaml +++ b/.github/workflows/e2e-test-kind.yaml @@ -95,9 +95,9 @@ jobs: \"k8s\":$(wget -q -O - "https://hub.docker.com/v2/namespaces/kindest/repositories/node/tags?page_size=50" | grep -o '"name": *"[^"]*' | grep -o '[^"]*$' | grep -v -E "alpha|beta" | grep -E "v[1-9]\.(2[5-9]|[3-9][0-9])" | awk -F. '{if(!a[$1"."$2]++)print $1"."$2"."$NF}' | sort -r | sed s/v//g | jq -R -c -s 'split("\n")[:-1]'),\ \"labels\":[\ \"Basic && (ClusterResource || NodePort || StorageClass)\", \ - \"ResourceFiltering && !Restic\", \ + \"ResourceFiltering && !FSBackup\", \ \"ResourceModifier || (Backups && BackupsSync) || PrivilegesMgmt || OrderedResources\", \ - \"(NamespaceMapping && Single && Restic) || (NamespaceMapping && Multiple && Restic)\"\ + \"(NamespaceMapping && Single && FSBackup) || (NamespaceMapping && Multiple && FSBackup)\"\ ]}" >> $GITHUB_OUTPUT # Run E2E test against all Kubernetes versions on kind @@ -136,7 +136,7 @@ jobs: - uses: engineerd/setup-kind@v0.6.2 with: skipClusterLogsExport: true - version: "v0.27.0" + version: "v0.32.0" image: "kindest/node:v${{ matrix.k8s }}" - name: Fetch built CLI id: cli-cache diff --git a/.github/workflows/stale-issues.yml b/.github/workflows/stale-issues.yml index 58e1ca89f..0dffc96c4 100644 --- a/.github/workflows/stale-issues.yml +++ b/.github/workflows/stale-issues.yml @@ -20,4 +20,4 @@ jobs: days-before-pr-close: -1 # Only issues made after Feb 09 2021. start-date: "2021-09-02T00:00:00" - exempt-issue-labels: "Epic,Area/CLI,Area/Cloud/AWS,Area/Cloud/Azure,Area/Cloud/GCP,Area/Cloud/vSphere,Area/CSI,Area/Design,Area/Documentation,Area/Plugins,Bug,Enhancement/User,kind/requirement,kind/refactor,kind/tech-debt,limitation,Needs investigation,Needs triage,Needs Product,P0 - Hair on fire,P1 - Important,P2 - Long-term important,P3 - Wouldn't it be nice if...,Product Requirements,Restic - GA,Restic,release-blocker,Security,backlog" + exempt-issue-labels: "Epic,Area/CLI,Area/Cloud/AWS,Area/Cloud/Azure,Area/Cloud/GCP,Area/Cloud/vSphere,Area/CSI,Area/Design,Area/Documentation,Area/Plugins,Bug,Enhancement/User,kind/requirement,kind/refactor,kind/tech-debt,limitation,Needs investigation,Needs triage,Needs Product,P0 - Hair on fire,P1 - Important,P2 - Long-term important,P3 - Wouldn't it be nice if...,Product Requirements,release-blocker,Security,backlog" diff --git a/changelogs/unreleased/9848-adam-jian-zhang b/changelogs/unreleased/9848-adam-jian-zhang new file mode 100644 index 000000000..499c41fa5 --- /dev/null +++ b/changelogs/unreleased/9848-adam-jian-zhang @@ -0,0 +1 @@ +Fix issue #9814, add validations for NamespacedFilterPolicies diff --git a/changelogs/unreleased/9862-Lyndon-Li b/changelogs/unreleased/9862-Lyndon-Li new file mode 100644 index 000000000..04601d969 --- /dev/null +++ b/changelogs/unreleased/9862-Lyndon-Li @@ -0,0 +1 @@ +Enhance backup exposer for block data mover \ No newline at end of file diff --git a/changelogs/unreleased/9864-Lyndon-Li b/changelogs/unreleased/9864-Lyndon-Li new file mode 100644 index 000000000..1d885de8f --- /dev/null +++ b/changelogs/unreleased/9864-Lyndon-Li @@ -0,0 +1 @@ +Add cbt service parameters to node-agent-config for block data mover \ No newline at end of file diff --git a/changelogs/unreleased/9867-Joeavaikath b/changelogs/unreleased/9867-Joeavaikath new file mode 100644 index 000000000..6ae2f7396 --- /dev/null +++ b/changelogs/unreleased/9867-Joeavaikath @@ -0,0 +1 @@ +Remove Restic cases and workflow from E2E diff --git a/changelogs/unreleased/9875-Lyndon-Li b/changelogs/unreleased/9875-Lyndon-Li new file mode 100644 index 000000000..c0afbee71 --- /dev/null +++ b/changelogs/unreleased/9875-Lyndon-Li @@ -0,0 +1 @@ +Add totalSize to repo snapshot operation; and pin unified repo snapshots to Kopia repository and so pin them with Velero backup lifecycle \ No newline at end of file diff --git a/internal/resourcepolicies/resource_policies.go b/internal/resourcepolicies/resource_policies.go index ce32dba19..232633484 100644 --- a/internal/resourcepolicies/resource_policies.go +++ b/internal/resourcepolicies/resource_policies.go @@ -23,12 +23,14 @@ import ( "k8s.io/apimachinery/pkg/util/sets" + "github.com/gobwas/glob" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" crclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/util/wildcard" ) type VolumeActionType string @@ -259,6 +261,10 @@ func (p *Policies) Validate() error { } } + if err := p.validateNamespacedFilterPolicies(); err != nil { + return errors.WithStack(err) + } + return nil } @@ -335,3 +341,76 @@ func getResourcePoliciesFromConfig(cm *corev1api.ConfigMap) (*Policies, error) { return policies, nil } + +func (p *Policies) validateNamespacedFilterPolicies() error { + seenPatterns := make(map[string][]int) // pattern -> list of policy indices + + // Rule 1-7: Basic validation rules + for i, nfp := range p.namespacedFilterPolicies { + if len(nfp.Namespaces) == 0 { + return fmt.Errorf("namespacedFilterPolicies[%d]: at least one namespace must be specified", i) + } + if len(nfp.ResourceFilters) == 0 { + return fmt.Errorf("namespacedFilterPolicies[%d]: at least one resourceFilter must be specified", i) + } + + // Rule 8 & 9: Validate glob patterns and collect namespace patterns for duplicate check + for j, pattern := range nfp.Namespaces { + if err := wildcard.ValidateNamespaceName(pattern); err != nil { + return fmt.Errorf("namespacedFilterPolicies[%d].namespaces[%d]: %w", i, j, err) + } + seenPatterns[pattern] = append(seenPatterns[pattern], i) + } + + seenKinds := make(map[string]int) + hasCatchAll := false + for j, rf := range nfp.ResourceFilters { + if rf.IsCatchAll() { + if hasCatchAll { + return fmt.Errorf("namespacedFilterPolicies[%d]: only one catch-all resource filter is allowed", i) + } + hasCatchAll = true + if len(rf.Names) > 0 || len(rf.ExcludedNames) > 0 { + return fmt.Errorf("namespacedFilterPolicies[%d].resourceFilters[%d]: names or excludedNames cannot be specified for catch-all filters", i, j) + } + } + + for _, kind := range rf.Kinds { + if kind == "*" { + continue // "*" is handled by IsCatchAll, no need to check duplicates against other kinds + } + if prevJ, ok := seenKinds[kind]; ok { + return fmt.Errorf("namespacedFilterPolicies[%d]: kind %q appears in both resourceFilters[%d] and resourceFilters[%d]", i, kind, prevJ, j) + } + seenKinds[kind] = j + } + + if len(rf.LabelSelector) > 0 && len(rf.OrLabelSelectors) > 0 { + return fmt.Errorf("namespacedFilterPolicies[%d].resourceFilters[%d]: labelSelector and orLabelSelectors cannot co-exist", i, j) + } + + // Validate glob patterns for names and excludedNames using gobwas/glob + for k, pattern := range rf.Names { + if _, err := glob.Compile(pattern); err != nil { + return fmt.Errorf("namespacedFilterPolicies[%d].resourceFilters[%d].names[%d]: invalid glob pattern %q: %v", i, j, k, pattern, err) + } + } + for k, pattern := range rf.ExcludedNames { + if _, err := glob.Compile(pattern); err != nil { + return fmt.Errorf("namespacedFilterPolicies[%d].resourceFilters[%d].excludedNames[%d]: invalid glob pattern %q: %v", i, j, k, pattern, err) + } + } + } + } + + // Rule 8: Report exact duplicates only + for pattern, policyIndices := range seenPatterns { + if len(policyIndices) > 1 { + return fmt.Errorf( + "namespacedFilterPolicies: duplicate namespace pattern '%s' found in policies %v", + pattern, policyIndices) + } + } + + return nil +} diff --git a/internal/resourcepolicies/resource_policies_test.go b/internal/resourcepolicies/resource_policies_test.go index 06b1bea1c..898c6d1ca 100644 --- a/internal/resourcepolicies/resource_policies_test.go +++ b/internal/resourcepolicies/resource_policies_test.go @@ -1242,3 +1242,297 @@ func TestPVCPhaseMatch(t *testing.T) { }) } } + +func TestNamespacedFilterPolicies(t *testing.T) { + testCases := []struct { + name string + yamlData string + wantErr bool + errMsg string + }{ + { + name: "valid namespacedFilterPolicies with multiple kinds", + yamlData: `version: v1 +namespacedFilterPolicies: +- namespaces: ["frontend", "backend"] + resourceFilters: + - kinds: ["Pod", "ConfigMap"] + labelSelector: + app: web + names: ["app-*"] + - kinds: ["Secret"] + excludedNames: ["temp-*"]`, + wantErr: false, + }, + { + name: "valid namespacedFilterPolicies with glob patterns", + yamlData: `version: v1 +namespacedFilterPolicies: +- namespaces: ["team-*"] + resourceFilters: + - kinds: ["Pod"] + orLabelSelectors: + - env: prod + - env: staging`, + wantErr: false, + }, + { + name: "valid - overlapping patterns allowed (first-match semantics)", + yamlData: `version: v1 +namespacedFilterPolicies: +- namespaces: ["team-frontend-*"] + resourceFilters: + - kinds: ["Pod", "ConfigMap", "Secret"] +- namespaces: ["team-*"] + resourceFilters: + - kinds: ["Deployment", "Service"]`, + wantErr: false, + }, + { + name: "invalid - no namespaces", + yamlData: `version: v1 +namespacedFilterPolicies: +- namespaces: [] + resourceFilters: + - kinds: ["Pod"]`, + wantErr: true, + errMsg: "at least one namespace must be specified", + }, + { + name: "invalid - no resourceFilters", + yamlData: `version: v1 +namespacedFilterPolicies: +- namespaces: ["test"] + resourceFilters: []`, + wantErr: true, + errMsg: "at least one resourceFilter must be specified", + }, + { + name: "valid - asterisk catch-all", + yamlData: `version: v1 +namespacedFilterPolicies: +- namespaces: ["test"] + resourceFilters: + - kinds: ["*"] + labelSelector: + app: web`, + wantErr: false, + }, + { + name: "invalid - multiple asterisk kinds", + yamlData: `version: v1 +namespacedFilterPolicies: +- namespaces: ["test"] + resourceFilters: + - kinds: ["*"] + labelSelector: + app: web + - kinds: ["*"] + labelSelector: + app: db`, + wantErr: true, + errMsg: "only one catch-all resource filter is allowed", + }, + { + name: "invalid - empty and asterisk kinds", + yamlData: `version: v1 +namespacedFilterPolicies: +- namespaces: ["test"] + resourceFilters: + - kinds: [] + labelSelector: + app: web + - kinds: ["*"] + labelSelector: + app: db`, + wantErr: true, + errMsg: "only one catch-all resource filter is allowed", + }, + { + name: "invalid - multiple empty kinds", + yamlData: `version: v1 +namespacedFilterPolicies: +- namespaces: ["test"] + resourceFilters: + - kinds: [] + labelSelector: + app: web + - kinds: [] + labelSelector: + app: db`, + wantErr: true, + errMsg: "only one catch-all resource filter is allowed", + }, + { + name: "invalid - names with empty kinds", + yamlData: `version: v1 +namespacedFilterPolicies: +- namespaces: ["test"] + resourceFilters: + - kinds: [] + names: ["app-*"] + labelSelector: + app: web`, + wantErr: true, + errMsg: "names or excludedNames cannot be specified for catch-all filters", + }, + { + name: "invalid - excludedNames with empty kinds", + yamlData: `version: v1 +namespacedFilterPolicies: +- namespaces: ["test"] + resourceFilters: + - kinds: [] + excludedNames: ["app-*"] + labelSelector: + app: web`, + wantErr: true, + errMsg: "names or excludedNames cannot be specified for catch-all filters", + }, + { + name: "valid - no label selectors with catch-all", + yamlData: `version: v1 +namespacedFilterPolicies: +- namespaces: ["test"] + resourceFilters: + - kinds: ["*"]`, + wantErr: false, + }, + { + name: "invalid - duplicate kinds", + yamlData: `version: v1 +namespacedFilterPolicies: +- namespaces: ["test"] + resourceFilters: + - kinds: ["Pod"] + - kinds: ["Pod", "ConfigMap"]`, + wantErr: true, + errMsg: "kind \"Pod\" appears in both resourceFilters", + }, + { + name: "invalid - both labelSelector and orLabelSelectors", + yamlData: `version: v1 +namespacedFilterPolicies: +- namespaces: ["test"] + resourceFilters: + - kinds: ["Pod"] + labelSelector: + app: web + orLabelSelectors: + - env: prod`, + wantErr: true, + errMsg: "labelSelector and orLabelSelectors cannot co-exist", + }, + { + name: "invalid - bad glob pattern in names", + yamlData: `version: v1 +namespacedFilterPolicies: +- namespaces: ["test"] + resourceFilters: + - kinds: ["Pod"] + names: ["[invalid"]`, + wantErr: true, + errMsg: "invalid glob pattern", + }, + { + name: "invalid - duplicate namespace pattern", + yamlData: `version: v1 +namespacedFilterPolicies: +- namespaces: ["production"] + resourceFilters: + - kinds: ["Pod"] +- namespaces: ["production"] + resourceFilters: + - kinds: ["ConfigMap"]`, + wantErr: true, + errMsg: "duplicate namespace pattern", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + resPolicies, err := unmarshalResourcePolicies(&tc.yamlData) + require.NoError(t, err) // Unmarshal should always succeed for our test cases + + policies := &Policies{} + err = policies.BuildPolicy(resPolicies) + require.NoError(t, err) // BuildPolicy should always succeed for our test cases + + err = policies.Validate() + if tc.wantErr { + require.Error(t, err) + if tc.errMsg != "" { + assert.Contains(t, err.Error(), tc.errMsg) + } + } else { + require.NoError(t, err) + + // Verify that we can retrieve the policies + nfPolicies := policies.GetNamespacedFilterPolicies() + assert.GreaterOrEqual(t, len(nfPolicies), 1) // Valid test cases have at least 1 policy + } + }) + } +} + +func TestNamespacedFilterPoliciesAccessor(t *testing.T) { + yamlData := `version: v1 +namespacedFilterPolicies: +- namespaces: ["frontend"] + resourceFilters: + - kinds: ["Pod"] + labelSelector: + app: web` + + resPolicies, err := unmarshalResourcePolicies(&yamlData) + require.NoError(t, err) + + policies := &Policies{} + err = policies.BuildPolicy(resPolicies) + require.NoError(t, err) + + nfPolicies := policies.GetNamespacedFilterPolicies() + require.Len(t, nfPolicies, 1) + + policy := nfPolicies[0] + assert.Equal(t, []string{"frontend"}, policy.Namespaces) + assert.Len(t, policy.ResourceFilters, 1) + + rf := policy.ResourceFilters[0] + assert.Equal(t, []string{"Pod"}, rf.Kinds) + assert.Equal(t, map[string]string{"app": "web"}, rf.LabelSelector) +} + +func TestFirstMatchSemantics(t *testing.T) { + yamlData := `version: v1 +namespacedFilterPolicies: +- namespaces: ["team-frontend-*", "specific-ns"] + resourceFilters: + - kinds: ["Pod", "ConfigMap", "Secret"] +- namespaces: ["team-*", "another-pattern"] + resourceFilters: + - kinds: ["Deployment", "Service"]` + + resPolicies, err := unmarshalResourcePolicies(&yamlData) + require.NoError(t, err) + + policies := &Policies{} + err = policies.BuildPolicy(resPolicies) + require.NoError(t, err) + + err = policies.Validate() + require.NoError(t, err) + + nfPolicies := policies.GetNamespacedFilterPolicies() + require.Len(t, nfPolicies, 2) + + // Verify the first policy has the more specific patterns + policy1 := nfPolicies[0] + assert.Equal(t, []string{"team-frontend-*", "specific-ns"}, policy1.Namespaces) + assert.Equal(t, []string{"Pod", "ConfigMap", "Secret"}, policy1.ResourceFilters[0].Kinds) + + // Verify the second policy has the broader patterns + policy2 := nfPolicies[1] + assert.Equal(t, []string{"team-*", "another-pattern"}, policy2.Namespaces) + assert.Equal(t, []string{"Deployment", "Service"}, policy2.ResourceFilters[0].Kinds) +} diff --git a/pkg/cmd/cli/nodeagent/server.go b/pkg/cmd/cli/nodeagent/server.go index bb1764cb3..374dbcac9 100644 --- a/pkg/cmd/cli/nodeagent/server.go +++ b/pkg/cmd/cli/nodeagent/server.go @@ -385,6 +385,12 @@ func (s *nodeAgentServer) run() { s.logger.Info("Backup repo config is not provided, using default values for cache volume configs") } + var csiSnapshotMetadataServiceConfigs *velerotypes.CSISnapshotMetadataService + if s.dataPathConfigs != nil && s.dataPathConfigs.CSISnapshotMetadataServiceConfigs != nil { + csiSnapshotMetadataServiceConfigs = s.dataPathConfigs.CSISnapshotMetadataServiceConfigs + s.logger.Infof("Using CSI snapshot metadata service config %v", s.dataPathConfigs.CSISnapshotMetadataServiceConfigs) + } + pvbReconciler := controller.NewPodVolumeBackupReconciler( s.mgr.GetClient(), s.mgr, @@ -447,6 +453,7 @@ func (s *nodeAgentServer) run() { dataMovePriorityClass, podLabels, podAnnotations, + csiSnapshotMetadataServiceConfigs, ) if err := dataUploadReconciler.SetupWithManager(s.mgr); err != nil { s.logger.WithError(err).Fatal("Unable to create the data upload controller") diff --git a/pkg/controller/data_download_controller.go b/pkg/controller/data_download_controller.go index 36fec450b..738334ceb 100644 --- a/pkg/controller/data_download_controller.go +++ b/pkg/controller/data_download_controller.go @@ -150,7 +150,7 @@ func (r *DataDownloadReconciler) Reconcile(ctx context.Context, req ctrl.Request return ctrl.Result{}, err } - if !datamover.IsBuiltInUploader(dd.Spec.DataMover) { + if !datamover.IsBuiltInDataMover(dd.Spec.DataMover) { log.WithField("data mover", dd.Spec.DataMover).Info("it is not one built-in data mover which is not supported by Velero") return ctrl.Result{}, nil } diff --git a/pkg/controller/data_upload_controller.go b/pkg/controller/data_upload_controller.go index 13be03994..9aaf3d653 100644 --- a/pkg/controller/data_upload_controller.go +++ b/pkg/controller/data_upload_controller.go @@ -66,25 +66,26 @@ const ( // DataUploadReconciler reconciles a DataUpload object type DataUploadReconciler struct { - client client.Client - kubeClient kubernetes.Interface - csiSnapshotClient snapshotter.SnapshotV1Interface - mgr manager.Manager - Clock clocks.WithTickerAndDelayedExecution - nodeName string - logger logrus.FieldLogger - snapshotExposerList map[velerov2alpha1api.SnapshotType]exposer.SnapshotExposer - dataPathMgr *datapath.Manager - vgdpCounter *exposer.VgdpCounter - loadAffinity []*kube.LoadAffinity - backupPVCConfig map[string]velerotypes.BackupPVC - podResources corev1api.ResourceRequirements - preparingTimeout time.Duration - metrics *metrics.ServerMetrics - cancelledDataUpload map[string]time.Time - dataMovePriorityClass string - podLabels map[string]string - podAnnotations map[string]string + client client.Client + kubeClient kubernetes.Interface + csiSnapshotClient snapshotter.SnapshotV1Interface + mgr manager.Manager + Clock clocks.WithTickerAndDelayedExecution + nodeName string + logger logrus.FieldLogger + snapshotExposerList map[velerov2alpha1api.SnapshotType]exposer.SnapshotExposer + dataPathMgr *datapath.Manager + vgdpCounter *exposer.VgdpCounter + loadAffinity []*kube.LoadAffinity + backupPVCConfig map[string]velerotypes.BackupPVC + podResources corev1api.ResourceRequirements + preparingTimeout time.Duration + metrics *metrics.ServerMetrics + cancelledDataUpload map[string]time.Time + dataMovePriorityClass string + podLabels map[string]string + podAnnotations map[string]string + snapshotMetadataServiceConfigs *velerotypes.CSISnapshotMetadataService } func NewDataUploadReconciler( @@ -105,6 +106,7 @@ func NewDataUploadReconciler( dataMovePriorityClass string, podLabels map[string]string, podAnnotations map[string]string, + snapshotMetadataServiceConfigs *velerotypes.CSISnapshotMetadataService, ) *DataUploadReconciler { return &DataUploadReconciler{ client: client, @@ -121,17 +123,18 @@ func NewDataUploadReconciler( log, ), }, - dataPathMgr: dataPathMgr, - vgdpCounter: counter, - loadAffinity: loadAffinity, - backupPVCConfig: backupPVCConfig, - podResources: podResources, - preparingTimeout: preparingTimeout, - metrics: metrics, - cancelledDataUpload: make(map[string]time.Time), - dataMovePriorityClass: dataMovePriorityClass, - podLabels: podLabels, - podAnnotations: podAnnotations, + dataPathMgr: dataPathMgr, + vgdpCounter: counter, + loadAffinity: loadAffinity, + backupPVCConfig: backupPVCConfig, + podResources: podResources, + preparingTimeout: preparingTimeout, + metrics: metrics, + cancelledDataUpload: make(map[string]time.Time), + dataMovePriorityClass: dataMovePriorityClass, + podLabels: podLabels, + podAnnotations: podAnnotations, + snapshotMetadataServiceConfigs: snapshotMetadataServiceConfigs, } } @@ -156,7 +159,7 @@ func (r *DataUploadReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, errors.Wrap(err, "getting DataUpload") } - if !datamover.IsBuiltInUploader(du.Spec.DataMover) { + if !datamover.IsBuiltInDataMover(du.Spec.DataMover) { log.WithField("Data mover", du.Spec.DataMover).Debug("it is not one built-in data mover which is not supported by Velero") return ctrl.Result{}, nil } @@ -936,7 +939,12 @@ func (r *DataUploadReconciler) setupExposeParam(du *velerov2alpha1api.DataUpload return nil, errors.Wrapf(err, "failed to get source PV %s", pvc.Spec.VolumeName) } - nodeOS := kube.GetPVCAttachingNodeOS(pvc, r.kubeClient.CoreV1(), r.kubeClient.StorageV1(), log) + nodeOS := "" + if du.Spec.DataMover == datamover.DataMoverTypeVeleroBlock { + nodeOS = kube.NodeOSLinux + } else { + nodeOS = kube.GetPVCAttachingNodeOS(pvc, r.kubeClient.CoreV1(), r.kubeClient.StorageV1(), log) + } if err := kube.HasNodeWithOS(context.Background(), nodeOS, r.kubeClient.CoreV1()); err != nil { return nil, errors.Wrapf(err, "no appropriate node to run data upload for PVC %s/%s", du.Spec.SourceNamespace, du.Spec.SourcePVC) @@ -993,23 +1001,25 @@ func (r *DataUploadReconciler) setupExposeParam(du *velerov2alpha1api.DataUpload } return &exposer.CSISnapshotExposeParam{ - SnapshotName: du.Spec.CSISnapshot.VolumeSnapshot, - SourceNamespace: du.Spec.SourceNamespace, - SourcePVCName: pvc.Name, - SourcePVName: pv.Name, - StorageClass: du.Spec.CSISnapshot.StorageClass, - HostingPodLabels: hostingPodLabels, - HostingPodAnnotations: hostingPodAnnotation, - HostingPodTolerations: hostingPodTolerations, - AccessMode: accessMode, - OperationTimeout: du.Spec.OperationTimeout.Duration, - ExposeTimeout: r.preparingTimeout, - VolumeSize: pvc.Spec.Resources.Requests[corev1api.ResourceStorage], - Affinity: r.loadAffinity, - BackupPVCConfig: r.backupPVCConfig, - Resources: r.podResources, - NodeOS: nodeOS, - PriorityClassName: r.dataMovePriorityClass, + SnapshotName: du.Spec.CSISnapshot.VolumeSnapshot, + SourceNamespace: du.Spec.SourceNamespace, + SourcePVCName: pvc.Name, + SourcePVName: pv.Name, + StorageClass: du.Spec.CSISnapshot.StorageClass, + HostingPodLabels: hostingPodLabels, + HostingPodAnnotations: hostingPodAnnotation, + HostingPodTolerations: hostingPodTolerations, + AccessMode: accessMode, + OperationTimeout: du.Spec.OperationTimeout.Duration, + ExposeTimeout: r.preparingTimeout, + VolumeSize: pvc.Spec.Resources.Requests[corev1api.ResourceStorage], + Affinity: r.loadAffinity, + BackupPVCConfig: r.backupPVCConfig, + Resources: r.podResources, + NodeOS: nodeOS, + PriorityClassName: r.dataMovePriorityClass, + DataMover: du.Spec.DataMover, + SnapshotMetadataServiceConfigs: r.snapshotMetadataServiceConfigs, }, nil } diff --git a/pkg/controller/data_upload_controller_test.go b/pkg/controller/data_upload_controller_test.go index a73e3de94..e6d5474f3 100644 --- a/pkg/controller/data_upload_controller_test.go +++ b/pkg/controller/data_upload_controller_test.go @@ -251,6 +251,7 @@ func initDataUploaderReconcilerWithError(needError ...error) (*DataUploadReconci "", // dataMovePriorityClass nil, // podLabels nil, // podAnnotations + nil, ), nil } @@ -1513,6 +1514,7 @@ func TestDataUploadSetupExposeParam(t *testing.T) { "upload-priority", tt.args.customLabels, tt.args.customAnnotations, + nil, ) // Act diff --git a/pkg/datamover/dataupload_delete_action.go b/pkg/datamover/dataupload_delete_action.go index 6b36e1068..46c62a1f1 100644 --- a/pkg/datamover/dataupload_delete_action.go +++ b/pkg/datamover/dataupload_delete_action.go @@ -88,7 +88,7 @@ func (d *DataUploadDeleteAction) Execute(input *velero.DeleteItemActionExecuteIn // generate the configmap which is to be created and used as a way to communicate the snapshot info to the backup deletion controller func genConfigmap(bak *velerov1.Backup, du velerov2alpha1.DataUpload) *corev1api.ConfigMap { - if !IsBuiltInUploader(du.Spec.DataMover) || du.Status.SnapshotID == "" { + if !IsBuiltInDataMover(du.Spec.DataMover) || du.Status.SnapshotID == "" { return nil } snapshot := repotypes.SnapshotIdentifier{ diff --git a/pkg/datamover/util.go b/pkg/datamover/util.go index e4097f07e..7e37695b6 100644 --- a/pkg/datamover/util.go +++ b/pkg/datamover/util.go @@ -18,6 +18,11 @@ package datamover import "fmt" +const ( + DataMoverTypeVeleroFs string = "velero-fs" + DataMoverTypeVeleroBlock string = "velero-block" +) + func GetUploaderType(dataMover string) string { if dataMover == "" || dataMover == "velero" { return "kopia" @@ -26,7 +31,7 @@ func GetUploaderType(dataMover string) string { } } -func IsBuiltInUploader(dataMover string) bool { +func IsBuiltInDataMover(dataMover string) bool { return dataMover == "" || dataMover == "velero" } diff --git a/pkg/datamover/util_test.go b/pkg/datamover/util_test.go index d4e3e6efe..80e2f4e16 100644 --- a/pkg/datamover/util_test.go +++ b/pkg/datamover/util_test.go @@ -30,7 +30,7 @@ func TestIsBuiltInUploader(t *testing.T) { } for _, tc := range testcases { t.Run(tc.name, func(tt *testing.T) { - assert.Equal(tt, tc.want, IsBuiltInUploader(tc.dataMover)) + assert.Equal(tt, tc.want, IsBuiltInDataMover(tc.dataMover)) }) } } diff --git a/pkg/exposer/csi_snapshot.go b/pkg/exposer/csi_snapshot.go index 079a4b527..33dafff99 100644 --- a/pkg/exposer/csi_snapshot.go +++ b/pkg/exposer/csi_snapshot.go @@ -19,6 +19,7 @@ package exposer import ( "context" "fmt" + "maps" "time" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" @@ -33,6 +34,7 @@ import ( "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/vmware-tanzu/velero/pkg/datamover" "github.com/vmware-tanzu/velero/pkg/nodeagent" velerotypes "github.com/vmware-tanzu/velero/pkg/types" "github.com/vmware-tanzu/velero/pkg/util" @@ -93,6 +95,12 @@ type CSISnapshotExposeParam struct { // PriorityClassName is the priority class name for the data mover pod PriorityClassName string + + // DataMover is the data mover type, e.g., velero-fs, velero-block + DataMover string + + // SnapshotMetadataServiceConfigs is the config for CSI snapshot metadata service + SnapshotMetadataServiceConfigs *velerotypes.CSISnapshotMetadataService } // CSISnapshotExposeWaitParam define the input param for WaitExposed of CSI snapshots @@ -234,7 +242,7 @@ func (e *csiSnapshotExposer) Expose(ctx context.Context, ownerObject corev1api.O } } - backupPVC, err := e.createBackupPVC(ctx, ownerObject, backupVS.Name, backupPVCStorageClass, csiExposeParam.AccessMode, volumeSize, backupPVCReadOnly, backupPVCAnnotations) + backupPVC, err := e.createBackupPVC(ctx, ownerObject, backupVS.Name, backupPVCStorageClass, csiExposeParam.AccessMode, volumeSize, backupPVCReadOnly, backupPVCAnnotations, csiExposeParam.DataMover) if err != nil { return errors.Wrap(err, "error to create backup pvc") } @@ -264,6 +272,7 @@ func (e *csiSnapshotExposer) Expose(ctx context.Context, ownerObject corev1api.O csiExposeParam.PriorityClassName, intoleratableNodes, volumeTopology, + csiExposeParam.SnapshotMetadataServiceConfigs, ) if err != nil { return errors.Wrap(err, "error to create backup pod") @@ -450,7 +459,11 @@ func (e *csiSnapshotExposer) CleanUp(ctx context.Context, ownerObject corev1api. csi.DeleteVolumeSnapshotIfAny(ctx, e.csiSnapshotClient, vsName, sourceNamespace, e.log) } -func getVolumeModeByAccessMode(accessMode string) (corev1api.PersistentVolumeMode, error) { +func getVolumeModeByAccessMode(accessMode string, dataMover string) (corev1api.PersistentVolumeMode, error) { + if dataMover == datamover.DataMoverTypeVeleroBlock { + return corev1api.PersistentVolumeBlock, nil + } + switch accessMode { case AccessModeFileSystem: return corev1api.PersistentVolumeFilesystem, nil @@ -488,10 +501,14 @@ func (e *csiSnapshotExposer) createBackupVS(ctx context.Context, ownerObject cor func (e *csiSnapshotExposer) createBackupVSC(ctx context.Context, ownerObject corev1api.ObjectReference, snapshotVSC *snapshotv1api.VolumeSnapshotContent, vs *snapshotv1api.VolumeSnapshot) (*snapshotv1api.VolumeSnapshotContent, error) { backupVSCName := ownerObject.Name + anno := make(map[string]string) + maps.Copy(anno, snapshotVSC.Annotations) + anno[kube.KubeAnnAllowVolumeModeChange] = "true" + vsc := &snapshotv1api.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: backupVSCName, - Annotations: snapshotVSC.Annotations, + Annotations: anno, Labels: map[string]string{}, }, Spec: snapshotv1api.VolumeSnapshotContentSpec{ @@ -524,10 +541,10 @@ func (e *csiSnapshotExposer) createBackupVSC(ctx context.Context, ownerObject co return e.csiSnapshotClient.VolumeSnapshotContents().Create(ctx, vsc, metav1.CreateOptions{}) } -func (e *csiSnapshotExposer) createBackupPVC(ctx context.Context, ownerObject corev1api.ObjectReference, backupVS, storageClass, accessMode string, resource resource.Quantity, readOnly bool, annotations map[string]string) (*corev1api.PersistentVolumeClaim, error) { +func (e *csiSnapshotExposer) createBackupPVC(ctx context.Context, ownerObject corev1api.ObjectReference, backupVS, storageClass, accessMode string, resource resource.Quantity, readOnly bool, annotations map[string]string, dataMover string) (*corev1api.PersistentVolumeClaim, error) { backupPVCName := ownerObject.Name - volumeMode, err := getVolumeModeByAccessMode(accessMode) + volumeMode, err := getVolumeModeByAccessMode(accessMode, dataMover) if err != nil { return nil, err } @@ -600,6 +617,7 @@ func (e *csiSnapshotExposer) createBackupPod( priorityClassName string, intoleratableNodes []string, volumeTopology *corev1api.NodeSelector, + csiSnapshotMetadataServiceConfigs *velerotypes.CSISnapshotMetadataService, ) (*corev1api.Pod, error) { podName := ownerObject.Name @@ -655,6 +673,12 @@ func (e *csiSnapshotExposer) createBackupPod( args = append(args, podInfo.logFormatArgs...) args = append(args, podInfo.logLevelArgs...) + if csiSnapshotMetadataServiceConfigs != nil { + if csiSnapshotMetadataServiceConfigs.SAName != "" { + args = append(args, fmt.Sprintf("--csi-snapshot-metadata-service-sa=%s", csiSnapshotMetadataServiceConfigs.SAName)) + } + } + if affinity == nil { affinity = &kube.LoadAffinity{} } diff --git a/pkg/exposer/csi_snapshot_priority_test.go b/pkg/exposer/csi_snapshot_priority_test.go index d1ffa4700..8c3086f76 100644 --- a/pkg/exposer/csi_snapshot_priority_test.go +++ b/pkg/exposer/csi_snapshot_priority_test.go @@ -155,6 +155,7 @@ func TestCreateBackupPodWithPriorityClass(t *testing.T) { tc.expectedPriorityClass, nil, nil, + nil, ) require.NoError(t, err, tc.description) @@ -241,6 +242,7 @@ func TestCreateBackupPodWithMissingConfigMap(t *testing.T) { "", // empty priority class since config map is missing nil, nil, + nil, ) // Should succeed even when config map is missing diff --git a/pkg/exposer/csi_snapshot_test.go b/pkg/exposer/csi_snapshot_test.go index e1a9860eb..8e00654f9 100644 --- a/pkg/exposer/csi_snapshot_test.go +++ b/pkg/exposer/csi_snapshot_test.go @@ -18,6 +18,7 @@ package exposer import ( "fmt" + "maps" "testing" "time" @@ -1056,21 +1057,25 @@ func TestExpose(t *testing.T) { backupPVC, err := exposer.kubeClient.CoreV1().PersistentVolumeClaims(ownerObject.Namespace).Get(t.Context(), ownerObject.Name, metav1.GetOptions{}) require.NoError(t, err) - expectedVS, err := exposer.csiSnapshotClient.VolumeSnapshots(ownerObject.Namespace).Get(t.Context(), ownerObject.Name, metav1.GetOptions{}) + backupVS, err := exposer.csiSnapshotClient.VolumeSnapshots(ownerObject.Namespace).Get(t.Context(), ownerObject.Name, metav1.GetOptions{}) require.NoError(t, err) - expectedVSC, err := exposer.csiSnapshotClient.VolumeSnapshotContents().Get(t.Context(), ownerObject.Name, metav1.GetOptions{}) + backupVSC, err := exposer.csiSnapshotClient.VolumeSnapshotContents().Get(t.Context(), ownerObject.Name, metav1.GetOptions{}) require.NoError(t, err) - assert.Equal(t, expectedVS.Annotations, vsObject.Annotations) - assert.Equal(t, *expectedVS.Spec.VolumeSnapshotClassName, *vsObject.Spec.VolumeSnapshotClassName) - assert.Equal(t, expectedVSC.Name, *expectedVS.Spec.Source.VolumeSnapshotContentName) + assert.Equal(t, vsObject.Annotations, backupVS.Annotations) + assert.Equal(t, *vsObject.Spec.VolumeSnapshotClassName, *backupVS.Spec.VolumeSnapshotClassName) + assert.Equal(t, *backupVS.Spec.Source.VolumeSnapshotContentName, backupVSC.Name) - assert.Equal(t, expectedVSC.Annotations, vscObj.Annotations) - assert.Equal(t, expectedVSC.Labels, vscObj.Labels) - assert.Equal(t, expectedVSC.Spec.DeletionPolicy, vscObj.Spec.DeletionPolicy) - assert.Equal(t, expectedVSC.Spec.Driver, vscObj.Spec.Driver) - assert.Equal(t, *expectedVSC.Spec.VolumeSnapshotClassName, *vscObj.Spec.VolumeSnapshotClassName) + anno := make(map[string]string) + maps.Copy(anno, vscObj.Annotations) + anno[kube.KubeAnnAllowVolumeModeChange] = "true" + + assert.Equal(t, anno, backupVSC.Annotations) + assert.Equal(t, vscObj.Labels, backupVSC.Labels) + assert.Equal(t, vscObj.Spec.DeletionPolicy, backupVSC.Spec.DeletionPolicy) + assert.Equal(t, vscObj.Spec.Driver, backupVSC.Spec.Driver) + assert.Equal(t, *vscObj.Spec.VolumeSnapshotClassName, *backupVSC.Spec.VolumeSnapshotClassName) if test.expectedVolumeSize != nil { assert.Equal(t, *test.expectedVolumeSize, backupPVC.Spec.Resources.Requests[corev1api.ResourceStorage]) @@ -1514,7 +1519,7 @@ func Test_csiSnapshotExposer_createBackupPVC(t *testing.T) { APIVersion: tt.ownerBackup.APIVersion, } } - got, err := e.createBackupPVC(t.Context(), ownerObject, tt.backupVS, tt.storageClass, tt.accessMode, tt.resource, tt.readOnly, map[string]string{}) + got, err := e.createBackupPVC(t.Context(), ownerObject, tt.backupVS, tt.storageClass, tt.accessMode, tt.resource, tt.readOnly, map[string]string{}, "") if !tt.wantErr(t, err, fmt.Sprintf("createBackupPVC(%v, %v, %v, %v, %v, %v)", ownerObject, tt.backupVS, tt.storageClass, tt.accessMode, tt.resource, tt.readOnly)) { return } diff --git a/pkg/repository/udmrepo/kopialib/lib_repo.go b/pkg/repository/udmrepo/kopialib/lib_repo.go index 5b2efbd69..bb70e1b18 100644 --- a/pkg/repository/udmrepo/kopialib/lib_repo.go +++ b/pkg/repository/udmrepo/kopialib/lib_repo.go @@ -629,6 +629,9 @@ func (kr *kopiaRepository) SaveSnapshot(ctx context.Context, snap udmrepo.Snapsh Description: snap.Description, StartTime: fs.UTCTimestampFromTime(snap.StartTime), EndTime: fs.UTCTimestampFromTime(snap.EndTime), + Stats: snapshot.Stats{ + TotalFileSize: snap.TotalSize, + }, RootEntry: &snapshot.DirEntry{ Type: snapshot.EntryTypeDirectory, ObjectID: rootObj, @@ -637,8 +640,12 @@ func (kr *kopiaRepository) SaveSnapshot(ctx context.Context, snap udmrepo.Snapsh FileSize: snap.RootObject.Size, UserID: snap.RootObject.UserID, GroupID: snap.RootObject.GroupID, + DirSummary: &fs.DirectorySummary{ + TotalFileSize: snap.TotalSize, + }, }, Tags: snap.Tags, + Pins: []string{"velero-pin"}, // pins are meant to prevent snapshot from automatic expiration/deletion. } id, err := snapshot.SaveSnapshot(ctx, kr.rawWriter, &manifest) @@ -665,6 +672,7 @@ func (kr *kopiaRepository) GetSnapshot(ctx context.Context, id udmrepo.ID) (udmr StartTime: snap.StartTime.ToTime(), EndTime: snap.EndTime.ToTime(), Tags: snap.Tags, + TotalSize: snap.Stats.TotalFileSize, RootObject: udmrepo.ObjectMetadata{ ID: udmrepo.ID(snap.RootEntry.ObjectID.String()), Type: udmrepo.ObjectDataTypeMetadata, diff --git a/pkg/repository/udmrepo/repo.go b/pkg/repository/udmrepo/repo.go index 7b47eb735..bd0e6eb19 100644 --- a/pkg/repository/udmrepo/repo.go +++ b/pkg/repository/udmrepo/repo.go @@ -98,6 +98,7 @@ type Snapshot struct { StartTime time.Time EndTime time.Time Tags map[string]string + TotalSize int64 RootObject ObjectMetadata } diff --git a/pkg/types/node_agent.go b/pkg/types/node_agent.go index f456bbf55..42fe06f58 100644 --- a/pkg/types/node_agent.go +++ b/pkg/types/node_agent.go @@ -74,6 +74,10 @@ type CachePVC struct { ResidentThresholdInMB int64 `json:"residentThresholdInMB,omitempty"` } +type CSISnapshotMetadataService struct { + SAName string `json:"saName,omitempty"` +} + type NodeAgentConfigs struct { // LoadConcurrency is the config for data path load concurrency per node. LoadConcurrency *LoadConcurrency `json:"loadConcurrency,omitempty"` @@ -104,4 +108,7 @@ type NodeAgentConfigs struct { // PodLabels are labels to be added to pods created by node-agent, i.e., data mover pods. PodLabels map[string]string `json:"podLabels,omitempty"` + + // CSISnapshotMetadataServiceConfigs is the config for CSI snapshot metadata service + CSISnapshotMetadataServiceConfigs *CSISnapshotMetadataService `json:"csiSnapshotMetadataServiceConfigs,omitempty"` } diff --git a/pkg/util/kube/utils.go b/pkg/util/kube/utils.go index 5e5e97603..d93effd7f 100644 --- a/pkg/util/kube/utils.go +++ b/pkg/util/kube/utils.go @@ -53,6 +53,7 @@ const ( KubeAnnDynamicallyProvisioned = "pv.kubernetes.io/provisioned-by" KubeAnnMigratedTo = "pv.kubernetes.io/migrated-to" KubeAnnSelectedNode = "volume.kubernetes.io/selected-node" + KubeAnnAllowVolumeModeChange = "snapshot.storage.kubernetes.io/allow-volume-mode-change" ) // VolumeSnapshotContentManagedByLabel is applied by the snapshot controller diff --git a/test/e2e/README.md b/test/e2e/README.md index f4e7714af..a621ee70c 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -293,18 +293,18 @@ E2E tests can be run with specific cases to be included and/or excluded using th 1. Run Velero tests with specific cases to be included: ```bash -GINKGO_LABELS="Basic && Restic" \ +GINKGO_LABELS="Basic && FSBackup" \ CLOUD_PROVIDER=aws \ BSL_BUCKET=example-bucket \ CREDS_FILE=/path/to/aws-creds \ make test-e2e \ ``` -In this example, only case have both `Basic` and `Restic` labels are included. +In this example, only case have both `Basic` and `FSBackup` labels are included. 1. Run Velero tests with specific cases to be excluded: ```bash -GINKGO_LABELS="!(Scale || Schedule || TTL || (Upgrade && Restic) || (Migration && Restic))" \ +GINKGO_LABELS="!(Scale || Schedule || TTL || (Upgrade && FSBackup) || (Migration && FSBackup))" \ CLOUD_PROVIDER=aws \ BSL_BUCKET=example-bucket \ CREDS_FILE=/path/to/aws-creds \ @@ -315,8 +315,8 @@ In this example, cases are labelled as * `Scale` * `Schedule` * `TTL` -* `Upgrade` and `Restic` -* `Migration` and `Restic` +* `Upgrade` and `FSBackup` +* `Migration` and `FSBackup` will be skipped. #### VKS environment test @@ -370,9 +370,7 @@ Following pipelines should cover all E2E tests along with proper filters: 1. **CSI pipeline:** As we can see lots of labels in E2E test code, there're many snapshot-labeled test scripts. To cover CSI scenario, a pipeline with CSI enabled should be a good choice, otherwise, we will double all the snapshot cases for CSI scenario, it's very time-wasting. By providing `FEATURES=EnableCSI` and `PLUGINS=`, a CSI pipeline is ready for testing. 1. **Data mover pipeline:** Data mover scenario is the same scenario with migaration test except the restriction of migaration between different providers, so it better to separated it out from other pipelines. Please refer the example in previous. -1. **Restic/Kopia backup path pipelines:** - 1. **Restic pipeline:** For the same reason of saving time, set `UPLOADER_TYPE` to `restic` for all file system backup test cases; - 1. **Kopia pipeline:** Set `UPLOADER_TYPE` to `kopia` for all file system backup test cases; +1. **File system backup pipeline:** Set `UPLOADER_TYPE` to `kopia` for all file system backup test cases; 1. **Long time pipeline:** Long time cases should be group into one pipeline, currently these test cases with labels `Scale`, `Schedule` or `TTL` can be group into a pipeline, and make sure to skip them off in any other pipelines. **Note:** please organize filters among proper pipelines for other test cases. diff --git a/test/e2e/backup/backup.go b/test/e2e/backup/backup.go index 43a6f4f23..dd964e73c 100644 --- a/test/e2e/backup/backup.go +++ b/test/e2e/backup/backup.go @@ -43,7 +43,7 @@ func BackupRestoreWithSnapshots() { BackupRestoreTest(config) } -func BackupRestoreWithRestic() { +func BackupRestoreWithFSBackup() { config := BackupRestoreTestConfig{false, "", false} BackupRestoreTest(config) } @@ -53,7 +53,7 @@ func BackupRestoreRetainedPVWithSnapshots() { BackupRestoreTest(config) } -func BackupRestoreRetainedPVWithRestic() { +func BackupRestoreRetainedPVWithFSBackup() { config := BackupRestoreTestConfig{false, "overlays/sc-reclaim-policy/", true} BackupRestoreTest(config) } diff --git a/test/e2e/backups/deletion.go b/test/e2e/backups/deletion.go index 54388b70a..e35b94b53 100644 --- a/test/e2e/backups/deletion.go +++ b/test/e2e/backups/deletion.go @@ -34,13 +34,11 @@ import ( . "github.com/vmware-tanzu/velero/test/util/velero" ) -// Test backup and restore of Kibishii using restic - func BackupDeletionWithSnapshots() { backup_deletion_test(true) } -func BackupDeletionWithRestic() { +func BackupDeletionWithFSBackup() { backup_deletion_test(false) } func backup_deletion_test(useVolumeSnapshots bool) { diff --git a/test/e2e/basic/namespace-mapping.go b/test/e2e/basic/namespace-mapping.go index 6816b2df0..a5a0cc1be 100644 --- a/test/e2e/basic/namespace-mapping.go +++ b/test/e2e/basic/namespace-mapping.go @@ -21,8 +21,8 @@ type NamespaceMapping struct { const NamespaceBaseName string = "ns-mp-" -var OneNamespaceMappingResticTest func() = TestFunc(&NamespaceMapping{TestCase: TestCase{NamespacesTotal: 1, UseVolumeSnapshots: false}}) -var MultiNamespacesMappingResticTest func() = TestFunc(&NamespaceMapping{TestCase: TestCase{NamespacesTotal: 2, UseVolumeSnapshots: false}}) +var OneNamespaceMappingFSBackupTest func() = TestFunc(&NamespaceMapping{TestCase: TestCase{NamespacesTotal: 1, UseVolumeSnapshots: false}}) +var MultiNamespacesMappingFSBackupTest func() = TestFunc(&NamespaceMapping{TestCase: TestCase{NamespacesTotal: 2, UseVolumeSnapshots: false}}) var OneNamespaceMappingSnapshotTest func() = TestFunc(&NamespaceMapping{TestCase: TestCase{NamespacesTotal: 1, UseVolumeSnapshots: true}}) var MultiNamespacesMappingSnapshotTest func() = TestFunc(&NamespaceMapping{TestCase: TestCase{NamespacesTotal: 2, UseVolumeSnapshots: true}}) @@ -37,7 +37,7 @@ func (n *NamespaceMapping) Init() error { if n.VeleroCfg.CloudProvider == "kind" { n.kibishiiData = &KibishiiData{Levels: 0, DirsPerLevel: 0, FilesPerLevel: 0, FileLength: 0, BlockSize: 0, PassNum: 0, ExpectedNodes: 2} } - backupType := "restic" + backupType := "fs-backup" if n.UseVolumeSnapshots { backupType = "snapshot" } diff --git a/test/e2e/bsl-mgmt/deletion.go b/test/e2e/bsl-mgmt/deletion.go index 838aa67b5..6423b203c 100644 --- a/test/e2e/bsl-mgmt/deletion.go +++ b/test/e2e/bsl-mgmt/deletion.go @@ -41,13 +41,11 @@ const ( bslDeletionTestNs = "bsl-deletion" ) -// Test backup and restore of Kibishii using restic - func BslDeletionWithSnapshots() { BslDeletionTest(true) } -func BslDeletionWithRestic() { +func BslDeletionWithFSBackup() { BslDeletionTest(false) } func BslDeletionTest(useVolumeSnapshots bool) { @@ -89,7 +87,7 @@ func BslDeletionTest(useVolumeSnapshots bool) { }) When("kibishii is the sample workload", func() { - It("Local backups and restic repos (if Velero was installed with Restic) will be deleted once the corresponding backup storage location is deleted", func() { + It("Local backups and backup repos will be deleted once the corresponding backup storage location is deleted", func() { oneHourTimeout, ctxCancel := context.WithTimeout(context.Background(), time.Minute*60) defer ctxCancel() if veleroCfg.AdditionalBSLProvider == "" { @@ -165,7 +163,7 @@ func BslDeletionTest(useVolumeSnapshots bool) { )).To(Succeed()) }) - // Restic can not backup PV only, so pod need to be labeled also + // FS backup can not backup PV only, so pod need to be labeled also By("Label all 2 worker-pods of Kibishii", func() { Expect(AddLabelToPod(context.Background(), podName1, bslDeletionTestNs, label1)).To(Succeed()) Expect(AddLabelToPod(context.Background(), "kibishii-deployment-1", bslDeletionTestNs, label2)).To(Succeed()) diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 909cc8074..a8bbbce4c 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -397,11 +397,10 @@ var _ = Describe( APIExtensionsVersionsTest, ) -// Test backup and restore of Kibishii using restic var _ = Describe( - "Velero tests on cluster using the plugin provider for object storage and Restic for volume backups", - Label("Basic", "Restic", "AdditionalBSL"), - BackupRestoreWithRestic, + "Velero tests on cluster using the plugin provider for object storage and file system backup for volumes", + Label("Basic", "FSBackup", "AdditionalBSL"), + BackupRestoreWithFSBackup, ) var _ = Describe( @@ -417,9 +416,9 @@ var _ = Describe( ) var _ = Describe( - "Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", - Label("Basic", "Restic", "RetainPV", "AdditionalBSL"), - BackupRestoreRetainedPVWithRestic, + "Velero tests on cluster using the plugin provider for object storage and file system backup for volumes", + Label("Basic", "FSBackup", "RetainPV", "AdditionalBSL"), + BackupRestoreRetainedPVWithFSBackup, ) var _ = Describe( @@ -452,11 +451,10 @@ var _ = Describe( MultiNSBackupRestore, ) -// Upgrade test by Kibishii using Restic var _ = Describe( - "Velero upgrade tests on cluster using the plugin provider for object storage and Restic for volume backups", - Label("Upgrade", "Restic"), - BackupUpgradeRestoreWithRestic, + "Velero upgrade tests on cluster using the plugin provider for object storage and file system backup for volumes", + Label("Upgrade", "FSBackup"), + BackupUpgradeRestoreWithFSBackup, ) var _ = Describe( "Velero upgrade tests on cluster using the plugin provider for object storage and snapshots for volume backups", @@ -522,7 +520,7 @@ var _ = Describe( ) var _ = Describe( "Velero test on skip backup of volume by resource policies", - Label("ResourceFiltering", "ResourcePolicies", "Restic"), + Label("ResourceFiltering", "ResourcePolicies", "FSBackup"), ResourcePoliciesTest, ) @@ -560,9 +558,9 @@ var _ = Describe( ) var _ = Describe( - "Velero tests of Restic backup deletion", - Label("Backups", "Deletion", "Restic"), - BackupDeletionWithRestic, + "Velero tests of file system backup deletion", + Label("Backups", "Deletion", "FSBackup"), + BackupDeletionWithFSBackup, ) var _ = Describe( "Velero tests of snapshot backup deletion", @@ -570,7 +568,7 @@ var _ = Describe( BackupDeletionWithSnapshots, ) var _ = Describe( - "Local backups and Restic repos will be deleted once the corresponding backup storage location is deleted", + "Local backups and backup repos will be deleted once the corresponding backup storage location is deleted", Label("Backups", "TTL", "LongTime", "Snapshot", "SkipVanillaZfs"), TTLTest, ) @@ -608,9 +606,9 @@ var _ = Describe( BslDeletionWithSnapshots, ) var _ = Describe( - "Local backups and Restic repos will be deleted once the corresponding backup storage location is deleted", - Label("BSL", "Deletion", "Restic", "AdditionalBSL"), - BslDeletionWithRestic, + "Local backups and backup repos will be deleted once the corresponding backup storage location is deleted", + Label("BSL", "Deletion", "FSBackup", "AdditionalBSL"), + BslDeletionWithFSBackup, ) var _ = Describe( @@ -626,13 +624,13 @@ var _ = Describe( var _ = Describe( "Backup resources should follow the specific order in schedule", - Label("NamespaceMapping", "Single", "Restic"), - OneNamespaceMappingResticTest, + Label("NamespaceMapping", "Single", "FSBackup"), + OneNamespaceMappingFSBackupTest, ) var _ = Describe( "Backup resources should follow the specific order in schedule", - Label("NamespaceMapping", "Multiple", "Restic"), - MultiNamespacesMappingResticTest, + Label("NamespaceMapping", "Multiple", "FSBackup"), + MultiNamespacesMappingFSBackupTest, ) var _ = Describe( "Backup resources should follow the specific order in schedule", diff --git a/test/e2e/test/test.go b/test/e2e/test/test.go index 1a91c115f..7fd06aa74 100644 --- a/test/e2e/test/test.go +++ b/test/e2e/test/test.go @@ -179,10 +179,6 @@ func (t *TestCase) Start() error { Skip("Skip due to issue https://github.com/kubernetes/kubernetes/issues/114384 on AKS") } - if veleroCfg.UploaderType == UploaderTypeRestic && - strings.Contains(t.GetTestCase().CaseBaseName, "ParallelFiles") { - Skip("Skip Parallel Files upload and download test cases for environments using Restic as uploader.") - } return nil } diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go index 6d081f1c1..4e0a9a8ca 100644 --- a/test/e2e/upgrade/upgrade.go +++ b/test/e2e/upgrade/upgrade.go @@ -43,7 +43,7 @@ func BackupUpgradeRestoreWithSnapshots() { } } -func BackupUpgradeRestoreWithRestic() { +func BackupUpgradeRestoreWithFSBackup() { veleroCfg = VeleroCfg for _, upgradeFromVelero := range GetVersionList(veleroCfg.UpgradeFromVeleroCLI, veleroCfg.UpgradeFromVeleroVersion) { BackupUpgradeRestoreTest(false, upgradeFromVelero) @@ -108,8 +108,6 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC Expect(err).To(Succeed()) oneHourTimeout, ctxCancel := context.WithTimeout(context.Background(), time.Minute*60) defer ctxCancel() - supportUploaderType, err := IsSupportUploaderType(veleroCLI2Version.VeleroVersion) - Expect(err).To(Succeed()) if veleroCLI2Version.VeleroCLI == "" { //Assume tag of velero server image is identical to velero CLI version //Download velero CLI if it's empty according to velero CLI version @@ -182,8 +180,6 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC BackupCfg.UseVolumeSnapshots = useVolumeSnapshots BackupCfg.DefaultVolumesToFsBackup = !useVolumeSnapshots BackupCfg.Selector = "" - //TODO: pay attention to this param, remove it when restic is not the default backup tool any more. - BackupCfg.UseResticIfFSBackup = !supportUploaderType Expect(VeleroBackupNamespace(oneHourTimeout, tmpCfg.UpgradeFromVeleroCLI, tmpCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string { RunDebug(context.Background(), tmpCfg.UpgradeFromVeleroCLI, tmpCfg.VeleroNamespace, @@ -247,17 +243,9 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC tmpCfg.GCFrequency = "" tmpCfg.UseNodeAgent = !useVolumeSnapshots Expect(err).To(Succeed()) - if supportUploaderType { - Expect(VeleroInstall(context.Background(), &tmpCfg, false)).To(Succeed()) - Expect(CheckVeleroVersion(context.Background(), tmpCfg.VeleroCLI, - tmpCfg.VeleroVersion)).To(Succeed()) - } else { - // For upgrade from v1.9 or other version below v1.9 - tmpCfg.UploaderType = "restic" - Expect(VeleroUpgrade(context.Background(), tmpCfg)).To(Succeed()) - Expect(CheckVeleroVersion(context.Background(), tmpCfg.VeleroCLI, - tmpCfg.VeleroVersion)).To(Succeed()) - } + Expect(VeleroInstall(context.Background(), &tmpCfg, false)).To(Succeed()) + Expect(CheckVeleroVersion(context.Background(), tmpCfg.VeleroCLI, + tmpCfg.VeleroVersion)).To(Succeed()) }) // Wait for 70s to make sure the backups are synced after Velero reinstall diff --git a/test/types.go b/test/types.go index ef9582ce3..f09f36fe8 100644 --- a/test/types.go +++ b/test/types.go @@ -44,10 +44,7 @@ const CSI = "csi" const Velero = "velero" const VeleroRestoreHelper = "velero-restore-helper" -const ( - UploaderTypeRestic = "restic" - UploaderTypeKopia = "kopia" -) +const UploaderTypeKopia = "kopia" const ( KubeSystemNamespace = "kube-system" @@ -168,7 +165,6 @@ type BackupConfig struct { ExcludeResources string IncludeClusterResources bool OrderedResources string - UseResticIfFSBackup bool DefaultVolumesToFsBackup bool SnapshotMoveData bool } diff --git a/test/util/velero/install.go b/test/util/velero/install.go index e2c57392e..121f38760 100644 --- a/test/util/velero/install.go +++ b/test/util/velero/install.go @@ -616,10 +616,8 @@ func createVeleroResources(ctx context.Context, cli, namespace string, args []st return errors.Wrapf(err, "failed to run velero install dry run command, stdout=%s, stderr=%s", stdout, stderr) } - // From v1.15, the Restic uploader is deprecated, - // and a warning message is printed for the install CLI. - // Need to skip the deprecation of Restic message before the generated JSON. - // Redirect to the stdout to the first curly bracket to skip the warning. + // The install CLI may print warning messages before the generated JSON. + // Skip any text before the first curly bracket. if stdout[0] != '{' { newIndex := strings.Index(stdout, "{") stdout = stdout[newIndex:] @@ -730,7 +728,7 @@ func patchResources(resources *unstructured.UnstructuredList, namespace string, } } - // customize the restic restore helper image + // customize the restore helper image if len(options.RestoreHelperImage) > 0 { restoreActionConfig := corev1api.ConfigMap{ TypeMeta: metav1.TypeMeta{ @@ -755,7 +753,7 @@ func patchResources(resources *unstructured.UnstructuredList, namespace string, return errors.Wrapf(err, "failed to convert restore action config to unstructure") } resources.Items = append(resources.Items, un) - fmt.Printf("the restic restore helper image is set by the configmap %q \n", "fs-restore-action-config") + fmt.Printf("the restore helper image is set by the configmap %q \n", "fs-restore-action-config") } return nil diff --git a/test/util/velero/velero_utils.go b/test/util/velero/velero_utils.go index 5c9b39a84..df3aeba08 100644 --- a/test/util/velero/velero_utils.go +++ b/test/util/velero/velero_utils.go @@ -41,7 +41,6 @@ import ( schedulingv1api "k8s.io/api/scheduling/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" - ver "k8s.io/apimachinery/pkg/util/version" "k8s.io/apimachinery/pkg/util/wait" kbclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -240,7 +239,7 @@ func getProviderVeleroInstallOptions(veleroCfg *VeleroConfig, } io := cliinstall.NewInstallOptions() - // always wait for velero and restic pods to be running. + // always wait for velero and node-agent pods to be running. io.Wait = true io.ProviderName = veleroCfg.ObjectStoreProvider @@ -471,11 +470,7 @@ func VeleroBackupNamespace(ctx context.Context, veleroCLI, veleroNamespace strin } } if backupCfg.DefaultVolumesToFsBackup { - if backupCfg.UseResticIfFSBackup { - args = append(args, "--default-volumes-to-restic") - } else { - args = append(args, "--default-volumes-to-fs-backup") - } + args = append(args, "--default-volumes-to-fs-backup") // To workaround https://github.com/vmware-tanzu/velero-plugin-for-vsphere/issues/347 for vsphere plugin v1.1.1 // if the "--snapshot-volumes=false" isn't specified explicitly, the vSphere plugin will always take snapshots @@ -484,20 +479,11 @@ func VeleroBackupNamespace(ctx context.Context, veleroCLI, veleroNamespace strin if backupCfg.ProvideSnapshotsVolumeParam && !backupCfg.UseVolumeSnapshots { args = append(args, "--snapshot-volumes=false") } // if "--snapshot-volumes" is not provide, snapshot should be taken as default behavior. - } else { // DefaultVolumesToFsBackup is false + } else if backupCfg.UseVolumeSnapshots { // Although DefaultVolumesToFsBackup is false, but probably DefaultVolumesToFsBackup // was set to true in installation CLI in snapshot volume test, so set DefaultVolumesToFsBackup // to false specifically to make sure volume snapshot was taken - if backupCfg.UseVolumeSnapshots { - if backupCfg.UseResticIfFSBackup { - args = append(args, "--default-volumes-to-restic=false") - } else { - args = append(args, "--default-volumes-to-fs-backup=false") - } - } - // Although DefaultVolumesToFsBackup is false, but probably DefaultVolumesToFsBackup - // was set to true in installation CLI in FS volume backup test, so do nothing here, no DefaultVolumesToFsBackup - // appear in backup CLI + args = append(args, "--default-volumes-to-fs-backup=false") } if backupCfg.BackupLocation != "" { args = append(args, "--storage-location", backupCfg.BackupLocation) @@ -1282,14 +1268,14 @@ func SnapshotCRsCountShouldBe(ctx context.Context, namespace, backupName string, } func BackupRepositoriesCountShouldBe(ctx context.Context, veleroNamespace, targetNamespace string, expectedCount int) error { - resticArr, err := GetRepositories(ctx, veleroNamespace, targetNamespace) + repos, err := GetRepositories(ctx, veleroNamespace, targetNamespace) if err != nil { return errors.Wrapf(err, "Fail to get BackupRepositories") } - if len(resticArr) == expectedCount { + if len(repos) == expectedCount { return nil } else { - return errors.New(fmt.Sprintf("BackupRepositories count %d in namespace %s is not as expected %d", len(resticArr), targetNamespace, expectedCount)) + return errors.New(fmt.Sprintf("BackupRepositories count %d in namespace %s is not as expected %d", len(repos), targetNamespace, expectedCount)) } } @@ -1429,36 +1415,6 @@ func GetSchedule(ctx context.Context, veleroNamespace, scheduleName string) (str return stdout, err } -func VeleroUpgrade(ctx context.Context, veleroCfg VeleroConfig) error { - crd, err := ApplyCRDs(ctx, veleroCfg.VeleroCLI) - if err != nil { - return errors.Wrap(err, "Fail to Apply CRDs") - } - fmt.Println(crd) - deploy, err := UpdateVeleroDeployment(ctx, veleroCfg) - if err != nil { - return errors.Wrap(err, "Fail to update Velero deployment") - } - fmt.Println(deploy) - if veleroCfg.UseNodeAgent { - dsjson, err := KubectlGetDsJson(veleroCfg.VeleroNamespace) - if err != nil { - return errors.Wrap(err, "Fail to update Velero deployment") - } - - err = DeleteVeleroDs(ctx) - if err != nil { - return errors.Wrap(err, "Fail to delete Velero ds") - } - update, err := UpdateNodeAgent(ctx, veleroCfg, dsjson) - fmt.Println(update) - if err != nil { - return errors.Wrap(err, "Fail to update node agent") - } - } - return waitVeleroReady(ctx, veleroCfg.VeleroNamespace, veleroCfg.UseNodeAgent, veleroCfg.UseNodeAgentWindows) -} - func ApplyCRDs(ctx context.Context, veleroCLI string) ([]string, error) { cmds := []*common.OsCommandLine{} @@ -1476,78 +1432,6 @@ func ApplyCRDs(ctx context.Context, veleroCLI string) ([]string, error) { return common.GetListByCmdPipes(ctx, cmds) } -func UpdateVeleroDeployment(ctx context.Context, veleroCfg VeleroConfig) ([]string, error) { - cmds := []*common.OsCommandLine{} - - cmd := &common.OsCommandLine{ - Cmd: "kubectl", - Args: []string{"get", "deploy", "-n", veleroCfg.VeleroNamespace, "-ojson"}, - } - cmds = append(cmds, cmd) - - cmd = &common.OsCommandLine{ - Cmd: "sed", - Args: []string{fmt.Sprintf("s#\\\"server\\\",#\\\"server\\\",\\\"--uploader-type=%s\\\",#g", veleroCfg.UploaderType)}, - } - cmds = append(cmds, cmd) - - cmd = &common.OsCommandLine{ - Cmd: "sed", - Args: []string{"s#default-volumes-to-restic#default-volumes-to-fs-backup#g"}, - } - cmds = append(cmds, cmd) - - cmd = &common.OsCommandLine{ - Cmd: "sed", - Args: []string{"s#default-restic-prune-frequency#default-repo-maintain-frequency#g"}, - } - cmds = append(cmds, cmd) - - cmd = &common.OsCommandLine{ - Cmd: "sed", - Args: []string{"s#restic-timeout#fs-backup-timeout#g"}, - } - cmds = append(cmds, cmd) - - cmd = &common.OsCommandLine{ - Cmd: "kubectl", - Args: []string{"apply", "-f", "-"}, - } - cmds = append(cmds, cmd) - - return common.GetListByCmdPipes(ctx, cmds) -} - -func UpdateNodeAgent(ctx context.Context, veleroCfg VeleroConfig, dsjson string) ([]string, error) { - cmds := []*common.OsCommandLine{} - - cmd := &common.OsCommandLine{ - Cmd: "echo", - Args: []string{dsjson}, - } - cmds = append(cmds, cmd) - - cmd = &common.OsCommandLine{ - Cmd: "sed", - Args: []string{"s#\\\"name\\\"\\: \\\"restic\\\"#\\\"name\\\"\\: \\\"node-agent\\\"#g"}, - } - cmds = append(cmds, cmd) - - cmd = &common.OsCommandLine{ - Cmd: "sed", - Args: []string{"s#\\\"restic\\\",#\\\"node-agent\\\",#g"}, - } - cmds = append(cmds, cmd) - - cmd = &common.OsCommandLine{ - Cmd: "kubectl", - Args: []string{"create", "-f", "-"}, - } - cmds = append(cmds, cmd) - - return common.GetListByCmdPipes(ctx, cmds) -} - func ListVeleroPods(ctx context.Context, veleroNamespace string) ([]string, error) { cmds := []*common.OsCommandLine{} cmd := &common.OsCommandLine{ @@ -1591,22 +1475,6 @@ func RestorePVRNum(ctx context.Context, veleroNamespace, restoreName string) (in return len(outputList), err } -func IsSupportUploaderType(version string) (bool, error) { - verSupportUploaderType, err := ver.ParseSemantic("v1.10.0") - if err != nil { - return false, err - } - v, err := ver.ParseSemantic(version) - if err != nil { - return false, err - } - if v.AtLeast(verSupportUploaderType) { - return true, nil - } else { - return false, nil - } -} - func GetVeleroPodName(ctx context.Context) ([]string, error) { // Example: // NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE