mirror of
https://github.com/vmware-tanzu/velero.git
synced 2026-06-10 00:03:10 +00:00
Compare commits
105 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 323900adcc | |||
| 317ffd069f | |||
| d6d9e4ee16 | |||
| 2ee99e75cd | |||
| dda779de65 | |||
| 52860f986e | |||
| 283ee24632 | |||
| 50ea4eea74 | |||
| 3b545b506b | |||
| d46bf8a337 | |||
| b34c8436aa | |||
| 0d719f1d8a | |||
| ca0506daa8 | |||
| eb0659f06d | |||
| 5160fb1410 | |||
| 85a98b73a5 | |||
| 4374948830 | |||
| 09bfc69d63 | |||
| 869ec523af | |||
| d435b0509e | |||
| 47822b7ed2 | |||
| 2e1ab5ab62 | |||
| 5cc0398662 | |||
| 3d085de99c | |||
| b7d5d84983 | |||
| 1318d2c5dd | |||
| 74ffe25cbe | |||
| e4ecf26b33 | |||
| 2863f0df48 | |||
| cdf3b9ffaa | |||
| 89be6c01df | |||
| 0a7e5d600b | |||
| 44eaea8faf | |||
| 8575ff031d | |||
| 91e2d93576 | |||
| 1186db83bf | |||
| 2679db6b42 | |||
| 8bca779270 | |||
| a606610e02 | |||
| ac1e472d53 | |||
| 9552f9f656 | |||
| a34a676fa3 | |||
| b278d38f7e | |||
| b29185a62d | |||
| fb603765e2 | |||
| c51fac63d5 | |||
| 57c4c6fc99 | |||
| 3c4c793683 | |||
| 596e774582 | |||
| f5fe41dabf | |||
| 219975bee0 | |||
| 0b7eaaf4e6 | |||
| d987388698 | |||
| 3e88666872 | |||
| a2b1af9059 | |||
| d08790534a | |||
| d58139536b | |||
| 40025fbbe1 | |||
| 4dc6f2cc64 | |||
| c1fc6540fb | |||
| 53f25cde2b | |||
| 205ca71588 | |||
| cb9339d85e | |||
| e0ef8d7690 | |||
| 348f9227aa | |||
| 9f0194d8fe | |||
| b91d34065b | |||
| 440b473ca2 | |||
| 1e1eb0b4ec | |||
| 2f19c3158b | |||
| b0e72333a0 | |||
| cf5f5de911 | |||
| 343ed95a5e | |||
| 3103318c9b | |||
| 32969856af | |||
| d756df874f | |||
| f70e339fd0 | |||
| 6bf73dc7ac | |||
| 1d03217661 | |||
| d30d389b56 | |||
| 30381a60e4 | |||
| a26dd817b6 | |||
| 7472e37d16 | |||
| bbbff59eed | |||
| 174d76c197 | |||
| 8ef7e36054 | |||
| 8f6c563c4d | |||
| fb3f94bc88 | |||
| 2346314729 | |||
| 68fa6f4ee9 | |||
| 6257282117 | |||
| 0be9fc7d09 | |||
| 22be7e3218 | |||
| 6a67f4a8a4 | |||
| 6ca73a00b6 | |||
| 4f34ae17a3 | |||
| 44ab9a6a1a | |||
| 4befbc0afe | |||
| 9bbadf346d | |||
| 3886dc0e9a | |||
| 6b7df3ef4c | |||
| 26b125769e | |||
| a1fd85c791 | |||
| b1827074e5 | |||
| 3ef30897ae |
@@ -95,9 +95,9 @@ jobs:
|
||||
\"k8s\":$(wget -q -O - "https://hub.docker.com/v2/namespaces/kindest/repositories/node/tags?page_size=50" | grep -o '"name": *"[^"]*' | grep -o '[^"]*$' | grep -v -E "alpha|beta" | grep -E "v[1-9]\.(2[5-9]|[3-9][0-9])" | awk -F. '{if(!a[$1"."$2]++)print $1"."$2"."$NF}' | sort -r | sed s/v//g | jq -R -c -s 'split("\n")[:-1]'),\
|
||||
\"labels\":[\
|
||||
\"Basic && (ClusterResource || NodePort || StorageClass)\", \
|
||||
\"ResourceFiltering && !Restic\", \
|
||||
\"ResourceFiltering && !FSBackup\", \
|
||||
\"ResourceModifier || (Backups && BackupsSync) || PrivilegesMgmt || OrderedResources\", \
|
||||
\"(NamespaceMapping && Single && Restic) || (NamespaceMapping && Multiple && Restic)\"\
|
||||
\"(NamespaceMapping && Single && FSBackup) || (NamespaceMapping && Multiple && FSBackup)\"\
|
||||
]}" >> $GITHUB_OUTPUT
|
||||
|
||||
# Run E2E test against all Kubernetes versions on kind
|
||||
@@ -136,7 +136,7 @@ jobs:
|
||||
- uses: engineerd/setup-kind@v0.6.2
|
||||
with:
|
||||
skipClusterLogsExport: true
|
||||
version: "v0.27.0"
|
||||
version: "v0.32.0"
|
||||
image: "kindest/node:v${{ matrix.k8s }}"
|
||||
- name: Fetch built CLI
|
||||
id: cli-cache
|
||||
|
||||
@@ -24,7 +24,7 @@ jobs:
|
||||
- name: Make ci
|
||||
run: make ci
|
||||
- name: Upload test coverage
|
||||
uses: codecov/codecov-action@v5
|
||||
uses: codecov/codecov-action@v6
|
||||
with:
|
||||
token: ${{ secrets.CODECOV_TOKEN }}
|
||||
files: coverage.out
|
||||
|
||||
@@ -45,7 +45,7 @@ jobs:
|
||||
- name: Test
|
||||
run: make test
|
||||
- name: Upload test coverage
|
||||
uses: codecov/codecov-action@v5
|
||||
uses: codecov/codecov-action@v6
|
||||
with:
|
||||
token: ${{ secrets.CODECOV_TOKEN }}
|
||||
files: coverage.out
|
||||
|
||||
@@ -20,4 +20,4 @@ jobs:
|
||||
days-before-pr-close: -1
|
||||
# Only issues made after Feb 09 2021.
|
||||
start-date: "2021-09-02T00:00:00"
|
||||
exempt-issue-labels: "Epic,Area/CLI,Area/Cloud/AWS,Area/Cloud/Azure,Area/Cloud/GCP,Area/Cloud/vSphere,Area/CSI,Area/Design,Area/Documentation,Area/Plugins,Bug,Enhancement/User,kind/requirement,kind/refactor,kind/tech-debt,limitation,Needs investigation,Needs triage,Needs Product,P0 - Hair on fire,P1 - Important,P2 - Long-term important,P3 - Wouldn't it be nice if...,Product Requirements,Restic - GA,Restic,release-blocker,Security,backlog"
|
||||
exempt-issue-labels: "Epic,Area/CLI,Area/Cloud/AWS,Area/Cloud/Azure,Area/Cloud/GCP,Area/Cloud/vSphere,Area/CSI,Area/Design,Area/Documentation,Area/Plugins,Bug,Enhancement/User,kind/requirement,kind/refactor,kind/tech-debt,limitation,Needs investigation,Needs triage,Needs Product,P0 - Hair on fire,P1 - Important,P2 - Long-term important,P3 - Wouldn't it be nice if...,Product Requirements,release-blocker,Security,backlog"
|
||||
|
||||
+2
-2
@@ -49,9 +49,9 @@ RUN mkdir -p /output/usr/bin && \
|
||||
go clean -modcache -cache
|
||||
|
||||
# Velero image packing section
|
||||
FROM paketobuildpacks/run-jammy-tiny:latest
|
||||
FROM paketobuildpacks/ubuntu-noble-run-tiny:latest
|
||||
|
||||
LABEL maintainer="Xun Jiang <jxun@vmware.com>"
|
||||
LABEL maintainer="Xun Jiang <xun.jiang@broadcom.com>"
|
||||
|
||||
COPY --from=velero-builder /output /
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
![100]
|
||||
|
||||
[![Build Status][1]][2] [](https://bestpractices.coreinfrastructure.org/projects/3811)
|
||||

|
||||

|
||||
|
||||
## Overview
|
||||
|
||||
@@ -52,14 +52,29 @@ Velero supports IPv4, IPv6, and dual stack environments. Support for this was te
|
||||
|
||||
The Velero maintainers are continuously working to expand testing coverage, but are not able to test every combination of Velero and supported Kubernetes versions for each Velero release. The table above is meant to track the current testing coverage and the expected supported Kubernetes versions for each Velero version.
|
||||
|
||||
If you are interested in using a different version of Kubernetes with a given Velero version, we'd recommend that you perform testing before installing or upgrading your environment. For full information around capabilities within a release, also see the Velero [release notes](https://github.com/vmware-tanzu/velero/releases) or Kubernetes [release notes](https://github.com/kubernetes/kubernetes/tree/master/CHANGELOG). See the Velero [support page](https://velero.io/docs/latest/support-process/) for information about supported versions of Velero.
|
||||
If you are interested in using a different version of Kubernetes with a given Velero version, we'd recommend that you perform testing before installing or upgrading your environment. For full information around capabilities within a release, also see the Velero [release notes](https://github.com/velero-io/velero/releases) or Kubernetes [release notes](https://github.com/kubernetes/kubernetes/tree/master/CHANGELOG). See the Velero [support page](https://velero.io/docs/latest/support-process/) for information about supported versions of Velero.
|
||||
|
||||
For each release, Velero maintainers run the test to ensure the upgrade path from n-2 minor release. For example, before the release of v1.10.x, the test will verify that the backup created by v1.9.x and v1.8.x can be restored using the build to be tagged as v1.10.x.
|
||||
|
||||
[1]: https://github.com/vmware-tanzu/velero/workflows/Main%20CI/badge.svg
|
||||
[2]: https://github.com/vmware-tanzu/velero/actions?query=workflow%3A"Main+CI"
|
||||
[4]: https://github.com/vmware-tanzu/velero/issues
|
||||
[6]: https://github.com/vmware-tanzu/velero/releases
|
||||
## Cloud Native Computing Foundation
|
||||
<!-- remove sandbox once promoted -->
|
||||
Velero is a [Cloud Native Computing Foundation](https://www.cncf.io/) sandbox project.
|
||||
|
||||
<p align="center">
|
||||
<a href="https://www.cncf.io/">
|
||||
<img src="https://raw.githubusercontent.com/cncf/artwork/main/other/cncf/horizontal/color/cncf-color.svg"
|
||||
alt="Cloud Native Computing Foundation logo" width="300"/>
|
||||
</a>
|
||||
</p>
|
||||
|
||||
Copyright Contributors to Velero, established as Velero a Series of LF Projects, LLC.
|
||||
For website terms of use, trademark policy and other project policies please see
|
||||
<https://lfprojects.org/policies/>.
|
||||
|
||||
[1]: https://github.com/velero-io/velero/workflows/Main%20CI/badge.svg
|
||||
[2]: https://github.com/velero-io/velero/actions?query=workflow%3A"Main+CI"
|
||||
[4]: https://github.com/velero-io/velero/issues
|
||||
[6]: https://github.com/velero-io/velero/releases
|
||||
[9]: https://kubernetes.io/docs/setup/
|
||||
[10]: https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-with-homebrew-on-macos
|
||||
[11]: https://kubernetes.io/docs/tasks/tools/install-kubectl/#tabset-1
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
Add CBT bitmap implementation for block data mover
|
||||
@@ -0,0 +1 @@
|
||||
Fix DataUploadDeleteAction creating snapshot-info ConfigMaps labeled with the wrong backup name when a DataUpload CR from another backup is incidentally captured in the backup tarball, which caused Kopia snapshots to be leaked in object storage on expiry of the real owning backup.
|
||||
@@ -0,0 +1 @@
|
||||
Restores from backups not in a completed or partially failed phase are now rejected.
|
||||
@@ -0,0 +1 @@
|
||||
Uploader interface for block data mover
|
||||
@@ -0,0 +1 @@
|
||||
Add Kopia repo snapshot operations for block data mover
|
||||
@@ -0,0 +1 @@
|
||||
Add metadata operation to Kopia repo for block data mover
|
||||
@@ -0,0 +1 @@
|
||||
Data path naming adjustment for block data mover
|
||||
@@ -0,0 +1 @@
|
||||
Fix issue #9811, add interface to support ClusterScopedFilterPolicy and NamespacedFilterPolicy
|
||||
@@ -0,0 +1 @@
|
||||
Fix issue #9812, validate ClusterScopedFilterPolicy and NamespacedFilterPolicy incompatible with legacy filters
|
||||
@@ -0,0 +1 @@
|
||||
Replace vmware-tanzu/velero GitHub org references with velero-io/velero (#9844)
|
||||
@@ -0,0 +1 @@
|
||||
Fix issue #9823, add incremental aware object writer for block data mover
|
||||
@@ -0,0 +1 @@
|
||||
Implement the CBT service.
|
||||
@@ -0,0 +1 @@
|
||||
Fix issue #9813, add validations for ClusterScopedFilterPolicy
|
||||
@@ -0,0 +1 @@
|
||||
Fix issue #9814, add validations for NamespacedFilterPolicies
|
||||
@@ -0,0 +1 @@
|
||||
Add the Write implementation for incremental aware object writer
|
||||
@@ -0,0 +1 @@
|
||||
Remove restic command package
|
||||
@@ -0,0 +1 @@
|
||||
Enhance backup exposer for block data mover
|
||||
@@ -0,0 +1 @@
|
||||
Add cbt service parameters to node-agent-config for block data mover
|
||||
@@ -0,0 +1 @@
|
||||
Remove Restic cases and workflow from E2E
|
||||
@@ -0,0 +1 @@
|
||||
Add totalSize to repo snapshot operation; and pin unified repo snapshots to Kopia repository and so pin them with Velero backup lifecycle
|
||||
@@ -0,0 +1 @@
|
||||
Fix issue #9816, add cli support for backup with ClusterScopedFilterPolicy and NamespacedFilterPolicies
|
||||
@@ -0,0 +1 @@
|
||||
Make ToSystemAffinity deterministic by sorting MatchLabels keys to avoid spurious affinity spec diffs and restarts
|
||||
@@ -0,0 +1,839 @@
|
||||
# Fine Grained Backup Filters via Resource Policies
|
||||
|
||||
## Glossary & Abbreviation
|
||||
|
||||
**Backup Filter**: The mechanism in Velero that determines which Kubernetes resources are collected from the cluster and written into the backup archive. Backup filters currently operate on four dimensions: namespace, resource type, label, and cluster scope.
|
||||
**Global Filter**: A filter that applies uniformly across all namespaces in a backup. All existing Velero backup filters are global filters.
|
||||
**Namespace-Scoped Filter**: A filter that applies only within specific namespaces, overriding the global filter for those namespaces. This is the capability introduced by this design.
|
||||
**ClusterScopedFilterPolicy**: A global filter for cluster-scoped resources that allows per-kind label selectors and name patterns, functioning similarly to `NamespacedFilterPolicy` but applied to cluster-scoped resources globally.
|
||||
**Resource Filter**: A filter rule that pairs one or more resource kinds with their own label selector and/or name patterns. Multiple resource filters within a namespace-scoped policy allow different filtering criteria for different resource types.
|
||||
**Resource Name Filter**: A filter that matches individual resource instances by their metadata.name, using glob patterns. This is a new filter dimension introduced by this design.
|
||||
**Resource Policy**: An existing Velero mechanism where backup behavior rules are defined in a ConfigMap and referenced from `BackupSpec.ResourcePolicy`. Currently used for volume policies and global include/exclude policies.
|
||||
|
||||
## Background
|
||||
|
||||
Velero's backup filter system allows users to specify which resources to include or exclude from a backup. The filters operate on three dimensions:
|
||||
|
||||
1. **Namespace** — `IncludedNamespaces`/`ExcludedNamespaces` select which namespaces to back up
|
||||
2. **Resource Type** — `IncludedResources`/`ExcludedResources` (or the newer scoped variants `Included/ExcludedClusterScopedResources`, `Included/ExcludedNamespaceScopedResources`) select which Kubernetes resource types to back up
|
||||
3. **Labels** — `LabelSelector`/`OrLabelSelectors` filter individual objects by their labels
|
||||
|
||||
All three dimensions are applied **globally** — the same resource type filter, the same label selector, and the same namespace list apply uniformly throughout the entire backup operation. Specifically:
|
||||
|
||||
- In `item_collector.go`, the `ResourceIncludesExcludes.ShouldInclude()` check is a single global check applied to every resource type across all namespaces.
|
||||
- In `listResourceByLabelsPerNamespace()`, the same `LabelSelector` is passed to every Kubernetes API list call regardless of namespace.
|
||||
- There is no mechanism to filter resources by their individual `metadata.name`.
|
||||
|
||||
This creates three critical gaps for common backup scenarios:
|
||||
|
||||
**Gap 1: Different resource needs per namespace.** When multiple applications share the same cluster, different namespaces often require different backup strategies. For example, a namespace running a database workload may need all resource types backed up, while a namespace running a stateless frontend may only need Deployments, ConfigMaps, and Services. Setting `IncludedResources: [configmaps]` means *all* ConfigMaps in *all* included namespaces — you cannot say "only ConfigMaps in namespace-a but everything in namespace-b."
|
||||
|
||||
**Gap 2: Same resource type, different workloads.** Resources of the same type (e.g., ConfigMaps or Secrets) in the same namespace may belong to different workloads. For instance, a namespace may contain `app-config`, `app-secret`, `monitoring-config`, and `monitoring-secret`. Without name-based filtering, you cannot selectively back up only the `app-*` resources — the only option is label-based selection, which requires workloads to have been pre-labeled appropriately.
|
||||
|
||||
**Gap 3: Different kinds need different selectors.** Within a single namespace, different resource types may belong to different workloads with different labels. For example, Deployments labeled `app=workload-1` and StatefulSets labeled `app=workload-2` in the same namespace. The current single-label-selector-per-namespace model cannot express this — the label selector applies identically to all resource types.
|
||||
|
||||
## Goals
|
||||
|
||||
- Extend the `ResourcePolicies` ConfigMap format with a `namespacedFilterPolicies` section that allows per-namespace, per-kind resource filtering with independent label selectors and name patterns for each resource type
|
||||
- Extend the `ResourcePolicies` ConfigMap format with a `clusterScopedFilterPolicy` section that allows per-kind resource filtering with independent label selectors and name patterns for cluster-scoped resources globally
|
||||
- Support resource name filtering by glob patterns using the same `gobwas/glob` library that Velero uses for namespace patterns, ensuring consistency across the codebase
|
||||
- Support per-kind label selectors, so that different resource types within the same namespace can be filtered with different labels
|
||||
- Maintain full backward compatibility — existing backups with no `namespacedFilterPolicies` behave exactly as they do today
|
||||
- Define clear precedence rules for how per-namespace filters interact with global filters
|
||||
- Add corresponding validation within the Resource Policies validation pipeline using existing Velero wildcard validation functions
|
||||
- Update `velero backup describe` output to display per-namespace filter information when present
|
||||
- Ensure the restore process works correctly with backups produced by namespace-scoped filters, without requiring restore-side code changes in the initial phase
|
||||
|
||||
## Non-Goals
|
||||
|
||||
- Adding namespace-scoped filters to `RestoreSpec` or the restore pipeline is not part of the initial implementation. Restore from a namespace-filtered backup works automatically because the restore process reads whatever is in the backup archive. Restore-side namespace filters will be addressed in a follow-up.
|
||||
- Changing existing `BackupSpec` fields (`IncludedResources`, `LabelSelector`, etc.) or adding new CRD fields is explicitly avoided by this design.
|
||||
- Supporting regex patterns for resource names is not included. Glob patterns (already used throughout Velero) are sufficient and consistent.
|
||||
- Modifying the plugin `ResourceSelector` system (`AppliesTo()` / `resolvedAction.ShouldUse()`) is not part of this design.
|
||||
- CLI flags for inline specification of namespace-scoped filters are not part of the initial implementation. The configuration is expressed in the ResourcePolicy ConfigMap YAML.
|
||||
|
||||
## Architecture of Namespace-Scoped Filters
|
||||
|
||||
### Configuration Model
|
||||
|
||||
The namespace-scoped filters and fine-grained global filters are defined in the same ResourcePolicy ConfigMap that is already referenced by `BackupSpec.ResourcePolicy`. The YAML format is extended with two new top-level keys: `clusterScopedFilterPolicy` and `namespacedFilterPolicies`
|
||||
|
||||
```yaml
|
||||
version: v1
|
||||
volumePolicies:
|
||||
# existing volume policies (unchanged)
|
||||
- conditions:
|
||||
capacity: "0,100Gi"
|
||||
action:
|
||||
type: skip
|
||||
includeExcludePolicy:
|
||||
# existing global include/exclude policy (unchanged)
|
||||
includedNamespaceScopedResources:
|
||||
- configmaps
|
||||
clusterScopedFilterPolicy:
|
||||
# NEW: global overrides for cluster-scoped resources
|
||||
resourceFilters:
|
||||
- kinds: [ClusterRole, ClusterRoleBinding]
|
||||
names: ["my-app-*"]
|
||||
- kinds: [CustomResourceDefinition]
|
||||
labelSelector:
|
||||
app: my-app
|
||||
namespacedFilterPolicies:
|
||||
# NEW: per-namespace filter overrides
|
||||
- namespaces:
|
||||
- ns-a
|
||||
resourceFilters:
|
||||
- kinds: [ConfigMap, Secret, Deployment]
|
||||
labelSelector:
|
||||
app: my-app
|
||||
- namespaces:
|
||||
- ns-b
|
||||
resourceFilters:
|
||||
- kinds: [Deployment]
|
||||
names: [app-1, app-2]
|
||||
- kinds: [ConfigMap]
|
||||
labelSelector:
|
||||
app: my-service
|
||||
```
|
||||
|
||||
All four sections coexist in the same ConfigMap. They are independent — `volumePolicies` handles volume backup strategy, `includeExcludePolicy` handles global resource type filtering, `clusterScopedFilterPolicy` handles cluster-scoped resource filtering by kind/name/label, and `namespacedFilterPolicies` handles per-namespace, per-kind overrides.
|
||||
|
||||
### The `resourceFilters` Model
|
||||
|
||||
Each `namespacedFilterPolicies` entry targets one or more namespaces and contains a `resourceFilters` array. Each entry in `resourceFilters` pairs one or more resource kinds with their own label selector and name patterns:
|
||||
|
||||
```yaml
|
||||
namespacedFilterPolicies:
|
||||
- namespaces: [ns-a]
|
||||
resourceFilters:
|
||||
- kinds: [ConfigMap, Secret] # these kinds share a selector
|
||||
labelSelector: {app: my-app}
|
||||
names: ["app-*"]
|
||||
- kinds: [Deployment] # this kind has its own selector
|
||||
names: [workload-1, workload-2]
|
||||
- kinds: [StatefulSet] # this kind has no extra filtering
|
||||
```
|
||||
|
||||
This model has one way to express filters — there is no ambiguity about how to structure the configuration. Only resource kinds listed in `resourceFilters` entries are included in the backup for the matched namespaces; unlisted kinds are implicitly excluded.
|
||||
|
||||
#### Catch-All Resource Filter (Empty `kinds` or `["*"]`)
|
||||
|
||||
A `ResourceFilter` entry with an empty (or omitted) `kinds` field, or a field explicitly set to `["*"]`, acts as a **catch-all**. Its `labelSelector` or `orLabelSelectors` (if provided) is applied to **all resource types in the namespace that are not already matched by a kind-specific filter entry**. If no selectors are provided, all unlisted resources are included. Using `["*"]` is highly recommended as it makes the catch-all intention explicit and self-documenting.
|
||||
|
||||
**Rules for catch-all entries:**
|
||||
- At most **one** catch-all entry is allowed per `NamespacedFilterPolicy`.
|
||||
- `names` and `excludedNames` are **not** supported on catch-all entries. Name patterns are kind-specific by nature and cannot be applied across arbitrary kinds; use kind-specific entries for name-based filtering.
|
||||
- The catch-all applies to kinds that are **not listed in any other `resourceFilters` entry** in the same policy. Kind-specific entries take precedence over the catch-all.
|
||||
- A catch-all entry **does not inherit or fall back to `BackupSpec.LabelSelector`**. If a catch-all entry has no `labelSelector`/`orLabelSelectors`, all unlisted resource kinds in the namespace are included with **no label filtering** — the global label selector is not applied. Define a catch-all with an explicit `labelSelector` if label-based filtering is desired for unlisted kinds.
|
||||
|
||||
**Evaluation order within a namespace filter policy:**
|
||||
1. For each resource kind encountered during backup, the system first checks whether a kind-specific `resourceFilters` entry exists for that kind.
|
||||
2. If a kind-specific entry exists, it is used exclusively (its label selectors and name patterns apply).
|
||||
3. If no kind-specific entry exists but a catch-all entry is present, the catch-all's `labelSelector`/`orLabelSelectors` is applied to that kind.
|
||||
4. If neither a kind-specific entry nor a catch-all entry exists, the kind is excluded from the backup for that namespace.
|
||||
|
||||
### Filter Precedence Model
|
||||
|
||||
The namespace-scoped filter system and fine-grained global filter system layer on top of the existing global filter system. They intentionally behave differently:
|
||||
- **`namespacedFilterPolicies`** acts as an **exclusive allowlist (boundary)**. Only kinds explicitly listed (or matched by a catch-all) are backed up from that namespace. This gives namespace owners complete and isolated control over their namespace's backup contents, preventing unexpected data spillage from global fallbacks.
|
||||
- **`clusterScopedFilterPolicy`** acts as a **refinement overlay (tweak)**. Unlisted cluster-scoped kinds fall back to the standard global filters. This allows administrators to selectively adjust filtering for a few specific cluster-scoped kinds without rewriting the entire global inclusion list.
|
||||
|
||||
**For Namespace-Scoped Resources:**
|
||||
|
||||
The evaluation order is:
|
||||
|
||||
1. **Global namespace filter** (`BackupSpec.IncludedNamespaces`/`ExcludedNamespaces`) is checked first. A namespace must pass this filter to be considered at all. `namespacedFilterPolicies` cannot override namespace exclusion — if a namespace is excluded globally, no filter policy entry can bring it back.
|
||||
|
||||
2. **Per-namespace filter lookup.** For each namespace that passes the global namespace filter, the system checks whether any `namespacedFilterPolicies` entry matches (by namespace name or glob pattern). If a match is found, the `resourceFilters` array determines what gets backed up for that namespace:
|
||||
- Only resource kinds listed in `resourceFilters[].kinds` are included
|
||||
- Each kind uses its own `labelSelector`/`orLabelSelectors` (if specified)
|
||||
- Each kind uses its own `names`/`excludedNames` patterns (if specified)
|
||||
|
||||
3. **Namespaces without a matching filter policy** continue to use the global filters (`BackupSpec.IncludedResources`, `BackupSpec.LabelSelector`, etc., combined with `includeExcludePolicy`) exactly as they do today.
|
||||
|
||||
4. **If multiple filter policy entries could match the same namespace** (e.g., `team-*` and `team-frontend-*` both matching `team-frontend-prod`), the **first matching policy in the list** is used. **Important: Place more specific patterns before broader patterns** to achieve the intended filtering behavior.
|
||||
|
||||
5. **The `velero.io/exclude-from-backup=true` label** always takes precedence over all filters, regardless of whether the item matches global or per-namespace filters.
|
||||
|
||||
6. **Interaction with `includeExcludePolicy`**: `namespacedFilterPolicies` is a **refinement** of the global resource filter system, not a replacement. Global exclusions defined in `includeExcludePolicy` (e.g., `excludedNamespaceScopedResources: [secrets]`) are applied first at the resource-type level before per-namespace filter policies are consulted. A namespace-scoped filter policy cannot re-include a resource kind that has been globally excluded by `includeExcludePolicy`. For example, if `secrets` is listed under `excludedNamespaceScopedResources`, no `Secret` resources will be backed up from any namespace, even if a `namespacedFilterPolicies` entry explicitly lists `Secret` for that namespace. Users who need per-namespace secret selection must remove `secrets` from the global exclusion list.
|
||||
|
||||
To help users catch this misconfiguration early, Velero logs a warning at backup start when a `namespacedFilterPolicies` entry lists a kind that is globally excluded by `includeExcludePolicy`:
|
||||
```
|
||||
level=warn msg="namespacedFilterPolicies entry lists a kind that is globally excluded by includeExcludePolicy; the per-namespace filter entry has no effect" kind="secrets" namespacePattern="ns-a"
|
||||
```
|
||||
|
||||
**For Cluster-Scoped Resources:**
|
||||
|
||||
1. If `clusterScopedFilterPolicy` is present, it acts as a **refinement overlay** over the existing global filters for cluster-scoped resources. It is NOT an exclusive allowlist.
|
||||
- To back up cluster-scoped resources in a namespace-filtered backup, you must still explicitly include them via `BackupSpec.IncludedClusterScopedResources`.
|
||||
- If a cluster-scoped kind is listed in its `resourceFilters`, its specific `labelSelector`/`orLabelSelectors` and `names`/`excludedNames` patterns are applied.
|
||||
- If a cluster-scoped kind is **not listed**, it falls back to the standard global filters (`BackupSpec.LabelSelector`, etc.) and is included in the backup.
|
||||
|
||||
2. If `clusterScopedFilterPolicy` is absent, Velero falls back to the existing global filters (`IncludedClusterScopedResources`, `IncludedResources`, `LabelSelector`, etc.) for cluster-scoped resources.
|
||||
|
||||
3. **The `velero.io/exclude-from-backup=true` label** always takes precedence over all filters.
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
A["BackupSpec Global<br>IncludedNamespaces / ExcludedNamespaces"]
|
||||
B{Namespace passes<br>global filter?}
|
||||
C[Namespace excluded<br>from backup]
|
||||
D{namespacedFilterPolicies<br>lookup by namespace}
|
||||
E{"For each resource kind:<br>is kind in resourceFilters?"}
|
||||
F["Apply kind-specific filters:<br>- labelSelector / orLabelSelectors<br>- names / excludedNames"]
|
||||
G[Kind skipped for<br>this namespace]
|
||||
H["Use global filters:<br>- BackupSpec IncludedResources<br>- BackupSpec LabelSelector<br>- includeExcludePolicy"]
|
||||
|
||||
I{"Is resource<br>cluster-scoped?"}
|
||||
J{"Is clusterScopedFilterPolicy<br>present?"}
|
||||
K{"Is kind in resourceFilters?"}
|
||||
L["Apply kind-specific filters:<br>- labelSelector / orLabelSelectors<br>- names / excludedNames"]
|
||||
|
||||
I -- Yes --> J
|
||||
J -- Yes --> K
|
||||
K -- Yes --> L
|
||||
K -- No --> H
|
||||
J -- No --> H
|
||||
|
||||
I -- No --> A
|
||||
A --> B
|
||||
B -- No --> C
|
||||
B -- Yes --> D
|
||||
D -- Match found --> E
|
||||
E -- Yes --> F
|
||||
E -- No --> G
|
||||
D -- No match found --> H
|
||||
```
|
||||
|
||||
### Data Flow in the Backup Pipeline
|
||||
|
||||
The existing backup pipeline has two stages: item collection and item backup. Namespace-scoped filters and fine-grained global filters are applied at both stages:
|
||||
|
||||
**Stage 1 — Item Collection (`item_collector.go`).** Resources are listed from the Kubernetes API.
|
||||
|
||||
- **Resource type check** in `getResourceItems()`: Before iterating namespaces, the global resource type check still applies.
|
||||
- **For Cluster-Scoped Resources:** The global resource type check (`ShouldInclude`) determines if the kind is collected. `ClusterScopedFilterPolicy` does not skip unlisted cluster-scoped kinds at this stage.
|
||||
- **For Namespace-Scoped Resources:** Within the namespace loop, a per-namespace resource type check is added. If a filter policy matches the current namespace, only resource kinds listed in `resourceFilters[].kinds` are included — if the current resource type is not listed, it is skipped for that namespace.
|
||||
- **Label selector** in `listResourceByLabelsPerNamespace()` and `listResourceByLabelsGlobally()`: The function looks up the filter policy (either the namespace-specific one or the fine-grained global one). If found, it retrieves the `ResourceFilter` entry for the current resource kind and uses that entry's `labelSelector`/`orLabelSelectors` for the Kubernetes API list call. If no filter policy is found, the global selectors are used as before.
|
||||
|
||||
**Stage 2 — Item Backup (`item_backupper.go`).** Collected items are validated and written to the archive.
|
||||
|
||||
- **Name pattern check** in `itemInclusionChecks()`: After the existing namespace and resource type re-validation, the item's `metadata.name` is checked against the `ResourceFilter` entry's `names`/`excludedNames` glob patterns for the item's kind (checking the cluster-scoped map for cluster resources and namespace map for namespace resources). If the name doesn't match, the item is excluded.
|
||||
- **Important:** If the item's kind is not listed in the namespace filter map **and** there is no catch-all entry, the item passes through Stage 2 without a name check. This is intentional — see [Plugin AdditionalItems and Auto-Backed Up CRDs](#edge-cases-and-behavior-documentation) below.
|
||||
|
||||
### Impact on Restore
|
||||
|
||||
The restore process (`pkg/restore/restore.go`) is **not modified** in this design. The reason:
|
||||
|
||||
- Restore reads the backup archive as-is. Items excluded by namespace-scoped filter policies during backup are simply absent from the archive. The restore process iterates what's in the tarball and applies `RestoreSpec` filters on top. No items excluded during backup will appear during restore.
|
||||
- Restore plugins that request "additional items" (via `RestoreItemAction`) may reference items excluded from the backup. These items won't be in the archive, so the restore will skip them silently. This is the same behavior that occurs today with any incomplete backup — no new risk is introduced.
|
||||
- Users can still use `RestoreSpec.IncludedNamespaces` to selectively restore from a namespace-filtered backup.
|
||||
|
||||
A follow-up design will add namespace-scoped filters to the restore pipeline.
|
||||
|
||||
### Edge Cases and Behavior Documentation
|
||||
|
||||
This section documents the system behavior in edge cases and error conditions:
|
||||
|
||||
**Plugin AdditionalItems and Auto-Backed Up CRDs:**
|
||||
Cluster-scoped resources injected dynamically (such as `VolumeSnapshotClass` from the CSI plugin or `CustomResourceDefinition` from Velero's auto-backup loop) do not require hardcoded exceptions. In `itemInclusionChecks()`, Velero natively allows unlisted cluster-scoped resources to pass through unless explicitly excluded by the user. `ClusterScopedFilterPolicy` preserves this permissive behavior: if a dynamically injected cluster-scoped resource is NOT listed in the policy, it passes through untouched. If it IS listed, its specific `names` and `excludedNames` filters are strictly enforced.
|
||||
|
||||
**Plugin-injected namespace-scoped additional items** follow the same permissive principle. When a `BackupItemAction` returns additional items whose kind is not listed in the matched `namespacedFilterPolicies` entry and there is no catch-all entry, those items still pass through Stage 2 (`itemInclusionChecks`). This is intentional: blocking plugin-injected items at Stage 2 would break backup completeness — for example, a CSI plugin may inject a `VolumeSnapshotContent` that is required for a correct restore even when the user's filter policy only lists application resource types.
|
||||
|
||||
The kind-level exclusion that makes `namespacedFilterPolicies` an exclusive allowlist applies only during **Stage 1** (the primary collection pass in `item_collector.go`). At Stage 2, `itemInclusionChecks` enforces only:
|
||||
- The `velero.io/exclude-from-backup=true` label (always takes precedence).
|
||||
- The `names`/`excludedNames` patterns for **listed** kinds.
|
||||
|
||||
Plugin-injected items of unlisted kinds are therefore included as long as they are not explicitly excluded by label. Users who need to suppress a specific plugin-injected kind should apply the `velero.io/exclude-from-backup=true` label to those resources.
|
||||
|
||||
**Multiple Glob Patterns Matching Same Namespace (Incorrect Order):**
|
||||
```yaml
|
||||
namespacedFilterPolicies:
|
||||
- namespaces: ["team-*"] # Broader pattern listed first
|
||||
resourceFilters:
|
||||
- kinds: [Deployment, Service]
|
||||
- namespaces: ["team-frontend-*"] # More specific pattern listed second
|
||||
resourceFilters:
|
||||
- kinds: [ConfigMap, Secret, Deployment, Service]
|
||||
```
|
||||
**Behavior:** For namespace `team-frontend-prod`, the broader `team-*` pattern matches first, so only `Deployment` and `Service` are backed up. The more specific `team-frontend-*` rule is never reached.
|
||||
|
||||
**Multiple Glob Patterns Matching Same Namespace (Correct Order):**
|
||||
```yaml
|
||||
namespacedFilterPolicies:
|
||||
- namespaces: ["team-frontend-*"] # More specific pattern listed first
|
||||
resourceFilters:
|
||||
- kinds: [ConfigMap, Secret, Deployment, Service]
|
||||
- namespaces: ["team-*"] # Broader pattern listed second
|
||||
resourceFilters:
|
||||
- kinds: [Deployment, Service]
|
||||
```
|
||||
**Behavior:** For namespace `team-frontend-prod`, the specific `team-frontend-*` pattern matches first, backing up all specified resources. For `team-backend-dev`, the broader `team-*` pattern matches, backing up only `Deployment` and `Service`. This achieves the intended behavior.
|
||||
|
||||
**Namespace Included Globally But No Matching Filter Policy:**
|
||||
```yaml
|
||||
# BackupSpec includes "production" namespace
|
||||
# ResourcePolicy has no namespacedFilterPolicies entry for "production"
|
||||
```
|
||||
**Behavior:** The namespace uses global filters exactly as it does today. This is the backward compatibility behavior — only namespaces with explicit filter policies get namespace-scoped filtering.
|
||||
|
||||
|
||||
**Empty ResourceFilters Array:**
|
||||
```yaml
|
||||
namespacedFilterPolicies:
|
||||
- namespaces: ["test-namespace"]
|
||||
resourceFilters: [] # empty array
|
||||
```
|
||||
**Behavior:** Validation error during backup creation:
|
||||
```
|
||||
namespacedFilterPolicies[0]: at least one resourceFilter must be specified
|
||||
```
|
||||
|
||||
**Namespace Pattern with No Matches:**
|
||||
```yaml
|
||||
namespacedFilterPolicies:
|
||||
- namespaces: ["nonexistent-*"]
|
||||
resourceFilters: [...]
|
||||
```
|
||||
**Behavior:** No error. The filter policy is loaded but never applied since no namespaces match the pattern. This allows for conditional filtering based on namespace existence.
|
||||
|
||||
**Resource Kind Not Present in Target Namespaces:**
|
||||
```yaml
|
||||
resourceFilters:
|
||||
- kinds: ["StatefulSet"] # namespace has no StatefulSets
|
||||
names: ["workload-1"]
|
||||
```
|
||||
**Behavior:** No error. The filter is applied but finds no matching resources. Empty result set is valid.
|
||||
|
||||
**Conflicting Name Patterns:**
|
||||
```yaml
|
||||
resourceFilters:
|
||||
- kinds: ["ConfigMap"]
|
||||
names: ["app-*"]
|
||||
excludedNames: ["app-config"] # conflicts with names pattern
|
||||
```
|
||||
**Behavior:** The `excludedNames` takes precedence. Resources matching `app-*` are included, then `app-config` is excluded. Net result: includes `app-secret`, `app-data`, etc., but excludes `app-config`.
|
||||
|
||||
**Invalid Label Selector Syntax:**
|
||||
```yaml
|
||||
resourceFilters:
|
||||
- kinds: ["Pod"]
|
||||
labelSelector:
|
||||
"invalid label key!": "value" # invalid key syntax
|
||||
```
|
||||
**Behavior:** Validation error during backup creation when `labels.SelectorFromSet()` fails:
|
||||
```
|
||||
namespacedFilterPolicies[0].resourceFilters[0]: invalid label selector: "invalid label key!" is not a valid label key
|
||||
```
|
||||
|
||||
**Out-of-Scope Kinds in Filter Entries:**
|
||||
A user may accidentally list a cluster-scoped kind (e.g., `ClusterRole`) inside a `namespacedFilterPolicies` entry, or a namespace-scoped kind (e.g., `ConfigMap`) inside `clusterScopedFilterPolicy`. The system silently ignores such entries at the Kubernetes API level — namespace-scoped items are never listed globally, and cluster-scoped items are never listed per-namespace, so no matching resources will ever be found. A warning is logged at backup start to help the user detect the misconfiguration. No validation error is raised — the entry is harmless but ineffective.
|
||||
|
||||
**Discovery Helper Unavailable:**
|
||||
If the discovery helper is completely unavailable during backup initialization, the backup fails with:
|
||||
```
|
||||
failed to resolve namespace filter policies: discovery client unavailable
|
||||
```
|
||||
This is consistent with how other discovery-dependent features handle this error condition.
|
||||
|
||||
# Detailed Design
|
||||
|
||||
## ResourceFilter Field Notes
|
||||
|
||||
**`labelSelector`** supports equality-based selectors only (`key=value`). Set-based requirements (e.g., `environment in (prod, staging)`) are not supported. To match resources with any of several label combinations, use `orLabelSelectors` with multiple maps — each map is AND-evaluated internally, and the maps are OR-evaluated across the list. `labelSelector` and `orLabelSelectors` cannot co-exist in the same entry.
|
||||
|
||||
**`names` / `excludedNames`** accept exact resource names or glob patterns. If `names` is empty, all resource names are included (subject to label filters). `excludedNames` takes precedence over `names` when a name matches both.
|
||||
|
||||
## Validation
|
||||
|
||||
**Validation functions for `namespacedFilterPolicies`:**
|
||||
|
||||
1. **Each filter policy must specify at least one namespace:**
|
||||
```
|
||||
namespacedFilterPolicies[N]: at least one namespace must be specified
|
||||
```
|
||||
|
||||
2. **Each filter policy must specify at least one resource filter:**
|
||||
```
|
||||
namespacedFilterPolicies[N]: at least one resourceFilter must be specified
|
||||
```
|
||||
|
||||
3. **Each resource filter without kinds can only be defined once, and cannot specify names/excludedNames.**
|
||||
```
|
||||
namespacedFilterPolicies[N]: only one resource filter with empty kinds is allowed
|
||||
namespacedFilterPolicies[N].resourceFilters[M]: names or excludedNames cannot be specified when kinds is empty
|
||||
```
|
||||
|
||||
4. **No duplicate kinds across resource filter entries** within the same namespace filter:
|
||||
```
|
||||
namespacedFilterPolicies[N]: kind "Pod" appears in both resourceFilters[0] and resourceFilters[2]
|
||||
```
|
||||
|
||||
5. **`labelSelector` and `orLabelSelectors` mutual exclusion** within each resource filter:
|
||||
```
|
||||
namespacedFilterPolicies[N].resourceFilters[M]: labelSelector and orLabelSelectors cannot co-exist
|
||||
```
|
||||
|
||||
6. **No duplicate namespace patterns across filter policies.** This validates only exact duplicates - runtime behavior handles overlapping patterns.
|
||||
|
||||
**Rationale:** Detecting all possible pattern overlaps (like `team-*` vs `team-frontend-*`) is computationally complex and may reject valid configurations. Instead, the runtime uses first-match semantics - the first matching filter policy in the list is applied. This allows users flexibility while preventing obvious configuration errors.
|
||||
|
||||
7. **Namespace patterns must be valid globs.**
|
||||
|
||||
8. **Resource name patterns must be valid globs.**
|
||||
|
||||
9. **Resource kind validation with discovery helper** (performed during backup initialization).
|
||||
|
||||
**Validation functions for `clusterScopedFilterPolicy`:**
|
||||
|
||||
1. **At least one resourceFilter must be specified.**
|
||||
```
|
||||
clusterScopedFilterPolicy: at least one resourceFilter must be specified
|
||||
```
|
||||
|
||||
2. **No duplicate kinds across resource filters.**
|
||||
|
||||
3. **`labelSelector` and `orLabelSelectors` mutual exclusion.**
|
||||
|
||||
4. **Resource name patterns must be valid globs.**
|
||||
|
||||
Additionally, in `backup_controller.go`, a validation check ensures that `namespacedFilterPolicies` and `clusterScopedFilterPolicy` are not used with old-style resource filters (`IncludedResources`/`ExcludedResources`/`IncludeClusterResources`), similar to the existing check for `includeExcludePolicy`.
|
||||
|
||||
## ConfigMap Examples
|
||||
|
||||
### Per-Namespace Resource Type Filtering
|
||||
|
||||
Back up only ConfigMaps, Secrets, and Deployments (with label `app=my-app`) from `ns-a`, but everything from `ns-b`:
|
||||
|
||||
```yaml
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: backup-filter-policy
|
||||
namespace: velero
|
||||
data:
|
||||
policy: |
|
||||
version: v1
|
||||
namespacedFilterPolicies:
|
||||
- namespaces:
|
||||
- ns-a
|
||||
resourceFilters:
|
||||
- kinds: [ConfigMap, Secret, Deployment]
|
||||
labelSelector:
|
||||
app: my-app
|
||||
# ns-b has no filter policy entry, so global filters apply (include everything)
|
||||
```
|
||||
|
||||
Backup CR referencing it:
|
||||
|
||||
```yaml
|
||||
apiVersion: velero.io/v1
|
||||
kind: Backup
|
||||
metadata:
|
||||
name: selective-backup
|
||||
namespace: velero
|
||||
spec:
|
||||
includedNamespaces:
|
||||
- ns-a
|
||||
- ns-b
|
||||
resourcePolicy:
|
||||
kind: configmap
|
||||
name: backup-filter-policy
|
||||
storageLocation: default
|
||||
ttl: 720h0m0s
|
||||
```
|
||||
|
||||
### Per-Kind Label Selectors (Different Labels per Kind)
|
||||
|
||||
Back up Deployments with one label and StatefulSets with a different label from the same namespace:
|
||||
|
||||
```yaml
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: vm-filter-policy
|
||||
namespace: velero
|
||||
data:
|
||||
policy: |
|
||||
version: v1
|
||||
namespacedFilterPolicies:
|
||||
- namespaces:
|
||||
- target-namespace
|
||||
resourceFilters:
|
||||
- kinds: [Deployment]
|
||||
labelSelector:
|
||||
app: production-workload-1
|
||||
- kinds: [StatefulSet]
|
||||
labelSelector:
|
||||
app: production-workload-2
|
||||
```
|
||||
|
||||
### Per-Kind Exact Names
|
||||
|
||||
Back up specific Deployments, Configmaps, and Secrets by name:
|
||||
|
||||
```yaml
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: named-resource-filter
|
||||
namespace: velero
|
||||
data:
|
||||
policy: |
|
||||
version: v1
|
||||
namespacedFilterPolicies:
|
||||
- namespaces:
|
||||
- target-namespace
|
||||
resourceFilters:
|
||||
- kinds: [Deployment]
|
||||
names: [workload-1, workload-2]
|
||||
- kinds: [ConfigMap]
|
||||
names: [p1, p2]
|
||||
- kinds: [Secret]
|
||||
names: [c1, c2]
|
||||
```
|
||||
|
||||
### Name Pattern Filtering with Exclusion
|
||||
|
||||
Back up only `app-*` ConfigMaps and Secrets from `production`, excluding temporary and debug resources:
|
||||
|
||||
```yaml
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: app-config-filter-policy
|
||||
namespace: velero
|
||||
data:
|
||||
policy: |
|
||||
version: v1
|
||||
namespacedFilterPolicies:
|
||||
- namespaces:
|
||||
- production
|
||||
resourceFilters:
|
||||
- kinds: [ConfigMap, Secret]
|
||||
names: ["app-*"]
|
||||
excludedNames: ["*-tmp", "*-debug"]
|
||||
```
|
||||
|
||||
### Catch-All with No Label Selector (Override-Only)
|
||||
|
||||
A user may want to use the global configuration for 99% of resources in a namespace, but only apply a specific name filter to a single kind. To achieve this without explicitly listing all other kinds or adding dummy labels, a catch-all filter without a label selector can be used:
|
||||
|
||||
```yaml
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: override-only-filter-policy
|
||||
namespace: velero
|
||||
data:
|
||||
policy: |
|
||||
version: v1
|
||||
namespacedFilterPolicies:
|
||||
- namespaces:
|
||||
- ns-a
|
||||
resourceFilters:
|
||||
- kinds: [Secret]
|
||||
names: [my-secret] # Specific override for Secrets
|
||||
- kinds: ["*"] # Catch-all: NO label selector
|
||||
# Includes all other kinds unconditionally
|
||||
```
|
||||
|
||||
**Result:**
|
||||
- `Secret` resources: only `my-secret` is backed up.
|
||||
- All other resource types: backed up unconditionally (acting like a global fallback).
|
||||
|
||||
### Catch-All Label Selector (Back Up Everything with a Specific Label)
|
||||
|
||||
When a user wants to back up any resource type in a namespace that carries a particular label — without enumerating every kind — the catch-all entry (empty `kinds` or `["*"]`) achieves this with a single rule:
|
||||
|
||||
```yaml
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: label-based-filter-policy
|
||||
namespace: velero
|
||||
data:
|
||||
policy: |
|
||||
version: v1
|
||||
namespacedFilterPolicies:
|
||||
- namespaces:
|
||||
- production
|
||||
resourceFilters:
|
||||
- kinds: ["*"] # catch-all: applies to every kind not listed below
|
||||
labelSelector:
|
||||
backup: "true" # back up any resource carrying this label
|
||||
```
|
||||
|
||||
**Result:** Every resource type in `production` that has the label `backup=true` is backed up. Resources without that label are excluded. No kind enumeration is required.
|
||||
|
||||
### Catch-All with Per-Kind Name Overrides
|
||||
|
||||
A more advanced pattern: use exact names for specific kinds and fall back to a label selector for all remaining kinds. Kind-specific entries take precedence over the catch-all:
|
||||
|
||||
```yaml
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: mixed-filter-policy
|
||||
namespace: velero
|
||||
data:
|
||||
policy: |
|
||||
version: v1
|
||||
namespacedFilterPolicies:
|
||||
- namespaces:
|
||||
- production
|
||||
resourceFilters:
|
||||
- kinds: [Deployment]
|
||||
names: [api-server, worker] # these exact Deployments by name
|
||||
- kinds: [Secret]
|
||||
names: [db-credentials, tls-cert] # these exact Secrets by name
|
||||
- kinds: ["*"] # catch-all for all other kinds
|
||||
labelSelector:
|
||||
backup: "true" # back up by label
|
||||
```
|
||||
|
||||
**Result:**
|
||||
- `Deployment` resources: only `api-server` and `worker` are backed up (name filter; the catch-all does not apply).
|
||||
- `Secret` resources: only `db-credentials` and `tls-cert` are backed up (name filter; the catch-all does not apply).
|
||||
- All other resource types (ConfigMap, StatefulSet, Service, etc.): backed up only if they carry `backup=true`.
|
||||
|
||||
This pattern is useful when certain high-value resources need precise name-based selection, while the rest of the namespace is covered by a label convention.
|
||||
|
||||
### Glob Namespace Patterns and Ordering
|
||||
|
||||
Apply filters to namespaces matching patterns. **Critical: Order patterns from most specific to least specific:**
|
||||
|
||||
```yaml
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: team-filter-policy
|
||||
namespace: velero
|
||||
data:
|
||||
policy: |
|
||||
version: v1
|
||||
namespacedFilterPolicies:
|
||||
# More specific patterns first
|
||||
- namespaces:
|
||||
- "team-frontend-prod" # Most specific (exact match)
|
||||
resourceFilters:
|
||||
- kinds: [Deployment, Service, ConfigMap, Secret, PVC]
|
||||
- namespaces:
|
||||
- "team-frontend-*" # Less specific (pattern match)
|
||||
resourceFilters:
|
||||
- kinds: [Deployment, Service, ConfigMap]
|
||||
- namespaces:
|
||||
- "team-*" # Least specific (broad pattern)
|
||||
resourceFilters:
|
||||
- kinds: [Deployment, Service]
|
||||
```
|
||||
|
||||
**Pattern Matching Results:**
|
||||
- `team-frontend-prod` → Uses exact match policy (backs up 5 resource types)
|
||||
- `team-frontend-dev` → Uses `team-frontend-*` policy (backs up 3 resource types)
|
||||
- `team-backend-test` → Uses `team-*` policy (backs up 2 resource types)
|
||||
- `app-namespace` → No match, uses global filters
|
||||
|
||||
### Combined with Volume Policies
|
||||
|
||||
Both volume policies and namespace-scoped filters in the same ConfigMap:
|
||||
|
||||
```yaml
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: combined-policy
|
||||
namespace: velero
|
||||
data:
|
||||
policy: |
|
||||
version: v1
|
||||
volumePolicies:
|
||||
- conditions:
|
||||
capacity: "0,10Gi"
|
||||
storageClass:
|
||||
- standard
|
||||
action:
|
||||
type: fs-backup
|
||||
- conditions:
|
||||
capacity: "10Gi,100Gi"
|
||||
action:
|
||||
type: snapshot
|
||||
namespacedFilterPolicies:
|
||||
- namespaces:
|
||||
- production
|
||||
resourceFilters:
|
||||
- kinds: [Deployment]
|
||||
names: [workload-1, workload-2]
|
||||
- kinds: [StatefulSet]
|
||||
labelSelector:
|
||||
app: my-app
|
||||
- kinds: [ConfigMap, Secret]
|
||||
names: ["app-*"]
|
||||
excludedNames: ["*-tmp", "*-debug"]
|
||||
```
|
||||
|
||||
### Backup CR — No ResourcePolicy (backward compatible)
|
||||
|
||||
Existing backups continue to work exactly as before:
|
||||
|
||||
```yaml
|
||||
apiVersion: velero.io/v1
|
||||
kind: Backup
|
||||
metadata:
|
||||
name: full-backup
|
||||
namespace: velero
|
||||
spec:
|
||||
includedNamespaces:
|
||||
- "*"
|
||||
includedResources:
|
||||
- "*"
|
||||
labelSelector:
|
||||
matchLabels:
|
||||
backup: "true"
|
||||
storageLocation: default
|
||||
```
|
||||
|
||||
## CLI
|
||||
|
||||
### `velero backup describe`
|
||||
|
||||
The output is extended to display namespace-scoped filter policies when present in the ResourcePolicy ConfigMap:
|
||||
|
||||
```
|
||||
Name: selective-backup
|
||||
Namespace: velero
|
||||
Labels: <none>
|
||||
Annotations: <none>
|
||||
|
||||
Phase: Completed
|
||||
|
||||
Errors: 0
|
||||
Warnings: 0
|
||||
|
||||
Namespaces:
|
||||
Included: ns-a, ns-b
|
||||
Excluded: <none>
|
||||
|
||||
Resources:
|
||||
Included: *
|
||||
Excluded: <none>
|
||||
Cluster-scoped: auto
|
||||
|
||||
Label selector: <none>
|
||||
|
||||
Resource Policy: backup-filter-policy
|
||||
|
||||
Namespace-Scoped Filter Policies:
|
||||
ns-a:
|
||||
Resource Filters:
|
||||
ConfigMap, Secret, Deployment:
|
||||
Label selector: app=my-app
|
||||
Included names: <none>
|
||||
Excluded names: <none>
|
||||
target-namespace:
|
||||
Resource Filters:
|
||||
Deployment:
|
||||
Label selector: app=production-workload-1
|
||||
Included names: <none>
|
||||
Excluded names: <none>
|
||||
StatefulSet:
|
||||
Label selector: app=production-workload-2
|
||||
Included names: <none>
|
||||
Excluded names: <none>
|
||||
production:
|
||||
Resource Filters:
|
||||
Deployment:
|
||||
Label selector: <none>
|
||||
Included names: [api-server, worker]
|
||||
Excluded names: <none>
|
||||
<catch-all> (all other kinds):
|
||||
Label selector: backup=true
|
||||
Included names: <none>
|
||||
Excluded names: <none>
|
||||
|
||||
Fine-Grained Global Filter Policy:
|
||||
Resource Filters:
|
||||
ClusterRole, ClusterRoleBinding:
|
||||
Label selector: <none>
|
||||
Included names: [my-app-*]
|
||||
Excluded names: <none>
|
||||
CustomResourceDefinition:
|
||||
Label selector: app=my-app
|
||||
Included names: <none>
|
||||
Excluded names: <none>
|
||||
|
||||
Storage Location: default
|
||||
|
||||
...
|
||||
```
|
||||
|
||||
### `velero backup create`
|
||||
|
||||
No new CLI flags are added. The namespace-scoped filter policies are specified in the ResourcePolicy ConfigMap, which is already referenced via the existing `--resource-policies-configmap` flag:
|
||||
|
||||
```bash
|
||||
velero backup create selective-backup \
|
||||
--include-namespaces ns-a,ns-b \
|
||||
--resource-policies-configmap backup-filter-policy
|
||||
```
|
||||
|
||||
The `--help` output for `velero backup create` is updated to clarify the interaction between global and namespace-scoped filters:
|
||||
|
||||
```
|
||||
Backup Filtering Options:
|
||||
--include-namespaces stringArray namespaces to include in the backup (use '*' for all namespaces)
|
||||
--exclude-namespaces stringArray namespaces to exclude from the backup
|
||||
--include-resources stringArray resources to include in the backup, formatted as resource.group
|
||||
--exclude-resources stringArray resources to exclude from the backup, formatted as resource.group
|
||||
--include-cluster-resources optionalBool[=true] include cluster-scoped resources
|
||||
--exclude-cluster-resources exclude cluster-scoped resources
|
||||
--selector labelSelector only back up resources matching this label selector
|
||||
--or-selector labelSelector back up resources matching any of the label selectors (can be repeated)
|
||||
--resource-policies-configmap string reference to a configmap containing resource policies for volume snapshots and namespace-scoped filtering
|
||||
|
||||
Notes:
|
||||
- Global filters (--include-resources, --selector, etc.) apply to all included namespaces
|
||||
- Namespace-scoped filters defined in --resource-policies-configmap override global filters for matching namespaces
|
||||
- Fine-grained global filter policies defined in --resource-policies-configmap override global filters for cluster-scoped resources
|
||||
- Use 'velero backup describe' to view resolved filter policies after backup creation
|
||||
```
|
||||
|
||||
### CLI Integration Points
|
||||
|
||||
**Backup Creation Workflow:**
|
||||
1. User creates ResourcePolicy ConfigMap with `namespacedFilterPolicies`
|
||||
2. User references ConfigMap via `--resource-policies-configmap` flag
|
||||
3. Backup controller validates policies during backup initialization
|
||||
4. Validation errors are reported immediately with specific line/field references
|
||||
|
||||
**Help and Discovery:**
|
||||
- `velero backup create --help` includes updated filtering documentation
|
||||
- `velero backup describe` shows resolved filter policies for troubleshooting
|
||||
- Validation errors include ConfigMap field references for easy debugging
|
||||
|
||||
**Configuration Discovery:**
|
||||
- `velero backup create --help` includes namespace-scoped filtering documentation
|
||||
- `velero backup describe` shows resolved filter policies for verification
|
||||
|
||||
## User Perspective
|
||||
|
||||
This design provides fine-grained, per-namespace, per-kind control over backup filtering. Key user-facing aspects:
|
||||
|
||||
- **For users not using namespace-scoped filter policies**: Zero changes. All existing backups and workflows continue to work identically. The new YAML key is optional.
|
||||
- **For users adopting namespace-scoped filter policies**: Create a ConfigMap with the `namespacedFilterPolicies` section and reference it via `BackupSpec.ResourcePolicy` (or the existing `--resource-policies-configmap` flag). The backup will selectively include/exclude resources per namespace based on the filter rules.
|
||||
- **For users already using ResourcePolicy for volume policies**: Add the `namespacedFilterPolicies` section to the same ConfigMap. Both volume policies and namespace-scoped filters coexist.
|
||||
- **For restore from a namespace-filtered backup**: No changes to restore workflow. Restore processes whatever is in the archive. Users can use existing `RestoreSpec.IncludedNamespaces` for additional filtering at restore time.
|
||||
- **`velero backup describe` output**: Extended to show per-namespace, per-kind filter details when the ResourcePolicy ConfigMap contains `namespacedFilterPolicies`.
|
||||
- **Validation errors**: Reported at backup start when the ResourcePolicy ConfigMap contains invalid `namespacedFilterPolicies` configurations. Consistent with how volume policy validation errors are reported today.
|
||||
|
||||
## Alternatives Considered
|
||||
|
||||
1. **CRD-Based `NamespacedFilters` Field**: Add `NamespacedFilters []NamespaceFilter` directly to `BackupSpec`. Rejected for this iteration due to heavy CRD change overhead. The ResourcePolicy approach achieves the same functionality with less API surface change.
|
||||
|
||||
2. **Flat Fields on NamespacedFilterPolicy (No Per-Kind Selectors)**: Use flat fields (`includedResources`, `labelSelector`, `includedResourceNames`) shared across all kinds within a namespace. Rejected because it cannot express per-kind label selectors or per-kind name lists — a critical requirement for workloads where different resource types have different labels or naming conventions.
|
||||
|
||||
3. **Scoped Label Selectors Only**: Augment existing label selectors with an optional namespace scope. Rejected because it only addresses label-scoped filtering and does not support per-namespace resource type filtering or name filtering.
|
||||
|
||||
4. **Global Name Filter Only**: Add only global name filter fields. Rejected because it only addresses name filtering globally and does not address namespace-scoped or kind-scoped filtering.
|
||||
|
||||
5. **Separate ConfigMap for Namespace Filters**: Use a new `BackupSpec` field pointing to a different ConfigMap (separate from volume policies). Rejected because it adds a new CRD field (which has similar reasons with #1) and splits configuration across multiple ConfigMaps.
|
||||
@@ -9,6 +9,7 @@ require (
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.6.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4
|
||||
github.com/RoaringBitmap/roaring v1.9.4
|
||||
github.com/aws/aws-sdk-go-v2 v1.24.1
|
||||
github.com/aws/aws-sdk-go-v2/config v1.26.3
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.16.14
|
||||
@@ -26,6 +27,7 @@ require (
|
||||
github.com/hashicorp/go-plugin v1.7.0
|
||||
github.com/joho/godotenv v1.3.0
|
||||
github.com/kopia/kopia v0.16.0
|
||||
github.com/kubernetes-csi/external-snapshot-metadata v1.0.0
|
||||
github.com/kubernetes-csi/external-snapshotter/client/v8 v8.4.0
|
||||
github.com/onsi/ginkgo/v2 v2.28.3
|
||||
github.com/onsi/gomega v1.40.0
|
||||
@@ -92,6 +94,7 @@ require (
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.6 // indirect
|
||||
github.com/aws/smithy-go v1.19.0 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/bits-and-blooms/bitset v1.12.0 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/chmduquesne/rollinghash v4.0.0+incompatible // indirect
|
||||
github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect
|
||||
@@ -109,14 +112,25 @@ require (
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-ole/go-ole v1.3.0 // indirect
|
||||
github.com/go-openapi/jsonpointer v0.21.0 // indirect
|
||||
github.com/go-openapi/jsonreference v0.20.2 // indirect
|
||||
github.com/go-openapi/swag v0.23.0 // indirect
|
||||
github.com/go-openapi/jsonpointer v0.22.5 // indirect
|
||||
github.com/go-openapi/jsonreference v0.21.5 // indirect
|
||||
github.com/go-openapi/swag v0.25.5 // indirect
|
||||
github.com/go-openapi/swag/cmdutils v0.25.5 // indirect
|
||||
github.com/go-openapi/swag/conv v0.25.5 // indirect
|
||||
github.com/go-openapi/swag/fileutils v0.25.5 // indirect
|
||||
github.com/go-openapi/swag/jsonname v0.25.5 // indirect
|
||||
github.com/go-openapi/swag/jsonutils v0.25.5 // indirect
|
||||
github.com/go-openapi/swag/loading v0.25.5 // indirect
|
||||
github.com/go-openapi/swag/mangling v0.25.5 // indirect
|
||||
github.com/go-openapi/swag/netutils v0.25.5 // indirect
|
||||
github.com/go-openapi/swag/stringutils v0.25.5 // indirect
|
||||
github.com/go-openapi/swag/typeutils v0.25.5 // indirect
|
||||
github.com/go-openapi/swag/yamlutils v0.25.5 // indirect
|
||||
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
|
||||
github.com/gofrs/flock v0.13.0 // indirect
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1 // indirect
|
||||
github.com/golang/protobuf v1.5.4 // indirect
|
||||
github.com/google/gnostic-models v0.7.0 // indirect
|
||||
github.com/google/gnostic-models v0.7.1 // indirect
|
||||
github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 // indirect
|
||||
github.com/google/s2a-go v0.1.9 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.15 // indirect
|
||||
@@ -126,16 +140,15 @@ require (
|
||||
github.com/hashicorp/yamux v0.1.2 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/klauspost/compress v1.18.6 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
|
||||
github.com/klauspost/crc32 v1.3.0 // indirect
|
||||
github.com/klauspost/pgzip v1.2.6 // indirect
|
||||
github.com/klauspost/reedsolomon v1.14.0 // indirect
|
||||
github.com/kubernetes-csi/external-snapshot-metadata/client v1.0.0 // indirect
|
||||
github.com/kylelemons/godebug v1.1.0 // indirect
|
||||
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
|
||||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||
github.com/mattn/go-isatty v0.0.21 // indirect
|
||||
github.com/minio/crc64nvme v1.1.1 // indirect
|
||||
@@ -145,6 +158,7 @@ require (
|
||||
github.com/moby/term v0.5.0 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
|
||||
github.com/mschoch/smat v0.2.0 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/mxk/go-vss v1.2.1 // indirect
|
||||
github.com/natefinch/atomic v1.0.1 // indirect
|
||||
@@ -155,7 +169,7 @@ require (
|
||||
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||
github.com/prometheus/common v0.67.5 // indirect
|
||||
github.com/prometheus/procfs v0.19.2 // indirect
|
||||
github.com/prometheus/procfs v0.20.1 // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect
|
||||
github.com/stretchr/objx v0.5.2 // indirect
|
||||
@@ -175,7 +189,7 @@ require (
|
||||
go.opentelemetry.io/otel/trace v1.43.0 // indirect
|
||||
go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.3 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.4 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/crypto v0.50.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect
|
||||
|
||||
@@ -113,6 +113,8 @@ github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb0
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
|
||||
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
|
||||
github.com/RoaringBitmap/roaring v1.9.4 h1:yhEIoH4YezLYT04s1nHehNO64EKFTop/wBhxv2QzDdQ=
|
||||
github.com/RoaringBitmap/roaring v1.9.4/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
|
||||
@@ -168,8 +170,12 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
|
||||
github.com/bits-and-blooms/bitset v1.12.0 h1:U/q1fAF7xXRhFCrhROzIfffYnu+dlS38vCZtmFVPHmA=
|
||||
github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
|
||||
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
|
||||
github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
|
||||
github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
|
||||
github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
|
||||
github.com/bombsimon/logrusr/v3 v3.0.0 h1:tcAoLfuAhKP9npBxWzSdpsvKPQt1XV02nSf2lZA82TQ=
|
||||
github.com/bombsimon/logrusr/v3 v3.0.0/go.mod h1:PksPPgSFEL2I52pla2glgCyyd2OqOHAnFF5E+g8Ixco=
|
||||
github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/FBatYVw=
|
||||
@@ -189,6 +195,8 @@ github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnht
|
||||
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
|
||||
github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik=
|
||||
github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4=
|
||||
github.com/container-storage-interface/spec v1.12.0 h1:zrFOEqpR5AghNaaDG4qyedwPBqU2fU0dWjLQMP/azK0=
|
||||
github.com/container-storage-interface/spec v1.12.0/go.mod h1:txsm+MA2B2WDa5kW69jNbqPnvTtfvZma7T/zsAZ9qX8=
|
||||
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
|
||||
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
|
||||
github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
|
||||
@@ -285,18 +293,44 @@ github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
|
||||
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
|
||||
github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
|
||||
github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
|
||||
github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs=
|
||||
github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ=
|
||||
github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY=
|
||||
github.com/go-openapi/jsonpointer v0.22.5 h1:8on/0Yp4uTb9f4XvTrM2+1CPrV05QPZXu+rvu2o9jcA=
|
||||
github.com/go-openapi/jsonpointer v0.22.5/go.mod h1:gyUR3sCvGSWchA2sUBJGluYMbe1zazrYWIkWPjjMUY0=
|
||||
github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
|
||||
github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg=
|
||||
github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE=
|
||||
github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k=
|
||||
github.com/go-openapi/jsonreference v0.21.5 h1:6uCGVXU/aNF13AQNggxfysJ+5ZcU4nEAe+pJyVWRdiE=
|
||||
github.com/go-openapi/jsonreference v0.21.5/go.mod h1:u25Bw85sX4E2jzFodh1FOKMTZLcfifd1Q+iKKOUxExw=
|
||||
github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
|
||||
github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
|
||||
github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
|
||||
github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE=
|
||||
github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ=
|
||||
github.com/go-openapi/swag v0.25.5 h1:pNkwbUEeGwMtcgxDr+2GBPAk4kT+kJ+AaB+TMKAg+TU=
|
||||
github.com/go-openapi/swag v0.25.5/go.mod h1:B3RT6l8q7X803JRxa2e59tHOiZlX1t8viplOcs9CwTA=
|
||||
github.com/go-openapi/swag/cmdutils v0.25.5 h1:yh5hHrpgsw4NwM9KAEtaDTXILYzdXh/I8Whhx9hKj7c=
|
||||
github.com/go-openapi/swag/cmdutils v0.25.5/go.mod h1:pdae/AFo6WxLl5L0rq87eRzVPm/XRHM3MoYgRMvG4A0=
|
||||
github.com/go-openapi/swag/conv v0.25.5 h1:wAXBYEXJjoKwE5+vc9YHhpQOFj2JYBMF2DUi+tGu97g=
|
||||
github.com/go-openapi/swag/conv v0.25.5/go.mod h1:CuJ1eWvh1c4ORKx7unQnFGyvBbNlRKbnRyAvDvzWA4k=
|
||||
github.com/go-openapi/swag/fileutils v0.25.5 h1:B6JTdOcs2c0dBIs9HnkyTW+5gC+8NIhVBUwERkFhMWk=
|
||||
github.com/go-openapi/swag/fileutils v0.25.5/go.mod h1:V3cT9UdMQIaH4WiTrUc9EPtVA4txS0TOmRURmhGF4kc=
|
||||
github.com/go-openapi/swag/jsonname v0.25.5 h1:8p150i44rv/Drip4vWI3kGi9+4W9TdI3US3uUYSFhSo=
|
||||
github.com/go-openapi/swag/jsonname v0.25.5/go.mod h1:jNqqikyiAK56uS7n8sLkdaNY/uq6+D2m2LANat09pKU=
|
||||
github.com/go-openapi/swag/jsonutils v0.25.5 h1:XUZF8awQr75MXeC+/iaw5usY/iM7nXPDwdG3Jbl9vYo=
|
||||
github.com/go-openapi/swag/jsonutils v0.25.5/go.mod h1:48FXUaz8YsDAA9s5AnaUvAmry1UcLcNVWUjY42XkrN4=
|
||||
github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.5 h1:SX6sE4FrGb4sEnnxbFL/25yZBb5Hcg1inLeErd86Y1U=
|
||||
github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.5/go.mod h1:/2KvOTrKWjVA5Xli3DZWdMCZDzz3uV/T7bXwrKWPquo=
|
||||
github.com/go-openapi/swag/loading v0.25.5 h1:odQ/umlIZ1ZVRteI6ckSrvP6e2w9UTF5qgNdemJHjuU=
|
||||
github.com/go-openapi/swag/loading v0.25.5/go.mod h1:I8A8RaaQ4DApxhPSWLNYWh9NvmX2YKMoB9nwvv6oW6g=
|
||||
github.com/go-openapi/swag/mangling v0.25.5 h1:hyrnvbQRS7vKePQPHHDso+k6CGn5ZBs5232UqWZmJZw=
|
||||
github.com/go-openapi/swag/mangling v0.25.5/go.mod h1:6hadXM/o312N/h98RwByLg088U61TPGiltQn71Iw0NY=
|
||||
github.com/go-openapi/swag/netutils v0.25.5 h1:LZq2Xc2QI8+7838elRAaPCeqJnHODfSyOa7ZGfxDKlU=
|
||||
github.com/go-openapi/swag/netutils v0.25.5/go.mod h1:lHbtmj4m57APG/8H7ZcMMSWzNqIQcu0RFiXrPUara14=
|
||||
github.com/go-openapi/swag/stringutils v0.25.5 h1:NVkoDOA8YBgtAR/zvCx5rhJKtZF3IzXcDdwOsYzrB6M=
|
||||
github.com/go-openapi/swag/stringutils v0.25.5/go.mod h1:PKK8EZdu4QJq8iezt17HM8RXnLAzY7gW0O1KKarrZII=
|
||||
github.com/go-openapi/swag/typeutils v0.25.5 h1:EFJ+PCga2HfHGdo8s8VJXEVbeXRCYwzzr9u4rJk7L7E=
|
||||
github.com/go-openapi/swag/typeutils v0.25.5/go.mod h1:itmFmScAYE1bSD8C4rS0W+0InZUBrB2xSPbWt6DLGuc=
|
||||
github.com/go-openapi/swag/yamlutils v0.25.5 h1:kASCIS+oIeoc55j28T4o8KwlV2S4ZLPT6G0iq2SSbVQ=
|
||||
github.com/go-openapi/swag/yamlutils v0.25.5/go.mod h1:Gek1/SjjfbYvM+Iq4QGwa/2lEXde9n2j4a3wI3pNuOQ=
|
||||
github.com/go-openapi/testify/enable/yaml/v2 v2.4.0 h1:7SgOMTvJkM8yWrQlU8Jm18VeDPuAvB/xWrdxFJkoFag=
|
||||
github.com/go-openapi/testify/enable/yaml/v2 v2.4.0/go.mod h1:14iV8jyyQlinc9StD7w1xVPW3CO3q1Gj04Jy//Kw4VM=
|
||||
github.com/go-openapi/testify/v2 v2.4.0 h1:8nsPrHVCWkQ4p8h1EsRVymA2XABB4OT40gcvAu+voFM=
|
||||
github.com/go-openapi/testify/v2 v2.4.0/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
|
||||
github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
|
||||
@@ -328,6 +362,8 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt
|
||||
github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
|
||||
github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
|
||||
github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
|
||||
github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
|
||||
github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
@@ -350,8 +386,8 @@ github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6
|
||||
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
|
||||
github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo=
|
||||
github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ=
|
||||
github.com/google/gnostic-models v0.7.1 h1:SisTfuFKJSKM5CPZkffwi6coztzzeYUhc3v4yxLWH8c=
|
||||
github.com/google/gnostic-models v0.7.1/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
@@ -466,7 +502,6 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfC
|
||||
github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
|
||||
github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
|
||||
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
|
||||
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
|
||||
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
|
||||
github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE=
|
||||
github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung=
|
||||
@@ -501,13 +536,20 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/kubernetes-csi/csi-lib-utils v0.23.2 h1:+x9W4RRyuRnJcTiSHKEFpl0cYUeLUqshM5ioPeYPRXw=
|
||||
github.com/kubernetes-csi/csi-lib-utils v0.23.2/go.mod h1:aIcqnC6EyesZpe7kX5PxHUZePw1tKrYFKwg7RaqlPh8=
|
||||
github.com/kubernetes-csi/csi-test/v5 v5.4.0 h1:u5DgYNIreSNO2+u4Nq2Wpl+bbakRSjNyxZHmDTAqnYA=
|
||||
github.com/kubernetes-csi/csi-test/v5 v5.4.0/go.mod h1:anAJKFUb/SdHhIHECgSKxC5LSiLzib+1I6mrWF5Hve8=
|
||||
github.com/kubernetes-csi/external-snapshot-metadata v1.0.0 h1:0UaIl/+A+wgwU8/vsh8XmgOvr4eBQMpq42CLQud3rRg=
|
||||
github.com/kubernetes-csi/external-snapshot-metadata v1.0.0/go.mod h1:kMm9tr8gdf0Dtr1JviPgi4rMy7A4cWFV7ML+0xF0bE0=
|
||||
github.com/kubernetes-csi/external-snapshot-metadata/client v1.0.0 h1:yRSDNeTNeZq7lSuDKXMv29ejtIRDgmJHCq36EjLQxLU=
|
||||
github.com/kubernetes-csi/external-snapshot-metadata/client v1.0.0/go.mod h1:Nufr1PWalLkuIyjS4Zr+ugyYEbK7MzbEUtZQUPeAJF0=
|
||||
github.com/kubernetes-csi/external-snapshotter/client/v8 v8.4.0 h1:bMqrb3UHgHbP+PW9VwiejfDJU1R0PpXVZNMdeH8WYKI=
|
||||
github.com/kubernetes-csi/external-snapshotter/client/v8 v8.4.0/go.mod h1:E3vdYxHj2C2q6qo8/Da4g7P+IcwqRZyy3gJBzYybV9Y=
|
||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||
@@ -521,8 +563,6 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN
|
||||
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
|
||||
github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
|
||||
github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
|
||||
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
|
||||
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
|
||||
github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo=
|
||||
github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg=
|
||||
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
|
||||
@@ -568,6 +608,8 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY
|
||||
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8=
|
||||
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
|
||||
github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM=
|
||||
github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw=
|
||||
github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
@@ -635,8 +677,8 @@ github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTU
|
||||
github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw=
|
||||
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
|
||||
github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws=
|
||||
github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
|
||||
github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc=
|
||||
github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo=
|
||||
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
|
||||
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
|
||||
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
|
||||
@@ -690,8 +732,6 @@ github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
|
||||
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
@@ -700,10 +740,7 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
|
||||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
|
||||
@@ -792,8 +829,8 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
|
||||
go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
|
||||
go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo=
|
||||
go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q=
|
||||
go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
|
||||
go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
|
||||
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
|
||||
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
@@ -1247,6 +1284,8 @@ k8s.io/cli-runtime v0.36.0/go.mod h1:KObkknK9Ro5LYX+1RdiKc7C8CvGg4aX+V/Zv+E8WPHA
|
||||
k8s.io/client-go v0.22.2/go.mod h1:sAlhrkVDf50ZHx6z4K0S40wISNTarf1r800F+RlCF6U=
|
||||
k8s.io/client-go v0.36.0 h1:pOYi7C4RHChYjMiHpZSpSbIM6ZxVbRXBy7CuiIwqA3c=
|
||||
k8s.io/client-go v0.36.0/go.mod h1:ZKKcpwF0aLYfkHFCjillCKaTK/yBkEDHTDXCFY6AS9Y=
|
||||
k8s.io/component-base v0.36.0 h1:hFjEktssxiJhrK1zfybkH4kJOi8iZuF+mIDCqS5+jRo=
|
||||
k8s.io/component-base v0.36.0/go.mod h1:JZvIfcNHk+uck+8LhJzhSBtydWXaZNQwX2OdL+Mnwsk=
|
||||
k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
|
||||
k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
|
||||
k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
|
||||
|
||||
@@ -96,7 +96,9 @@ RUN ARCH=$(go env GOARCH) && \
|
||||
chmod +x /usr/bin/goreleaser
|
||||
|
||||
# get golangci-lint
|
||||
RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.5.0
|
||||
# Use "go install" so the download goes through GOPROXY instead of the GitHub
|
||||
# release API/CDN, which has been returning intermittent/persistent HTTP 504s.
|
||||
RUN go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.5.0
|
||||
|
||||
# install kubectl
|
||||
RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/$(go env GOARCH)/kubectl
|
||||
|
||||
@@ -23,12 +23,14 @@ import (
|
||||
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
|
||||
"github.com/gobwas/glob"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/sirupsen/logrus"
|
||||
corev1api "k8s.io/api/core/v1"
|
||||
crclient "sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
|
||||
"github.com/vmware-tanzu/velero/pkg/util/wildcard"
|
||||
)
|
||||
|
||||
type VolumeActionType string
|
||||
@@ -54,6 +56,31 @@ type Action struct {
|
||||
Parameters map[string]any `yaml:"parameters,omitempty"`
|
||||
}
|
||||
|
||||
// ResourceFilter defines a filter for specific resource kinds.
|
||||
type ResourceFilter struct {
|
||||
Kinds []string `yaml:"kinds"`
|
||||
LabelSelector map[string]string `yaml:"labelSelector,omitempty"`
|
||||
OrLabelSelectors []map[string]string `yaml:"orLabelSelectors,omitempty"`
|
||||
Names []string `yaml:"names,omitempty"`
|
||||
ExcludedNames []string `yaml:"excludedNames,omitempty"`
|
||||
}
|
||||
|
||||
// IsCatchAll returns true if the filter is a catch-all entry (empty kinds or ["*"])
|
||||
func (rf *ResourceFilter) IsCatchAll() bool {
|
||||
return len(rf.Kinds) == 0 || (len(rf.Kinds) == 1 && rf.Kinds[0] == "*")
|
||||
}
|
||||
|
||||
// ClusterScopedFilterPolicy defines backup filters scoped globally to cluster-scoped resources.
|
||||
type ClusterScopedFilterPolicy struct {
|
||||
ResourceFilters []ResourceFilter `yaml:"resourceFilters"`
|
||||
}
|
||||
|
||||
// NamespacedFilterPolicy defines backup filters scoped to specific namespaces.
|
||||
type NamespacedFilterPolicy struct {
|
||||
Namespaces []string `yaml:"namespaces"`
|
||||
ResourceFilters []ResourceFilter `yaml:"resourceFilters"`
|
||||
}
|
||||
|
||||
// IncludeExcludePolicy defined policy to include or exclude resources based on the names
|
||||
type IncludeExcludePolicy struct {
|
||||
// The following fields have the same semantics as those from the spec of backup.
|
||||
@@ -95,17 +122,21 @@ type VolumePolicy struct {
|
||||
|
||||
// ResourcePolicies currently defined slice of volume policies to handle backup
|
||||
type ResourcePolicies struct {
|
||||
Version string `yaml:"version"`
|
||||
VolumePolicies []VolumePolicy `yaml:"volumePolicies"`
|
||||
IncludeExcludePolicy *IncludeExcludePolicy `yaml:"includeExcludePolicy"`
|
||||
Version string `yaml:"version"`
|
||||
VolumePolicies []VolumePolicy `yaml:"volumePolicies"`
|
||||
IncludeExcludePolicy *IncludeExcludePolicy `yaml:"includeExcludePolicy"`
|
||||
ClusterScopedFilterPolicy *ClusterScopedFilterPolicy `yaml:"clusterScopedFilterPolicy,omitempty"`
|
||||
NamespacedFilterPolicies []NamespacedFilterPolicy `yaml:"namespacedFilterPolicies,omitempty"`
|
||||
// we may support other resource policies in the future, and they could be added separately
|
||||
// OtherResourcePolicies []OtherResourcePolicy
|
||||
}
|
||||
|
||||
type Policies struct {
|
||||
version string
|
||||
volumePolicies []volPolicy
|
||||
includeExcludePolicy *IncludeExcludePolicy
|
||||
version string
|
||||
volumePolicies []volPolicy
|
||||
includeExcludePolicy *IncludeExcludePolicy
|
||||
clusterScopedFilterPolicy *ClusterScopedFilterPolicy
|
||||
namespacedFilterPolicies []NamespacedFilterPolicy
|
||||
// OtherPolicies
|
||||
}
|
||||
|
||||
@@ -158,6 +189,8 @@ func (p *Policies) BuildPolicy(resPolicies *ResourcePolicies) error {
|
||||
|
||||
p.version = resPolicies.Version
|
||||
p.includeExcludePolicy = resPolicies.IncludeExcludePolicy
|
||||
p.clusterScopedFilterPolicy = resPolicies.ClusterScopedFilterPolicy
|
||||
p.namespacedFilterPolicies = resPolicies.NamespacedFilterPolicies
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -228,6 +261,14 @@ func (p *Policies) Validate() error {
|
||||
}
|
||||
}
|
||||
|
||||
if err := p.validateClusterScopedFilterPolicy(); err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
if err := p.validateNamespacedFilterPolicies(); err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -235,6 +276,14 @@ func (p *Policies) GetIncludeExcludePolicy() *IncludeExcludePolicy {
|
||||
return p.includeExcludePolicy
|
||||
}
|
||||
|
||||
func (p *Policies) GetClusterScopedFilterPolicy() *ClusterScopedFilterPolicy {
|
||||
return p.clusterScopedFilterPolicy
|
||||
}
|
||||
|
||||
func (p *Policies) GetNamespacedFilterPolicies() []NamespacedFilterPolicy {
|
||||
return p.namespacedFilterPolicies
|
||||
}
|
||||
|
||||
func GetResourcePoliciesFromBackup(
|
||||
backup velerov1api.Backup,
|
||||
client crclient.Client,
|
||||
@@ -296,3 +345,117 @@ func getResourcePoliciesFromConfig(cm *corev1api.ConfigMap) (*Policies, error) {
|
||||
|
||||
return policies, nil
|
||||
}
|
||||
|
||||
func (p *Policies) validateNamespacedFilterPolicies() error {
|
||||
seenPatterns := make(map[string][]int) // pattern -> list of policy indices
|
||||
|
||||
// Rule 1-7: Basic validation rules
|
||||
for i, nfp := range p.namespacedFilterPolicies {
|
||||
if len(nfp.Namespaces) == 0 {
|
||||
return fmt.Errorf("namespacedFilterPolicies[%d]: at least one namespace must be specified", i)
|
||||
}
|
||||
if len(nfp.ResourceFilters) == 0 {
|
||||
return fmt.Errorf("namespacedFilterPolicies[%d]: at least one resourceFilter must be specified", i)
|
||||
}
|
||||
|
||||
// Rule 8 & 9: Validate glob patterns and collect namespace patterns for duplicate check
|
||||
for j, pattern := range nfp.Namespaces {
|
||||
if err := wildcard.ValidateNamespaceName(pattern); err != nil {
|
||||
return fmt.Errorf("namespacedFilterPolicies[%d].namespaces[%d]: %w", i, j, err)
|
||||
}
|
||||
seenPatterns[pattern] = append(seenPatterns[pattern], i)
|
||||
}
|
||||
|
||||
seenKinds := make(map[string]int)
|
||||
hasCatchAll := false
|
||||
for j, rf := range nfp.ResourceFilters {
|
||||
if rf.IsCatchAll() {
|
||||
if hasCatchAll {
|
||||
return fmt.Errorf("namespacedFilterPolicies[%d]: only one catch-all resource filter is allowed", i)
|
||||
}
|
||||
hasCatchAll = true
|
||||
if len(rf.Names) > 0 || len(rf.ExcludedNames) > 0 {
|
||||
return fmt.Errorf("namespacedFilterPolicies[%d].resourceFilters[%d]: names or excludedNames cannot be specified for catch-all filters", i, j)
|
||||
}
|
||||
}
|
||||
|
||||
for _, kind := range rf.Kinds {
|
||||
if kind == "*" {
|
||||
continue // "*" is handled by IsCatchAll, no need to check duplicates against other kinds
|
||||
}
|
||||
if prevJ, ok := seenKinds[kind]; ok {
|
||||
return fmt.Errorf("namespacedFilterPolicies[%d]: kind %q appears in both resourceFilters[%d] and resourceFilters[%d]", i, kind, prevJ, j)
|
||||
}
|
||||
seenKinds[kind] = j
|
||||
}
|
||||
|
||||
if len(rf.LabelSelector) > 0 && len(rf.OrLabelSelectors) > 0 {
|
||||
return fmt.Errorf("namespacedFilterPolicies[%d].resourceFilters[%d]: labelSelector and orLabelSelectors cannot co-exist", i, j)
|
||||
}
|
||||
|
||||
// Validate glob patterns for names and excludedNames using gobwas/glob
|
||||
for k, pattern := range rf.Names {
|
||||
if _, err := glob.Compile(pattern); err != nil {
|
||||
return fmt.Errorf("namespacedFilterPolicies[%d].resourceFilters[%d].names[%d]: invalid glob pattern %q: %v", i, j, k, pattern, err)
|
||||
}
|
||||
}
|
||||
for k, pattern := range rf.ExcludedNames {
|
||||
if _, err := glob.Compile(pattern); err != nil {
|
||||
return fmt.Errorf("namespacedFilterPolicies[%d].resourceFilters[%d].excludedNames[%d]: invalid glob pattern %q: %v", i, j, k, pattern, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Rule 8: Report exact duplicates only
|
||||
for pattern, policyIndices := range seenPatterns {
|
||||
if len(policyIndices) > 1 {
|
||||
return fmt.Errorf(
|
||||
"namespacedFilterPolicies: duplicate namespace pattern '%s' found in policies %v",
|
||||
pattern, policyIndices)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Policies) validateClusterScopedFilterPolicy() error {
|
||||
if p.clusterScopedFilterPolicy == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if len(p.clusterScopedFilterPolicy.ResourceFilters) == 0 {
|
||||
return fmt.Errorf("clusterScopedFilterPolicy: resourceFilters cannot be empty; remove the policy block entirely if it is not needed")
|
||||
}
|
||||
|
||||
seenKinds := make(map[string]int)
|
||||
for j, rf := range p.clusterScopedFilterPolicy.ResourceFilters {
|
||||
if rf.IsCatchAll() {
|
||||
return fmt.Errorf("clusterScopedFilterPolicy.resourceFilters[%d]: kinds must be specified (catch-all is not supported)", j)
|
||||
}
|
||||
|
||||
for _, kind := range rf.Kinds {
|
||||
if prevJ, ok := seenKinds[kind]; ok {
|
||||
return fmt.Errorf("clusterScopedFilterPolicy: kind %q appears in both resourceFilters[%d] and resourceFilters[%d]", kind, prevJ, j)
|
||||
}
|
||||
seenKinds[kind] = j
|
||||
}
|
||||
|
||||
if len(rf.LabelSelector) > 0 && len(rf.OrLabelSelectors) > 0 {
|
||||
return fmt.Errorf("clusterScopedFilterPolicy.resourceFilters[%d]: labelSelector and orLabelSelectors cannot co-exist", j)
|
||||
}
|
||||
|
||||
for k, pattern := range rf.Names {
|
||||
if _, err := glob.Compile(pattern); err != nil {
|
||||
return fmt.Errorf("clusterScopedFilterPolicy.resourceFilters[%d].names[%d]: invalid glob pattern %q: %v", j, k, pattern, err)
|
||||
}
|
||||
}
|
||||
for k, pattern := range rf.ExcludedNames {
|
||||
if _, err := glob.Compile(pattern); err != nil {
|
||||
return fmt.Errorf("clusterScopedFilterPolicy.resourceFilters[%d].excludedNames[%d]: invalid glob pattern %q: %v", j, k, pattern, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1242,3 +1242,441 @@ func TestPVCPhaseMatch(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNamespacedFilterPolicies(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
yamlData string
|
||||
wantErr bool
|
||||
errMsg string
|
||||
}{
|
||||
{
|
||||
name: "valid namespacedFilterPolicies with multiple kinds",
|
||||
yamlData: `version: v1
|
||||
namespacedFilterPolicies:
|
||||
- namespaces: ["frontend", "backend"]
|
||||
resourceFilters:
|
||||
- kinds: ["Pod", "ConfigMap"]
|
||||
labelSelector:
|
||||
app: web
|
||||
names: ["app-*"]
|
||||
- kinds: ["Secret"]
|
||||
excludedNames: ["temp-*"]`,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "valid namespacedFilterPolicies with glob patterns",
|
||||
yamlData: `version: v1
|
||||
namespacedFilterPolicies:
|
||||
- namespaces: ["team-*"]
|
||||
resourceFilters:
|
||||
- kinds: ["Pod"]
|
||||
orLabelSelectors:
|
||||
- env: prod
|
||||
- env: staging`,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "valid - overlapping patterns allowed (first-match semantics)",
|
||||
yamlData: `version: v1
|
||||
namespacedFilterPolicies:
|
||||
- namespaces: ["team-frontend-*"]
|
||||
resourceFilters:
|
||||
- kinds: ["Pod", "ConfigMap", "Secret"]
|
||||
- namespaces: ["team-*"]
|
||||
resourceFilters:
|
||||
- kinds: ["Deployment", "Service"]`,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "invalid - no namespaces",
|
||||
yamlData: `version: v1
|
||||
namespacedFilterPolicies:
|
||||
- namespaces: []
|
||||
resourceFilters:
|
||||
- kinds: ["Pod"]`,
|
||||
wantErr: true,
|
||||
errMsg: "at least one namespace must be specified",
|
||||
},
|
||||
{
|
||||
name: "invalid - no resourceFilters",
|
||||
yamlData: `version: v1
|
||||
namespacedFilterPolicies:
|
||||
- namespaces: ["test"]
|
||||
resourceFilters: []`,
|
||||
wantErr: true,
|
||||
errMsg: "at least one resourceFilter must be specified",
|
||||
},
|
||||
{
|
||||
name: "valid - asterisk catch-all",
|
||||
yamlData: `version: v1
|
||||
namespacedFilterPolicies:
|
||||
- namespaces: ["test"]
|
||||
resourceFilters:
|
||||
- kinds: ["*"]
|
||||
labelSelector:
|
||||
app: web`,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "invalid - multiple asterisk kinds",
|
||||
yamlData: `version: v1
|
||||
namespacedFilterPolicies:
|
||||
- namespaces: ["test"]
|
||||
resourceFilters:
|
||||
- kinds: ["*"]
|
||||
labelSelector:
|
||||
app: web
|
||||
- kinds: ["*"]
|
||||
labelSelector:
|
||||
app: db`,
|
||||
wantErr: true,
|
||||
errMsg: "only one catch-all resource filter is allowed",
|
||||
},
|
||||
{
|
||||
name: "invalid - empty and asterisk kinds",
|
||||
yamlData: `version: v1
|
||||
namespacedFilterPolicies:
|
||||
- namespaces: ["test"]
|
||||
resourceFilters:
|
||||
- kinds: []
|
||||
labelSelector:
|
||||
app: web
|
||||
- kinds: ["*"]
|
||||
labelSelector:
|
||||
app: db`,
|
||||
wantErr: true,
|
||||
errMsg: "only one catch-all resource filter is allowed",
|
||||
},
|
||||
{
|
||||
name: "invalid - multiple empty kinds",
|
||||
yamlData: `version: v1
|
||||
namespacedFilterPolicies:
|
||||
- namespaces: ["test"]
|
||||
resourceFilters:
|
||||
- kinds: []
|
||||
labelSelector:
|
||||
app: web
|
||||
- kinds: []
|
||||
labelSelector:
|
||||
app: db`,
|
||||
wantErr: true,
|
||||
errMsg: "only one catch-all resource filter is allowed",
|
||||
},
|
||||
{
|
||||
name: "invalid - names with empty kinds",
|
||||
yamlData: `version: v1
|
||||
namespacedFilterPolicies:
|
||||
- namespaces: ["test"]
|
||||
resourceFilters:
|
||||
- kinds: []
|
||||
names: ["app-*"]
|
||||
labelSelector:
|
||||
app: web`,
|
||||
wantErr: true,
|
||||
errMsg: "names or excludedNames cannot be specified for catch-all filters",
|
||||
},
|
||||
{
|
||||
name: "invalid - excludedNames with empty kinds",
|
||||
yamlData: `version: v1
|
||||
namespacedFilterPolicies:
|
||||
- namespaces: ["test"]
|
||||
resourceFilters:
|
||||
- kinds: []
|
||||
excludedNames: ["app-*"]
|
||||
labelSelector:
|
||||
app: web`,
|
||||
wantErr: true,
|
||||
errMsg: "names or excludedNames cannot be specified for catch-all filters",
|
||||
},
|
||||
{
|
||||
name: "valid - no label selectors with catch-all",
|
||||
yamlData: `version: v1
|
||||
namespacedFilterPolicies:
|
||||
- namespaces: ["test"]
|
||||
resourceFilters:
|
||||
- kinds: ["*"]`,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "invalid - duplicate kinds",
|
||||
yamlData: `version: v1
|
||||
namespacedFilterPolicies:
|
||||
- namespaces: ["test"]
|
||||
resourceFilters:
|
||||
- kinds: ["Pod"]
|
||||
- kinds: ["Pod", "ConfigMap"]`,
|
||||
wantErr: true,
|
||||
errMsg: "kind \"Pod\" appears in both resourceFilters",
|
||||
},
|
||||
{
|
||||
name: "invalid - both labelSelector and orLabelSelectors",
|
||||
yamlData: `version: v1
|
||||
namespacedFilterPolicies:
|
||||
- namespaces: ["test"]
|
||||
resourceFilters:
|
||||
- kinds: ["Pod"]
|
||||
labelSelector:
|
||||
app: web
|
||||
orLabelSelectors:
|
||||
- env: prod`,
|
||||
wantErr: true,
|
||||
errMsg: "labelSelector and orLabelSelectors cannot co-exist",
|
||||
},
|
||||
{
|
||||
name: "invalid - bad glob pattern in names",
|
||||
yamlData: `version: v1
|
||||
namespacedFilterPolicies:
|
||||
- namespaces: ["test"]
|
||||
resourceFilters:
|
||||
- kinds: ["Pod"]
|
||||
names: ["[invalid"]`,
|
||||
wantErr: true,
|
||||
errMsg: "invalid glob pattern",
|
||||
},
|
||||
{
|
||||
name: "invalid - duplicate namespace pattern",
|
||||
yamlData: `version: v1
|
||||
namespacedFilterPolicies:
|
||||
- namespaces: ["production"]
|
||||
resourceFilters:
|
||||
- kinds: ["Pod"]
|
||||
- namespaces: ["production"]
|
||||
resourceFilters:
|
||||
- kinds: ["ConfigMap"]`,
|
||||
wantErr: true,
|
||||
errMsg: "duplicate namespace pattern",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
resPolicies, err := unmarshalResourcePolicies(&tc.yamlData)
|
||||
require.NoError(t, err) // Unmarshal should always succeed for our test cases
|
||||
|
||||
policies := &Policies{}
|
||||
err = policies.BuildPolicy(resPolicies)
|
||||
require.NoError(t, err) // BuildPolicy should always succeed for our test cases
|
||||
|
||||
err = policies.Validate()
|
||||
if tc.wantErr {
|
||||
require.Error(t, err)
|
||||
if tc.errMsg != "" {
|
||||
assert.Contains(t, err.Error(), tc.errMsg)
|
||||
}
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify that we can retrieve the policies
|
||||
nfPolicies := policies.GetNamespacedFilterPolicies()
|
||||
assert.GreaterOrEqual(t, len(nfPolicies), 1) // Valid test cases have at least 1 policy
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNamespacedFilterPoliciesAccessor(t *testing.T) {
|
||||
yamlData := `version: v1
|
||||
namespacedFilterPolicies:
|
||||
- namespaces: ["frontend"]
|
||||
resourceFilters:
|
||||
- kinds: ["Pod"]
|
||||
labelSelector:
|
||||
app: web`
|
||||
|
||||
resPolicies, err := unmarshalResourcePolicies(&yamlData)
|
||||
require.NoError(t, err)
|
||||
|
||||
policies := &Policies{}
|
||||
err = policies.BuildPolicy(resPolicies)
|
||||
require.NoError(t, err)
|
||||
|
||||
nfPolicies := policies.GetNamespacedFilterPolicies()
|
||||
require.Len(t, nfPolicies, 1)
|
||||
|
||||
policy := nfPolicies[0]
|
||||
assert.Equal(t, []string{"frontend"}, policy.Namespaces)
|
||||
assert.Len(t, policy.ResourceFilters, 1)
|
||||
|
||||
rf := policy.ResourceFilters[0]
|
||||
assert.Equal(t, []string{"Pod"}, rf.Kinds)
|
||||
assert.Equal(t, map[string]string{"app": "web"}, rf.LabelSelector)
|
||||
}
|
||||
|
||||
func TestFirstMatchSemantics(t *testing.T) {
|
||||
yamlData := `version: v1
|
||||
namespacedFilterPolicies:
|
||||
- namespaces: ["team-frontend-*", "specific-ns"]
|
||||
resourceFilters:
|
||||
- kinds: ["Pod", "ConfigMap", "Secret"]
|
||||
- namespaces: ["team-*", "another-pattern"]
|
||||
resourceFilters:
|
||||
- kinds: ["Deployment", "Service"]`
|
||||
|
||||
resPolicies, err := unmarshalResourcePolicies(&yamlData)
|
||||
require.NoError(t, err)
|
||||
|
||||
policies := &Policies{}
|
||||
err = policies.BuildPolicy(resPolicies)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = policies.Validate()
|
||||
require.NoError(t, err)
|
||||
|
||||
nfPolicies := policies.GetNamespacedFilterPolicies()
|
||||
require.Len(t, nfPolicies, 2)
|
||||
|
||||
// Verify the first policy has the more specific patterns
|
||||
policy1 := nfPolicies[0]
|
||||
assert.Equal(t, []string{"team-frontend-*", "specific-ns"}, policy1.Namespaces)
|
||||
assert.Equal(t, []string{"Pod", "ConfigMap", "Secret"}, policy1.ResourceFilters[0].Kinds)
|
||||
|
||||
// Verify the second policy has the broader patterns
|
||||
policy2 := nfPolicies[1]
|
||||
assert.Equal(t, []string{"team-*", "another-pattern"}, policy2.Namespaces)
|
||||
assert.Equal(t, []string{"Deployment", "Service"}, policy2.ResourceFilters[0].Kinds)
|
||||
}
|
||||
|
||||
func TestClusterScopedFilterPolicies(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
yamlData string
|
||||
wantErr bool
|
||||
errMsg string
|
||||
}{
|
||||
{
|
||||
name: "valid - single kind with names",
|
||||
yamlData: `version: v1
|
||||
clusterScopedFilterPolicy:
|
||||
resourceFilters:
|
||||
- kinds: ["ClusterRole"]
|
||||
names: ["my-app-*"]`,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "valid - multi-kind with labelSelector",
|
||||
yamlData: `version: v1
|
||||
clusterScopedFilterPolicy:
|
||||
resourceFilters:
|
||||
- kinds: ["ClusterRole", "ClusterRoleBinding"]
|
||||
labelSelector:
|
||||
app: my-app`,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "valid - orLabelSelectors",
|
||||
yamlData: `version: v1
|
||||
clusterScopedFilterPolicy:
|
||||
resourceFilters:
|
||||
- kinds: ["CustomResourceDefinition"]
|
||||
orLabelSelectors:
|
||||
- app: my-app
|
||||
- app: other-app`,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "valid - excludedNames",
|
||||
yamlData: `version: v1
|
||||
clusterScopedFilterPolicy:
|
||||
resourceFilters:
|
||||
- kinds: ["ClusterRole"]
|
||||
names: ["my-*"]
|
||||
excludedNames: ["my-debug-*"]`,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "invalid - empty resourceFilters",
|
||||
yamlData: `version: v1
|
||||
clusterScopedFilterPolicy:
|
||||
resourceFilters: []`,
|
||||
wantErr: true,
|
||||
errMsg: "resourceFilters cannot be empty; remove the policy block entirely if it is not needed",
|
||||
},
|
||||
{
|
||||
name: "invalid - empty kinds in clusterScopedFilterPolicy",
|
||||
yamlData: `version: v1
|
||||
clusterScopedFilterPolicy:
|
||||
resourceFilters:
|
||||
- kinds: []
|
||||
names: ["my-app-*"]`,
|
||||
wantErr: true,
|
||||
errMsg: "kinds must be specified",
|
||||
},
|
||||
{
|
||||
name: "invalid - asterisk kinds (explicit catch-all) in clusterScopedFilterPolicy",
|
||||
yamlData: `version: v1
|
||||
clusterScopedFilterPolicy:
|
||||
resourceFilters:
|
||||
- kinds: ["*"]
|
||||
labelSelector:
|
||||
app: my-app`,
|
||||
wantErr: true,
|
||||
errMsg: "kinds must be specified",
|
||||
},
|
||||
{
|
||||
name: "invalid - duplicate kinds across entries",
|
||||
yamlData: `version: v1
|
||||
clusterScopedFilterPolicy:
|
||||
resourceFilters:
|
||||
- kinds: ["ClusterRole"]
|
||||
names: ["my-app-*"]
|
||||
- kinds: ["ClusterRole"]
|
||||
labelSelector:
|
||||
app: other`,
|
||||
wantErr: true,
|
||||
errMsg: `kind "ClusterRole" appears in both`,
|
||||
},
|
||||
{
|
||||
name: "invalid - labelSelector and orLabelSelectors co-exist",
|
||||
yamlData: `version: v1
|
||||
clusterScopedFilterPolicy:
|
||||
resourceFilters:
|
||||
- kinds: ["ClusterRole"]
|
||||
labelSelector:
|
||||
app: my-app
|
||||
orLabelSelectors:
|
||||
- app: other`,
|
||||
wantErr: true,
|
||||
errMsg: "labelSelector and orLabelSelectors cannot co-exist",
|
||||
},
|
||||
{
|
||||
name: "invalid - bad glob in names",
|
||||
yamlData: `version: v1
|
||||
clusterScopedFilterPolicy:
|
||||
resourceFilters:
|
||||
- kinds: ["ClusterRole"]
|
||||
names: ["[invalid"]`,
|
||||
wantErr: true,
|
||||
errMsg: "invalid glob pattern",
|
||||
},
|
||||
{
|
||||
name: "invalid - bad glob in excludedNames",
|
||||
yamlData: `version: v1
|
||||
clusterScopedFilterPolicy:
|
||||
resourceFilters:
|
||||
- kinds: ["ClusterRole"]
|
||||
excludedNames: ["[bad"]`,
|
||||
wantErr: true,
|
||||
errMsg: "invalid glob pattern",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
resPolicies, err := unmarshalResourcePolicies(&tc.yamlData)
|
||||
require.NoError(t, err)
|
||||
|
||||
policies := &Policies{}
|
||||
err = policies.BuildPolicy(resPolicies)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = policies.Validate()
|
||||
if tc.wantErr {
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), tc.errMsg)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
[build]
|
||||
base = "site/"
|
||||
command = "hugo --gc --minify"
|
||||
publish = "site/public"
|
||||
publish = "public"
|
||||
|
||||
[context.production.environment]
|
||||
HUGO_VERSION = "0.73.0"
|
||||
|
||||
@@ -19,6 +19,9 @@ package backup
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/gobwas/glob"
|
||||
"k8s.io/apimachinery/pkg/labels"
|
||||
|
||||
"github.com/vmware-tanzu/velero/internal/hook"
|
||||
"github.com/vmware-tanzu/velero/internal/resourcepolicies"
|
||||
"github.com/vmware-tanzu/velero/internal/volume"
|
||||
@@ -34,6 +37,21 @@ type itemKey struct {
|
||||
name string
|
||||
}
|
||||
|
||||
// ResolvedResourceFilter holds the materialized filter state for one kind-group
|
||||
// within a namespace.
|
||||
type ResolvedResourceFilter struct {
|
||||
LabelSelector labels.Selector
|
||||
OrLabelSelectors []labels.Selector
|
||||
NameIE *collections.IncludesExcludes
|
||||
}
|
||||
|
||||
// ResolvedNamespaceFilter holds the materialized filter state for a namespace.
|
||||
// ResourceFilterMap is keyed by the resolved group-resource string.
|
||||
type ResolvedNamespaceFilter struct {
|
||||
ResourceFilterMap map[string]*ResolvedResourceFilter
|
||||
CatchAllFilter *ResolvedResourceFilter
|
||||
}
|
||||
|
||||
type SynchronizedVSList struct {
|
||||
sync.Mutex
|
||||
VolumeSnapshotList []*volume.Snapshot
|
||||
@@ -70,6 +88,27 @@ type Request struct {
|
||||
SkippedPVTracker *skipPVTracker
|
||||
VolumesInformation volume.BackupVolumesInformation
|
||||
WorkerPool *ItemBlockWorkerPool
|
||||
|
||||
// ClusterScopedFilterMap holds resolved global filters for cluster-scoped resources.
|
||||
// Key is the resolved group-resource string.
|
||||
ClusterScopedFilterMap map[string]*ResolvedResourceFilter
|
||||
|
||||
// NamespacedFilterMap holds resolved per-namespace filters.
|
||||
// Key is either an exact namespace name or a glob pattern.
|
||||
NamespacedFilterMap map[string]*ResolvedNamespaceFilter
|
||||
|
||||
// NamespacedFilterPatterns preserves the order of patterns for first-match semantics
|
||||
// and caches pre-compiled globs to avoid repeated compilation in the hot path.
|
||||
NamespacedFilterPatterns []NamespacedFilterPattern
|
||||
}
|
||||
|
||||
// NamespacedFilterPattern pairs a namespace pattern string with its pre-compiled
|
||||
// glob so that GetNamespaceFilter does not recompile on every call.
|
||||
// Compiled is nil for exact-match (non-glob) patterns, which are looked up
|
||||
// directly in NamespacedFilterMap.
|
||||
type NamespacedFilterPattern struct {
|
||||
Pattern string
|
||||
Compiled glob.Glob
|
||||
}
|
||||
|
||||
// BackupVolumesInformation contains the information needs by generating
|
||||
@@ -107,3 +146,25 @@ func (r *Request) FillVolumesInformation() {
|
||||
func (r *Request) StopWorkerPool() {
|
||||
r.WorkerPool.Stop()
|
||||
}
|
||||
|
||||
// GetNamespaceFilter returns the resolved filter for a namespace, or nil
|
||||
// if the namespace should use global filters. Uses first-match semantics
|
||||
// when multiple patterns could match the same namespace.
|
||||
func (r *Request) GetNamespaceFilter(namespace string) *ResolvedNamespaceFilter {
|
||||
if r.NamespacedFilterMap == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// First check for exact match
|
||||
if f, ok := r.NamespacedFilterMap[namespace]; ok {
|
||||
return f
|
||||
}
|
||||
|
||||
// Walk patterns in definition order using pre-compiled globs (no allocation per call)
|
||||
for _, p := range r.NamespacedFilterPatterns {
|
||||
if p.Compiled != nil && p.Compiled.Match(namespace) {
|
||||
return r.NamespacedFilterMap[p.Pattern]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
/*
|
||||
Copyright The Velero Contributors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package cbtservice
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/kubernetes-csi/external-snapshot-metadata/pkg/iterator"
|
||||
"github.com/sirupsen/logrus"
|
||||
"k8s.io/client-go/rest"
|
||||
)
|
||||
|
||||
var (
|
||||
buildClients = iterator.BuildClients
|
||||
getSnapshotMetadata = iterator.GetSnapshotMetadata
|
||||
)
|
||||
|
||||
type emitterImpl struct {
|
||||
logger logrus.FieldLogger
|
||||
recordCallBack func([]Range) error
|
||||
}
|
||||
|
||||
func (e *emitterImpl) SnapshotMetadataIteratorRecord(recordNumber int, metadata iterator.IteratorMetadata) error {
|
||||
for _, b := range metadata.BlockMetadata {
|
||||
// Offset and size should not be negative, if they are, it indicates some error in the metadata and we should return error to stop the iteration.
|
||||
if b.ByteOffset < 0 || b.SizeBytes < 0 {
|
||||
return fmt.Errorf("invalid CBT metadata: offset: %v, size %v", b.ByteOffset, b.SizeBytes)
|
||||
}
|
||||
|
||||
e.logger.Debugf("recording metadata for record number %d, offset: %v, size: %v",
|
||||
recordNumber, b.ByteOffset, b.SizeBytes)
|
||||
|
||||
if err := e.recordCallBack([]Range{{Offset: uint64(b.ByteOffset), Length: uint64(b.SizeBytes)}}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *emitterImpl) SnapshotMetadataIteratorDone(numberRecords int) error {
|
||||
e.logger.Infof("finished iterating snapshot metadata, total number of records: %d", numberRecords)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type ServiceImpl struct {
|
||||
logger logrus.FieldLogger
|
||||
vsNamespace string
|
||||
SAName string
|
||||
clientConfig *rest.Config
|
||||
}
|
||||
|
||||
func NewService(
|
||||
logger logrus.FieldLogger,
|
||||
vsNamespace,
|
||||
saName string,
|
||||
clientConfig *rest.Config,
|
||||
) Service {
|
||||
return &ServiceImpl{
|
||||
logger: logger,
|
||||
vsNamespace: vsNamespace,
|
||||
SAName: saName,
|
||||
clientConfig: clientConfig,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *ServiceImpl) GetAllocatedBlocks(ctx context.Context, snapshot string, record func([]Range) error) error {
|
||||
clients, err := buildClients(s.clientConfig)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
args := iterator.Args{
|
||||
SnapshotName: snapshot,
|
||||
Emitter: &emitterImpl{
|
||||
logger: s.logger,
|
||||
recordCallBack: record,
|
||||
},
|
||||
|
||||
Clients: clients,
|
||||
Namespace: s.vsNamespace, // DataUpload is created in the same namespace as Velero server. vsNamespace is the namespace of the Velero server.
|
||||
SANamespace: s.vsNamespace, // The SA is created in the same namespace as Velero server. vsNamespace is the namespace of Velero server.
|
||||
SAName: s.SAName,
|
||||
TokenExpirySecs: iterator.DefaultTokenExpirySeconds,
|
||||
MaxResults: 0, // If 0 then the CSI driver decides the value.
|
||||
}
|
||||
|
||||
return getSnapshotMetadata(ctx, args)
|
||||
}
|
||||
|
||||
func (s *ServiceImpl) GetChangedBlocks(ctx context.Context, snapshot string, changeID string, record func([]Range) error) error {
|
||||
clients, err := buildClients(s.clientConfig)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
args := iterator.Args{
|
||||
SnapshotName: snapshot,
|
||||
PrevSnapshotName: changeID,
|
||||
Emitter: &emitterImpl{
|
||||
logger: s.logger,
|
||||
recordCallBack: record,
|
||||
},
|
||||
|
||||
Clients: clients,
|
||||
Namespace: s.vsNamespace,
|
||||
SANamespace: s.vsNamespace,
|
||||
SAName: s.SAName,
|
||||
TokenExpirySecs: iterator.DefaultTokenExpirySeconds,
|
||||
MaxResults: 0, // If 0 then the CSI driver decides the value.
|
||||
}
|
||||
|
||||
return getSnapshotMetadata(ctx, args)
|
||||
}
|
||||
@@ -0,0 +1,258 @@
|
||||
/*
|
||||
Copyright The Velero Contributors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package cbtservice
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/kubernetes-csi/external-snapshot-metadata/pkg/api"
|
||||
"github.com/kubernetes-csi/external-snapshot-metadata/pkg/iterator"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"k8s.io/client-go/rest"
|
||||
)
|
||||
|
||||
func TestEmitterImplSnapshotMetadataIteratorRecord(t *testing.T) {
|
||||
t.Run("records block metadata as ranges", func(t *testing.T) {
|
||||
var got [][]Range
|
||||
emitter := &emitterImpl{
|
||||
logger: logrus.New(),
|
||||
recordCallBack: func(ranges []Range) error {
|
||||
got = append(got, ranges)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
err := emitter.SnapshotMetadataIteratorRecord(7, iterator.IteratorMetadata{
|
||||
BlockMetadata: []*api.BlockMetadata{
|
||||
{ByteOffset: 10, SizeBytes: 20},
|
||||
{ByteOffset: 40, SizeBytes: 50},
|
||||
},
|
||||
})
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, [][]Range{
|
||||
{{Offset: 10, Length: 20}},
|
||||
{{Offset: 40, Length: 50}},
|
||||
}, got)
|
||||
})
|
||||
|
||||
t.Run("rejects negative block metadata", func(t *testing.T) {
|
||||
callbackCalled := false
|
||||
emitter := &emitterImpl{
|
||||
logger: logrus.New(),
|
||||
recordCallBack: func(ranges []Range) error {
|
||||
callbackCalled = true
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
err := emitter.SnapshotMetadataIteratorRecord(3, iterator.IteratorMetadata{
|
||||
BlockMetadata: []*api.BlockMetadata{{ByteOffset: -1, SizeBytes: 20}},
|
||||
})
|
||||
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "invalid CBT metadata")
|
||||
assert.False(t, callbackCalled)
|
||||
})
|
||||
|
||||
t.Run("returns callback error", func(t *testing.T) {
|
||||
wantErr := errors.New("record failed")
|
||||
emitter := &emitterImpl{
|
||||
logger: logrus.New(),
|
||||
recordCallBack: func(ranges []Range) error {
|
||||
return wantErr
|
||||
},
|
||||
}
|
||||
|
||||
err := emitter.SnapshotMetadataIteratorRecord(5, iterator.IteratorMetadata{
|
||||
BlockMetadata: []*api.BlockMetadata{{ByteOffset: 1, SizeBytes: 2}},
|
||||
})
|
||||
|
||||
require.ErrorIs(t, err, wantErr)
|
||||
})
|
||||
}
|
||||
|
||||
func TestEmitterImplSnapshotMetadataIteratorDone(t *testing.T) {
|
||||
emitter := &emitterImpl{logger: logrus.New()}
|
||||
assert.NoError(t, emitter.SnapshotMetadataIteratorDone(4))
|
||||
}
|
||||
|
||||
func TestNewServiceInitializesFields(t *testing.T) {
|
||||
logger := logrus.New()
|
||||
cfg := &rest.Config{Host: "https://example.com"}
|
||||
|
||||
svc := NewService(logger, "velero-ns", "velero-sa", cfg)
|
||||
|
||||
impl, ok := svc.(*ServiceImpl)
|
||||
require.True(t, ok)
|
||||
assert.Same(t, logger, impl.logger)
|
||||
assert.Equal(t, "velero-ns", impl.vsNamespace)
|
||||
assert.Equal(t, "velero-sa", impl.SAName)
|
||||
assert.Same(t, cfg, impl.clientConfig)
|
||||
}
|
||||
|
||||
func TestNewServiceForwardsSANameToArgs(t *testing.T) {
|
||||
originalBuildClients := buildClients
|
||||
originalGetSnapshotMetadata := getSnapshotMetadata
|
||||
t.Cleanup(func() {
|
||||
buildClients = originalBuildClients
|
||||
getSnapshotMetadata = originalGetSnapshotMetadata
|
||||
})
|
||||
|
||||
buildClients = func(config *rest.Config) (iterator.Clients, error) {
|
||||
return iterator.Clients{}, nil
|
||||
}
|
||||
|
||||
var capturedArgs iterator.Args
|
||||
getSnapshotMetadata = func(ctx context.Context, args iterator.Args) error {
|
||||
capturedArgs = args
|
||||
return nil
|
||||
}
|
||||
|
||||
svc := NewService(logrus.New(), "velero-ns", "velero-sa", &rest.Config{Host: "https://example.com"})
|
||||
|
||||
err := svc.GetAllocatedBlocks(t.Context(), "snap-1", func([]Range) error { return nil })
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "velero-ns", capturedArgs.Namespace)
|
||||
assert.Equal(t, "velero-ns", capturedArgs.SANamespace)
|
||||
assert.Equal(t, "velero-sa", capturedArgs.SAName)
|
||||
}
|
||||
|
||||
func TestServiceImplGetAllocatedBlocks(t *testing.T) {
|
||||
originalBuildClients := buildClients
|
||||
originalGetSnapshotMetadata := getSnapshotMetadata
|
||||
t.Cleanup(func() {
|
||||
buildClients = originalBuildClients
|
||||
getSnapshotMetadata = originalGetSnapshotMetadata
|
||||
})
|
||||
|
||||
t.Run("build clients error is returned", func(t *testing.T) {
|
||||
wantErr := errors.New("build clients failed")
|
||||
getSnapshotCalled := false
|
||||
buildClients = func(config *rest.Config) (iterator.Clients, error) {
|
||||
return iterator.Clients{}, wantErr
|
||||
}
|
||||
getSnapshotMetadata = func(ctx context.Context, args iterator.Args) error {
|
||||
getSnapshotCalled = true
|
||||
return nil
|
||||
}
|
||||
|
||||
service := &ServiceImpl{logger: logrus.New(), clientConfig: &rest.Config{Host: "https://example.com"}}
|
||||
|
||||
err := service.GetAllocatedBlocks(t.Context(), "snap-1", func([]Range) error { return nil })
|
||||
|
||||
require.ErrorIs(t, err, wantErr)
|
||||
assert.False(t, getSnapshotCalled)
|
||||
})
|
||||
|
||||
t.Run("forwards args and callback", func(t *testing.T) {
|
||||
cfg := &rest.Config{Host: "https://example.com"}
|
||||
fakeClients := iterator.Clients{}
|
||||
var capturedArgs iterator.Args
|
||||
var recorded [][]Range
|
||||
buildClients = func(config *rest.Config) (iterator.Clients, error) {
|
||||
assert.Same(t, cfg, config)
|
||||
return fakeClients, nil
|
||||
}
|
||||
getSnapshotMetadata = func(ctx context.Context, args iterator.Args) error {
|
||||
capturedArgs = args
|
||||
return args.Emitter.SnapshotMetadataIteratorRecord(1, iterator.IteratorMetadata{
|
||||
BlockMetadata: []*api.BlockMetadata{{ByteOffset: 11, SizeBytes: 22}},
|
||||
})
|
||||
}
|
||||
|
||||
service := &ServiceImpl{
|
||||
logger: logrus.New(),
|
||||
vsNamespace: "velero-ns",
|
||||
SAName: "sa-name",
|
||||
clientConfig: cfg,
|
||||
}
|
||||
|
||||
err := service.GetAllocatedBlocks(t.Context(), "snap-1", func(ranges []Range) error {
|
||||
recorded = append(recorded, ranges)
|
||||
return nil
|
||||
})
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, fakeClients, capturedArgs.Clients)
|
||||
assert.Equal(t, "snap-1", capturedArgs.SnapshotName)
|
||||
assert.Empty(t, capturedArgs.PrevSnapshotName)
|
||||
assert.Equal(t, "velero-ns", capturedArgs.Namespace)
|
||||
assert.Equal(t, "velero-ns", capturedArgs.SANamespace)
|
||||
assert.Equal(t, "sa-name", capturedArgs.SAName)
|
||||
assert.Equal(t, iterator.DefaultTokenExpirySeconds, capturedArgs.TokenExpirySecs)
|
||||
assert.Zero(t, capturedArgs.MaxResults)
|
||||
assert.Equal(t, [][]Range{{{Offset: 11, Length: 22}}}, recorded)
|
||||
})
|
||||
}
|
||||
|
||||
func TestServiceImplGetChangedBlocks(t *testing.T) {
|
||||
originalBuildClients := buildClients
|
||||
originalGetSnapshotMetadata := getSnapshotMetadata
|
||||
t.Cleanup(func() {
|
||||
buildClients = originalBuildClients
|
||||
getSnapshotMetadata = originalGetSnapshotMetadata
|
||||
})
|
||||
|
||||
buildClients = func(config *rest.Config) (iterator.Clients, error) {
|
||||
return iterator.Clients{}, nil
|
||||
}
|
||||
|
||||
t.Run("sets previous snapshot name", func(t *testing.T) {
|
||||
var capturedArgs iterator.Args
|
||||
getSnapshotMetadata = func(ctx context.Context, args iterator.Args) error {
|
||||
capturedArgs = args
|
||||
return nil
|
||||
}
|
||||
|
||||
service := &ServiceImpl{
|
||||
logger: logrus.New(),
|
||||
vsNamespace: "velero-ns",
|
||||
SAName: "sa-name",
|
||||
clientConfig: &rest.Config{Host: "https://example.com"},
|
||||
}
|
||||
|
||||
err := service.GetChangedBlocks(t.Context(), "snap-2", "snap-1", func([]Range) error { return nil })
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "snap-2", capturedArgs.SnapshotName)
|
||||
assert.Equal(t, "snap-1", capturedArgs.PrevSnapshotName)
|
||||
assert.Equal(t, "velero-ns", capturedArgs.Namespace)
|
||||
assert.Equal(t, iterator.DefaultTokenExpirySeconds, capturedArgs.TokenExpirySecs)
|
||||
assert.Zero(t, capturedArgs.MaxResults)
|
||||
})
|
||||
|
||||
t.Run("returns snapshot metadata error", func(t *testing.T) {
|
||||
wantErr := errors.New("metadata failed")
|
||||
getSnapshotMetadata = func(ctx context.Context, args iterator.Args) error {
|
||||
return wantErr
|
||||
}
|
||||
|
||||
service := &ServiceImpl{
|
||||
logger: logrus.New(),
|
||||
clientConfig: &rest.Config{Host: "https://example.com"},
|
||||
}
|
||||
|
||||
err := service.GetChangedBlocks(t.Context(), "snap-2", "snap-1", func([]Range) error { return nil })
|
||||
|
||||
require.ErrorIs(t, err, wantErr)
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
// Code generated by mockery; DO NOT EDIT.
|
||||
// github.com/vektra/mockery
|
||||
// template: testify
|
||||
|
||||
package mocks
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
mock "github.com/stretchr/testify/mock"
|
||||
"github.com/vmware-tanzu/velero/pkg/cbtservice"
|
||||
)
|
||||
|
||||
// NewService creates a new instance of Service. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
|
||||
// The first argument is typically a *testing.T value.
|
||||
func NewService(t interface {
|
||||
mock.TestingT
|
||||
Cleanup(func())
|
||||
}) *Service {
|
||||
mock := &Service{}
|
||||
mock.Mock.Test(t)
|
||||
|
||||
t.Cleanup(func() { mock.AssertExpectations(t) })
|
||||
|
||||
return mock
|
||||
}
|
||||
|
||||
// Service is an autogenerated mock type for the Service type
|
||||
type Service struct {
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
type Service_Expecter struct {
|
||||
mock *mock.Mock
|
||||
}
|
||||
|
||||
func (_m *Service) EXPECT() *Service_Expecter {
|
||||
return &Service_Expecter{mock: &_m.Mock}
|
||||
}
|
||||
|
||||
// GetAllocatedBlocks provides a mock function for the type Service
|
||||
func (_mock *Service) GetAllocatedBlocks(ctx context.Context, snapshot string, record func([]cbtservice.Range) error) error {
|
||||
ret := _mock.Called(ctx, snapshot, record)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for GetAllocatedBlocks")
|
||||
}
|
||||
|
||||
var r0 error
|
||||
if returnFunc, ok := ret.Get(0).(func(context.Context, string, func([]cbtservice.Range) error) error); ok {
|
||||
r0 = returnFunc(ctx, snapshot, record)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
return r0
|
||||
}
|
||||
|
||||
// Service_GetAllocatedBlocks_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetAllocatedBlocks'
|
||||
type Service_GetAllocatedBlocks_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// GetAllocatedBlocks is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - snapshot string
|
||||
// - record func([]cbtservice.Range) error
|
||||
func (_e *Service_Expecter) GetAllocatedBlocks(ctx interface{}, snapshot interface{}, record interface{}) *Service_GetAllocatedBlocks_Call {
|
||||
return &Service_GetAllocatedBlocks_Call{Call: _e.mock.On("GetAllocatedBlocks", ctx, snapshot, record)}
|
||||
}
|
||||
|
||||
func (_c *Service_GetAllocatedBlocks_Call) Run(run func(ctx context.Context, snapshot string, record func([]cbtservice.Range) error)) *Service_GetAllocatedBlocks_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
var arg0 context.Context
|
||||
if args[0] != nil {
|
||||
arg0 = args[0].(context.Context)
|
||||
}
|
||||
var arg1 string
|
||||
if args[1] != nil {
|
||||
arg1 = args[1].(string)
|
||||
}
|
||||
var arg2 func([]cbtservice.Range) error
|
||||
if args[2] != nil {
|
||||
arg2 = args[2].(func([]cbtservice.Range) error)
|
||||
}
|
||||
run(
|
||||
arg0,
|
||||
arg1,
|
||||
arg2,
|
||||
)
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Service_GetAllocatedBlocks_Call) Return(err error) *Service_GetAllocatedBlocks_Call {
|
||||
_c.Call.Return(err)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Service_GetAllocatedBlocks_Call) RunAndReturn(run func(ctx context.Context, snapshot string, record func([]cbtservice.Range) error) error) *Service_GetAllocatedBlocks_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// GetChangedBlocks provides a mock function for the type Service
|
||||
func (_mock *Service) GetChangedBlocks(ctx context.Context, snapshot string, changeID string, record func([]cbtservice.Range) error) error {
|
||||
ret := _mock.Called(ctx, snapshot, changeID, record)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for GetChangedBlocks")
|
||||
}
|
||||
|
||||
var r0 error
|
||||
if returnFunc, ok := ret.Get(0).(func(context.Context, string, string, func([]cbtservice.Range) error) error); ok {
|
||||
r0 = returnFunc(ctx, snapshot, changeID, record)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
return r0
|
||||
}
|
||||
|
||||
// Service_GetChangedBlocks_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetChangedBlocks'
|
||||
type Service_GetChangedBlocks_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// GetChangedBlocks is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - snapshot string
|
||||
// - changeID string
|
||||
// - record func([]cbtservice.Range) error
|
||||
func (_e *Service_Expecter) GetChangedBlocks(ctx interface{}, snapshot interface{}, changeID interface{}, record interface{}) *Service_GetChangedBlocks_Call {
|
||||
return &Service_GetChangedBlocks_Call{Call: _e.mock.On("GetChangedBlocks", ctx, snapshot, changeID, record)}
|
||||
}
|
||||
|
||||
func (_c *Service_GetChangedBlocks_Call) Run(run func(ctx context.Context, snapshot string, changeID string, record func([]cbtservice.Range) error)) *Service_GetChangedBlocks_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
var arg0 context.Context
|
||||
if args[0] != nil {
|
||||
arg0 = args[0].(context.Context)
|
||||
}
|
||||
var arg1 string
|
||||
if args[1] != nil {
|
||||
arg1 = args[1].(string)
|
||||
}
|
||||
var arg2 string
|
||||
if args[2] != nil {
|
||||
arg2 = args[2].(string)
|
||||
}
|
||||
var arg3 func([]cbtservice.Range) error
|
||||
if args[3] != nil {
|
||||
arg3 = args[3].(func([]cbtservice.Range) error)
|
||||
}
|
||||
run(
|
||||
arg0,
|
||||
arg1,
|
||||
arg2,
|
||||
arg3,
|
||||
)
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Service_GetChangedBlocks_Call) Return(err error) *Service_GetChangedBlocks_Call {
|
||||
_c.Call.Return(err)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Service_GetChangedBlocks_Call) RunAndReturn(run func(ctx context.Context, snapshot string, changeID string, record func([]cbtservice.Range) error) error) *Service_GetChangedBlocks_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
@@ -16,12 +16,14 @@ limitations under the License.
|
||||
|
||||
package cbtservice
|
||||
|
||||
import "context"
|
||||
import (
|
||||
"context"
|
||||
)
|
||||
|
||||
// Range defines the range of a change
|
||||
type Range struct {
|
||||
Offset int64
|
||||
Length int64
|
||||
Offset uint64
|
||||
Length uint64
|
||||
}
|
||||
|
||||
// SourceInfo is the information provided to the uploader, the uploader calls CBT service with this information
|
||||
|
||||
@@ -32,6 +32,7 @@ import (
|
||||
|
||||
"github.com/vmware-tanzu/velero/internal/credentials"
|
||||
"github.com/vmware-tanzu/velero/pkg/buildinfo"
|
||||
"github.com/vmware-tanzu/velero/pkg/cbtservice"
|
||||
"github.com/vmware-tanzu/velero/pkg/client"
|
||||
"github.com/vmware-tanzu/velero/pkg/cmd/util/signals"
|
||||
"github.com/vmware-tanzu/velero/pkg/datamover"
|
||||
@@ -56,6 +57,7 @@ type dataMoverBackupConfig struct {
|
||||
volumeMode string
|
||||
duName string
|
||||
resourceTimeout time.Duration
|
||||
cbtSAName string
|
||||
}
|
||||
|
||||
func NewBackupCommand(f client.Factory) *cobra.Command {
|
||||
@@ -92,6 +94,7 @@ func NewBackupCommand(f client.Factory) *cobra.Command {
|
||||
command.Flags().StringVar(&config.volumeMode, "volume-mode", config.volumeMode, "The mode of the volume to be backed up")
|
||||
command.Flags().StringVar(&config.duName, "data-upload", config.duName, "The data upload name")
|
||||
command.Flags().DurationVar(&config.resourceTimeout, "resource-timeout", config.resourceTimeout, "How long to wait for resource processes which are not covered by other specific timeout parameters.")
|
||||
command.Flags().StringVar(&config.cbtSAName, "cbt-sa-name", config.cbtSAName, "The name of the service account used by CSI's CBT service")
|
||||
|
||||
_ = command.MarkFlagRequired("volume-path")
|
||||
_ = command.MarkFlagRequired("volume-mode")
|
||||
@@ -112,6 +115,7 @@ type dataMoverBackup struct {
|
||||
config dataMoverBackupConfig
|
||||
kubeClient kubernetes.Interface
|
||||
dataPathMgr *datapath.Manager
|
||||
cbtService cbtservice.Service
|
||||
}
|
||||
|
||||
func newdataMoverBackup(logger logrus.FieldLogger, factory client.Factory, config dataMoverBackupConfig) (*dataMoverBackup, error) {
|
||||
@@ -197,6 +201,12 @@ func newdataMoverBackup(logger logrus.FieldLogger, factory client.Factory, confi
|
||||
config: config,
|
||||
namespace: factory.Namespace(),
|
||||
nodeName: nodeName,
|
||||
cbtService: cbtservice.NewService(
|
||||
logger,
|
||||
factory.Namespace(),
|
||||
config.cbtSAName,
|
||||
clientConfig,
|
||||
),
|
||||
}
|
||||
|
||||
s.kubeClient, err = factory.KubeClient()
|
||||
|
||||
@@ -22,14 +22,33 @@ import (
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/mock"
|
||||
"github.com/stretchr/testify/require"
|
||||
ctlclient "sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
"github.com/vmware-tanzu/velero/internal/credentials"
|
||||
"github.com/vmware-tanzu/velero/pkg/cbtservice"
|
||||
factorymocks "github.com/vmware-tanzu/velero/pkg/client/mocks"
|
||||
cacheMock "github.com/vmware-tanzu/velero/pkg/cmd/cli/datamover/mocks"
|
||||
velerotest "github.com/vmware-tanzu/velero/pkg/test"
|
||||
"github.com/vmware-tanzu/velero/pkg/util/filesystem"
|
||||
)
|
||||
|
||||
func TestNewBackupCommandCBTSANameFlag(t *testing.T) {
|
||||
f := factorymocks.NewFactory(t)
|
||||
cmd := NewBackupCommand(f)
|
||||
|
||||
cbtSAFlag := cmd.Flags().Lookup("cbt-sa-name")
|
||||
require.NotNil(t, cbtSAFlag)
|
||||
assert.Empty(t, cbtSAFlag.DefValue)
|
||||
|
||||
err := cmd.Flags().Parse([]string{"--cbt-sa-name", "velero-cbt-sa"})
|
||||
require.NoError(t, err)
|
||||
|
||||
flagValue, err := cmd.Flags().GetString("cbt-sa-name")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "velero-cbt-sa", flagValue)
|
||||
}
|
||||
|
||||
func fakeCreateDataPathServiceWithErr(_ *dataMoverBackup) (dataPathService, error) {
|
||||
return nil, errors.New("fake-create-data-path-error")
|
||||
}
|
||||
@@ -129,6 +148,7 @@ func TestRunDataPath(t *testing.T) {
|
||||
config: dataMoverBackupConfig{
|
||||
duName: test.duName,
|
||||
},
|
||||
cbtService: cbtservice.Service(nil),
|
||||
}
|
||||
|
||||
s.runDataPath()
|
||||
|
||||
@@ -385,6 +385,12 @@ func (s *nodeAgentServer) run() {
|
||||
s.logger.Info("Backup repo config is not provided, using default values for cache volume configs")
|
||||
}
|
||||
|
||||
var csiSnapshotMetadataServiceConfigs *velerotypes.CSISnapshotMetadataService
|
||||
if s.dataPathConfigs != nil && s.dataPathConfigs.CSISnapshotMetadataServiceConfigs != nil {
|
||||
csiSnapshotMetadataServiceConfigs = s.dataPathConfigs.CSISnapshotMetadataServiceConfigs
|
||||
s.logger.Infof("Using CSI snapshot metadata service config %v", s.dataPathConfigs.CSISnapshotMetadataServiceConfigs)
|
||||
}
|
||||
|
||||
pvbReconciler := controller.NewPodVolumeBackupReconciler(
|
||||
s.mgr.GetClient(),
|
||||
s.mgr,
|
||||
@@ -447,6 +453,7 @@ func (s *nodeAgentServer) run() {
|
||||
dataMovePriorityClass,
|
||||
podLabels,
|
||||
podAnnotations,
|
||||
csiSnapshotMetadataServiceConfigs,
|
||||
)
|
||||
if err := dataUploadReconciler.SetupWithManager(s.mgr); err != nil {
|
||||
s.logger.WithError(err).Fatal("Unable to create the data upload controller")
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -30,6 +31,7 @@ import (
|
||||
|
||||
snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/fatih/color"
|
||||
kbclient "sigs.k8s.io/controller-runtime/pkg/client"
|
||||
@@ -40,6 +42,7 @@ import (
|
||||
"github.com/vmware-tanzu/velero/pkg/cmd/util/downloadrequest"
|
||||
"github.com/vmware-tanzu/velero/pkg/itemoperation"
|
||||
|
||||
"github.com/vmware-tanzu/velero/internal/resourcepolicies"
|
||||
"github.com/vmware-tanzu/velero/internal/volume"
|
||||
"github.com/vmware-tanzu/velero/pkg/util/collections"
|
||||
"github.com/vmware-tanzu/velero/pkg/util/results"
|
||||
@@ -91,6 +94,9 @@ func DescribeBackup(
|
||||
if backup.Spec.ResourcePolicy != nil {
|
||||
d.Println()
|
||||
DescribeResourcePolicies(d, backup.Spec.ResourcePolicy)
|
||||
|
||||
// Display fine-grained filter policies if they exist
|
||||
DescribeFineGrainedFilterPolicies(ctx, kbClient, d, backup)
|
||||
}
|
||||
|
||||
if backup.Spec.UploaderConfig != nil && backup.Spec.UploaderConfig.ParallelFilesUpload > 0 {
|
||||
@@ -130,6 +136,119 @@ func DescribeResourcePolicies(d *Describer, resPolicies *corev1api.TypedLocalObj
|
||||
d.Printf("\tName:\t%s\n", resPolicies.Name)
|
||||
}
|
||||
|
||||
// DescribeFineGrainedFilterPolicies describes cluster-scoped and namespace-scoped filter policies if present
|
||||
func DescribeFineGrainedFilterPolicies(ctx context.Context, kbClient kbclient.Client, d *Describer, backup *velerov1api.Backup) {
|
||||
if backup.Spec.ResourcePolicy == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Create a discard logger for the resource policies function since this is CLI output context
|
||||
discardLogger := logrus.New()
|
||||
discardLogger.Out = io.Discard
|
||||
|
||||
resourcePolicies, err := resourcepolicies.GetResourcePoliciesFromBackup(*backup, kbClient, discardLogger)
|
||||
if err != nil {
|
||||
// Don't fail the describe if we can't read policies, just skip
|
||||
return
|
||||
}
|
||||
|
||||
if resourcePolicies == nil {
|
||||
return
|
||||
}
|
||||
|
||||
clusterScopedFilterPolicy := resourcePolicies.GetClusterScopedFilterPolicy()
|
||||
if clusterScopedFilterPolicy != nil {
|
||||
d.Printf("\nCluster Scoped Filter Policy:\n")
|
||||
d.Printf(" Resource Filters:\n")
|
||||
for _, rf := range clusterScopedFilterPolicy.ResourceFilters {
|
||||
kindsStr := strings.Join(rf.Kinds, ", ")
|
||||
d.Printf(" %s:\n", kindsStr)
|
||||
|
||||
// Label selector
|
||||
if len(rf.LabelSelector) > 0 {
|
||||
selectorStr := formatLabelMap(rf.LabelSelector)
|
||||
d.Printf(" Label selector: %s\n", selectorStr)
|
||||
} else if len(rf.OrLabelSelectors) > 0 {
|
||||
var orStrs []string
|
||||
for _, ols := range rf.OrLabelSelectors {
|
||||
orStrs = append(orStrs, formatLabelMap(ols))
|
||||
}
|
||||
d.Printf(" OR label selectors: [%s]\n", strings.Join(orStrs, ", "))
|
||||
} else {
|
||||
d.Printf(" Label selector: <none>\n")
|
||||
}
|
||||
|
||||
// Name patterns
|
||||
if len(rf.Names) > 0 {
|
||||
d.Printf(" Included names: [%s]\n", strings.Join(rf.Names, ", "))
|
||||
} else {
|
||||
d.Printf(" Included names: <none>\n")
|
||||
}
|
||||
|
||||
if len(rf.ExcludedNames) > 0 {
|
||||
d.Printf(" Excluded names: [%s]\n", strings.Join(rf.ExcludedNames, ", "))
|
||||
} else {
|
||||
d.Printf(" Excluded names: <none>\n")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
nfPolicies := resourcePolicies.GetNamespacedFilterPolicies()
|
||||
if len(nfPolicies) > 0 {
|
||||
d.Printf("\nNamespace-Scoped Filter Policies:\n")
|
||||
for _, policy := range nfPolicies {
|
||||
for _, ns := range policy.Namespaces {
|
||||
d.Printf(" %s:\n", ns)
|
||||
d.Printf(" Resource Filters:\n")
|
||||
for _, rf := range policy.ResourceFilters {
|
||||
var kindsStr string
|
||||
if rf.IsCatchAll() {
|
||||
kindsStr = "<catch-all> (all other kinds)"
|
||||
} else {
|
||||
kindsStr = strings.Join(rf.Kinds, ", ")
|
||||
}
|
||||
d.Printf(" %s:\n", kindsStr)
|
||||
|
||||
// Label selector
|
||||
if len(rf.LabelSelector) > 0 {
|
||||
selectorStr := formatLabelMap(rf.LabelSelector)
|
||||
d.Printf(" Label selector: %s\n", selectorStr)
|
||||
} else if len(rf.OrLabelSelectors) > 0 {
|
||||
var orStrs []string
|
||||
for _, ols := range rf.OrLabelSelectors {
|
||||
orStrs = append(orStrs, formatLabelMap(ols))
|
||||
}
|
||||
d.Printf(" OR label selectors: [%s]\n", strings.Join(orStrs, ", "))
|
||||
} else {
|
||||
d.Printf(" Label selector: <none>\n")
|
||||
}
|
||||
|
||||
// Name patterns
|
||||
if len(rf.Names) > 0 {
|
||||
d.Printf(" Included names: [%s]\n", strings.Join(rf.Names, ", "))
|
||||
} else {
|
||||
d.Printf(" Included names: <none>\n")
|
||||
}
|
||||
|
||||
if len(rf.ExcludedNames) > 0 {
|
||||
d.Printf(" Excluded names: [%s]\n", strings.Join(rf.ExcludedNames, ", "))
|
||||
} else {
|
||||
d.Printf(" Excluded names: <none>\n")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func formatLabelMap(labelMap map[string]string) string {
|
||||
var pairs []string
|
||||
for k, v := range labelMap {
|
||||
pairs = append(pairs, fmt.Sprintf("%s=%s", k, v))
|
||||
}
|
||||
return strings.Join(pairs, ",")
|
||||
}
|
||||
|
||||
// DescribeUploaderConfigForBackup describes uploader config in human-readable format
|
||||
func DescribeUploaderConfigForBackup(d *Describer, spec velerov1api.BackupSpec) {
|
||||
d.Printf("Uploader config:\n")
|
||||
|
||||
@@ -18,6 +18,7 @@ package output
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"testing"
|
||||
"text/tabwriter"
|
||||
"time"
|
||||
@@ -25,6 +26,8 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
corev1api "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
||||
|
||||
"github.com/vmware-tanzu/velero/internal/volume"
|
||||
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
|
||||
@@ -866,3 +869,85 @@ func TestDescribeBackupItemOperation(t *testing.T) {
|
||||
d.out.Flush()
|
||||
assert.Equal(t, expected, d.buf.String())
|
||||
}
|
||||
|
||||
func TestDescribeFineGrainedFilterPolicies(t *testing.T) {
|
||||
yamlData := `
|
||||
version: v1
|
||||
clusterScopedFilterPolicy:
|
||||
resourceFilters:
|
||||
- kinds: ["StorageClass"]
|
||||
labelSelector: {"app": "velero"}
|
||||
- kinds: ["ClusterRole"]
|
||||
orLabelSelectors:
|
||||
- {"app": "velero"}
|
||||
- {"app": "test"}
|
||||
names: ["role1"]
|
||||
excludedNames: ["role2"]
|
||||
namespacedFilterPolicies:
|
||||
- namespaces: ["ns1", "ns2"]
|
||||
resourceFilters:
|
||||
- kinds: ["Pod", "ConfigMap"]
|
||||
labelSelector: {"app": "velero"}
|
||||
- kinds: ["*"]
|
||||
`
|
||||
cm := &corev1api.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "test-policy",
|
||||
Namespace: "velero",
|
||||
},
|
||||
Data: map[string]string{
|
||||
"policy.yaml": yamlData,
|
||||
},
|
||||
}
|
||||
|
||||
client := fake.NewClientBuilder().WithRuntimeObjects(cm).Build()
|
||||
|
||||
backup := builder.ForBackup("velero", "test-backup").
|
||||
ResourcePolicies("test-policy").Result()
|
||||
|
||||
d := &Describer{
|
||||
Prefix: "",
|
||||
out: &tabwriter.Writer{},
|
||||
buf: &bytes.Buffer{},
|
||||
}
|
||||
d.out.Init(d.buf, 0, 8, 2, ' ', 0)
|
||||
|
||||
DescribeFineGrainedFilterPolicies(context.Background(), client, d, backup)
|
||||
d.out.Flush()
|
||||
|
||||
expected := `
|
||||
Cluster Scoped Filter Policy:
|
||||
Resource Filters:
|
||||
StorageClass:
|
||||
Label selector: app=velero
|
||||
Included names: <none>
|
||||
Excluded names: <none>
|
||||
ClusterRole:
|
||||
OR label selectors: [app=velero, app=test]
|
||||
Included names: [role1]
|
||||
Excluded names: [role2]
|
||||
|
||||
Namespace-Scoped Filter Policies:
|
||||
ns1:
|
||||
Resource Filters:
|
||||
Pod, ConfigMap:
|
||||
Label selector: app=velero
|
||||
Included names: <none>
|
||||
Excluded names: <none>
|
||||
<catch-all> (all other kinds):
|
||||
Label selector: <none>
|
||||
Included names: <none>
|
||||
Excluded names: <none>
|
||||
ns2:
|
||||
Resource Filters:
|
||||
Pod, ConfigMap:
|
||||
Label selector: app=velero
|
||||
Included names: <none>
|
||||
Excluded names: <none>
|
||||
<catch-all> (all other kinds):
|
||||
Label selector: <none>
|
||||
Included names: <none>
|
||||
Excluded names: <none>
|
||||
`
|
||||
assert.Equal(t, expected, d.buf.String())
|
||||
}
|
||||
|
||||
@@ -21,13 +21,16 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
corev1api "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
kbclient "sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
"github.com/vmware-tanzu/velero/internal/resourcepolicies"
|
||||
"github.com/vmware-tanzu/velero/internal/volume"
|
||||
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
|
||||
"github.com/vmware-tanzu/velero/pkg/cmd/util/cacert"
|
||||
@@ -54,6 +57,7 @@ func DescribeBackupInSF(
|
||||
|
||||
if backup.Spec.ResourcePolicy != nil {
|
||||
DescribeResourcePoliciesInSF(d, backup.Spec.ResourcePolicy)
|
||||
DescribeFineGrainedFilterPoliciesInSF(ctx, kbClient, d, backup)
|
||||
}
|
||||
|
||||
status := backup.Status
|
||||
@@ -222,6 +226,88 @@ func DescribeBackupSpecInSF(d *StructuredDescriber, spec velerov1api.BackupSpec)
|
||||
d.Describe("spec", backupSpecInfo)
|
||||
}
|
||||
|
||||
// DescribeFineGrainedFilterPoliciesInSF adds the clusterScopedFilterPolicy
|
||||
// and namespacedFilterPolicies sections to the structured describer output when present
|
||||
// in the ResourcePolicy ConfigMap referenced by the backup.
|
||||
func DescribeFineGrainedFilterPoliciesInSF(ctx context.Context, kbClient kbclient.Client, d *StructuredDescriber, backup *velerov1api.Backup) {
|
||||
if backup.Spec.ResourcePolicy == nil {
|
||||
return
|
||||
}
|
||||
|
||||
discardLogger := logrus.New()
|
||||
discardLogger.Out = io.Discard
|
||||
|
||||
resPolicies, err := resourcepolicies.GetResourcePoliciesFromBackup(*backup, kbClient, discardLogger)
|
||||
if err != nil || resPolicies == nil {
|
||||
return
|
||||
}
|
||||
|
||||
clusterScopedFilterPolicy := resPolicies.GetClusterScopedFilterPolicy()
|
||||
if clusterScopedFilterPolicy != nil {
|
||||
var clusterScopedFilters []map[string]any
|
||||
for _, rf := range clusterScopedFilterPolicy.ResourceFilters {
|
||||
entry := map[string]any{
|
||||
"kinds": rf.Kinds,
|
||||
}
|
||||
if len(rf.LabelSelector) > 0 {
|
||||
entry["labelSelector"] = rf.LabelSelector
|
||||
}
|
||||
if len(rf.OrLabelSelectors) > 0 {
|
||||
entry["orLabelSelectors"] = rf.OrLabelSelectors
|
||||
}
|
||||
if len(rf.Names) > 0 {
|
||||
entry["names"] = rf.Names
|
||||
}
|
||||
if len(rf.ExcludedNames) > 0 {
|
||||
entry["excludedNames"] = rf.ExcludedNames
|
||||
}
|
||||
clusterScopedFilters = append(clusterScopedFilters, entry)
|
||||
}
|
||||
d.Describe("clusterScopedFilterPolicy", map[string]any{
|
||||
"resourceFilters": clusterScopedFilters,
|
||||
})
|
||||
}
|
||||
|
||||
nfPolicies := resPolicies.GetNamespacedFilterPolicies()
|
||||
if len(nfPolicies) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
var structuredPolicies []map[string]any
|
||||
for _, policy := range nfPolicies {
|
||||
for _, ns := range policy.Namespaces {
|
||||
var rfEntries []map[string]any
|
||||
for _, rf := range policy.ResourceFilters {
|
||||
entry := map[string]any{}
|
||||
if rf.IsCatchAll() {
|
||||
entry["kinds"] = []string{}
|
||||
entry["isCatchAll"] = true
|
||||
} else {
|
||||
entry["kinds"] = rf.Kinds
|
||||
}
|
||||
if len(rf.LabelSelector) > 0 {
|
||||
entry["labelSelector"] = rf.LabelSelector
|
||||
}
|
||||
if len(rf.OrLabelSelectors) > 0 {
|
||||
entry["orLabelSelectors"] = rf.OrLabelSelectors
|
||||
}
|
||||
if len(rf.Names) > 0 {
|
||||
entry["names"] = rf.Names
|
||||
}
|
||||
if len(rf.ExcludedNames) > 0 {
|
||||
entry["excludedNames"] = rf.ExcludedNames
|
||||
}
|
||||
rfEntries = append(rfEntries, entry)
|
||||
}
|
||||
structuredPolicies = append(structuredPolicies, map[string]any{
|
||||
"namespace": ns,
|
||||
"resourceFilters": rfEntries,
|
||||
})
|
||||
}
|
||||
}
|
||||
d.Describe("namespacedFilterPolicies", structuredPolicies)
|
||||
}
|
||||
|
||||
// DescribeBackupStatusInSF describes a backup status in structured format.
|
||||
func DescribeBackupStatusInSF(ctx context.Context, kbClient kbclient.Client, d *StructuredDescriber, backup *velerov1api.Backup, details bool,
|
||||
insecureSkipTLSVerify bool, caCertPath string, podVolumeBackups []velerov1api.PodVolumeBackup) {
|
||||
|
||||
@@ -17,6 +17,7 @@ limitations under the License.
|
||||
package output
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -24,6 +25,8 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
corev1api "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
||||
|
||||
"github.com/vmware-tanzu/velero/internal/volume"
|
||||
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
|
||||
@@ -707,3 +710,96 @@ func TestDescribeDeleteBackupRequestsInSF(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDescribeFineGrainedFilterPoliciesInSF(t *testing.T) {
|
||||
yamlData := `
|
||||
version: v1
|
||||
clusterScopedFilterPolicy:
|
||||
resourceFilters:
|
||||
- kinds: ["StorageClass"]
|
||||
labelSelector: {"app": "velero"}
|
||||
- kinds: ["ClusterRole"]
|
||||
orLabelSelectors:
|
||||
- {"app": "velero"}
|
||||
- {"app": "test"}
|
||||
names: ["role1"]
|
||||
excludedNames: ["role2"]
|
||||
namespacedFilterPolicies:
|
||||
- namespaces: ["ns1", "ns2"]
|
||||
resourceFilters:
|
||||
- kinds: ["Pod", "ConfigMap"]
|
||||
labelSelector: {"app": "velero"}
|
||||
- kinds: ["*"]
|
||||
`
|
||||
cm := &corev1api.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "test-policy",
|
||||
Namespace: "velero",
|
||||
},
|
||||
Data: map[string]string{
|
||||
"policy.yaml": yamlData,
|
||||
},
|
||||
}
|
||||
|
||||
client := fake.NewClientBuilder().WithRuntimeObjects(cm).Build()
|
||||
|
||||
backup := builder.ForBackup("velero", "test-backup").
|
||||
ResourcePolicies("test-policy").Result()
|
||||
|
||||
sd := &StructuredDescriber{
|
||||
output: make(map[string]any),
|
||||
format: "",
|
||||
}
|
||||
|
||||
DescribeFineGrainedFilterPoliciesInSF(context.Background(), client, sd, backup)
|
||||
|
||||
expect := map[string]any{
|
||||
"clusterScopedFilterPolicy": map[string]any{
|
||||
"resourceFilters": []map[string]any{
|
||||
{
|
||||
"kinds": []string{"StorageClass"},
|
||||
"labelSelector": map[string]string{"app": "velero"},
|
||||
},
|
||||
{
|
||||
"kinds": []string{"ClusterRole"},
|
||||
"orLabelSelectors": []map[string]string{
|
||||
{"app": "velero"},
|
||||
{"app": "test"},
|
||||
},
|
||||
"names": []string{"role1"},
|
||||
"excludedNames": []string{"role2"},
|
||||
},
|
||||
},
|
||||
},
|
||||
"namespacedFilterPolicies": []map[string]any{
|
||||
{
|
||||
"namespace": "ns1",
|
||||
"resourceFilters": []map[string]any{
|
||||
{
|
||||
"kinds": []string{"Pod", "ConfigMap"},
|
||||
"labelSelector": map[string]string{"app": "velero"},
|
||||
},
|
||||
{
|
||||
"kinds": []string{},
|
||||
"isCatchAll": true,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"namespace": "ns2",
|
||||
"resourceFilters": []map[string]any{
|
||||
{
|
||||
"kinds": []string{"Pod", "ConfigMap"},
|
||||
"labelSelector": map[string]string{"app": "velero"},
|
||||
},
|
||||
{
|
||||
"kinds": []string{},
|
||||
"isCatchAll": true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
assert.True(t, reflect.DeepEqual(sd.output, expect))
|
||||
}
|
||||
|
||||
@@ -595,6 +595,13 @@ func (b *backupReconciler) prepareBackupRequest(ctx context.Context, backup *vel
|
||||
request.Status.ValidationErrors = append(request.Status.ValidationErrors, "include-resources, exclude-resources and include-cluster-resources are old filter parameters.\n"+
|
||||
"They cannot be used with include-exclude policies.")
|
||||
}
|
||||
// namespacedFilterPolicies and clusterScopedFilterPolicy incompatible with old-style filters
|
||||
if resourcePolicies != nil &&
|
||||
(len(resourcePolicies.GetNamespacedFilterPolicies()) > 0 || resourcePolicies.GetClusterScopedFilterPolicy() != nil) &&
|
||||
collections.UseOldResourceFilters(request.Spec) {
|
||||
request.Status.ValidationErrors = append(request.Status.ValidationErrors, "include-resources, exclude-resources and include-cluster-resources are old filter parameters.\n"+
|
||||
"They cannot be used with namespace-scoped or fine-grained global filter policies.")
|
||||
}
|
||||
request.ResPolicies = resourcePolicies
|
||||
return request
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"reflect"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -34,6 +35,7 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/mock"
|
||||
"github.com/stretchr/testify/require"
|
||||
corev1api "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
@@ -2020,3 +2022,237 @@ func TestPatchResourceWorksWithStatus(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestPrepareBackupRequest_NamespacedFilterPoliciesIncompatibleWithOldFilters verifies
|
||||
// that a backup referencing a ResourcePolicy ConfigMap with namespacedFilterPolicies
|
||||
// produces a validation error when old-style resource filters are also set on the spec.
|
||||
func TestPrepareBackupRequest_NamespacedFilterPoliciesIncompatibleWithOldFilters(t *testing.T) {
|
||||
formatFlag := logging.FormatText
|
||||
logger := logging.DefaultLogger(logrus.DebugLevel, formatFlag)
|
||||
|
||||
policyYAML := `version: v1
|
||||
namespacedFilterPolicies:
|
||||
- namespaces: ["production"]
|
||||
resourceFilters:
|
||||
- kinds: ["Deployment"]
|
||||
names: ["api-server"]
|
||||
`
|
||||
policyConfigMap := &corev1api.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "my-filter-policy",
|
||||
Namespace: velerov1api.DefaultNamespace,
|
||||
},
|
||||
Data: map[string]string{"policy": policyYAML},
|
||||
}
|
||||
|
||||
backup := defaultBackup().IncludedResources("deployments").Result()
|
||||
backup.Spec.ResourcePolicy = &corev1api.TypedLocalObjectReference{
|
||||
Kind: "configmap",
|
||||
Name: "my-filter-policy",
|
||||
}
|
||||
|
||||
fakeClient := velerotest.NewFakeControllerRuntimeClient(t, policyConfigMap)
|
||||
|
||||
apiServer := velerotest.NewAPIServer(t)
|
||||
discoveryHelper, err := discovery.NewHelper(apiServer.DiscoveryClient, logger)
|
||||
require.NoError(t, err)
|
||||
|
||||
c := &backupReconciler{
|
||||
logger: logger,
|
||||
discoveryHelper: discoveryHelper,
|
||||
kbClient: fakeClient,
|
||||
clock: &clock.RealClock{},
|
||||
formatFlag: formatFlag,
|
||||
}
|
||||
|
||||
res := c.prepareBackupRequest(ctx, backup, logger)
|
||||
|
||||
require.NotEmpty(t, res.Status.ValidationErrors)
|
||||
|
||||
hasTargetError := slices.ContainsFunc(res.Status.ValidationErrors, func(e string) bool {
|
||||
return strings.Contains(e, "namespace-scoped or fine-grained global filter policies")
|
||||
})
|
||||
|
||||
assert.True(t, hasTargetError, "expected validation error about namespacedFilterPolicies incompatibility with old-style filters, got: %v", res.Status.ValidationErrors)
|
||||
}
|
||||
|
||||
// TestPrepareBackupRequest_ClusterScopedFilterPolicyIncompatibleWithOldFilters verifies
|
||||
// that a backup referencing a ResourcePolicy ConfigMap with clusterScopedFilterPolicy
|
||||
// produces a validation error when old-style resource filters are also set on the spec.
|
||||
func TestPrepareBackupRequest_ClusterScopedFilterPolicyIncompatibleWithOldFilters(t *testing.T) {
|
||||
formatFlag := logging.FormatText
|
||||
logger := logging.DefaultLogger(logrus.DebugLevel, formatFlag)
|
||||
|
||||
policyYAML := `version: v1
|
||||
clusterScopedFilterPolicy:
|
||||
resourceFilters:
|
||||
- kinds: ["ClusterRole"]
|
||||
names: ["my-app-*"]
|
||||
`
|
||||
policyConfigMap := &corev1api.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "my-cluster-filter-policy",
|
||||
Namespace: velerov1api.DefaultNamespace,
|
||||
},
|
||||
Data: map[string]string{"policy": policyYAML},
|
||||
}
|
||||
|
||||
backup := defaultBackup().IncludedResources("clusterroles").Result()
|
||||
backup.Spec.ResourcePolicy = &corev1api.TypedLocalObjectReference{
|
||||
Kind: "configmap",
|
||||
Name: "my-cluster-filter-policy",
|
||||
}
|
||||
|
||||
fakeClient := velerotest.NewFakeControllerRuntimeClient(t, policyConfigMap)
|
||||
|
||||
apiServer := velerotest.NewAPIServer(t)
|
||||
discoveryHelper, err := discovery.NewHelper(apiServer.DiscoveryClient, logger)
|
||||
require.NoError(t, err)
|
||||
|
||||
c := &backupReconciler{
|
||||
logger: logger,
|
||||
discoveryHelper: discoveryHelper,
|
||||
kbClient: fakeClient,
|
||||
clock: &clock.RealClock{},
|
||||
formatFlag: formatFlag,
|
||||
}
|
||||
|
||||
res := c.prepareBackupRequest(ctx, backup, logger)
|
||||
|
||||
require.NotEmpty(t, res.Status.ValidationErrors)
|
||||
|
||||
hasClusterError := slices.ContainsFunc(res.Status.ValidationErrors, func(e string) bool {
|
||||
return strings.Contains(e, "namespace-scoped or fine-grained global filter policies")
|
||||
})
|
||||
|
||||
assert.True(t, hasClusterError, "expected validation error about clusterScopedFilterPolicy incompatibility with old-style filters, got: %v", res.Status.ValidationErrors)
|
||||
}
|
||||
|
||||
const (
|
||||
namespacedFilterPolicyYAML = `version: v1
|
||||
namespacedFilterPolicies:
|
||||
- namespaces: ["production"]
|
||||
resourceFilters:
|
||||
- kinds: ["Deployment"]
|
||||
names: ["api-server"]
|
||||
`
|
||||
clusterScopedFilterPolicyYAML = `version: v1
|
||||
clusterScopedFilterPolicy:
|
||||
resourceFilters:
|
||||
- kinds: ["ClusterRole"]
|
||||
names: ["my-app-*"]
|
||||
`
|
||||
bothFilterPoliciesYAML = `version: v1
|
||||
namespacedFilterPolicies:
|
||||
- namespaces: ["production"]
|
||||
resourceFilters:
|
||||
- kinds: ["Deployment"]
|
||||
names: ["api-server"]
|
||||
clusterScopedFilterPolicy:
|
||||
resourceFilters:
|
||||
- kinds: ["ClusterRole"]
|
||||
names: ["my-app-*"]
|
||||
`
|
||||
)
|
||||
|
||||
// TestPrepareBackupRequest_FilterPoliciesWithNewFilters verifies that backups referencing
|
||||
// a ResourcePolicy ConfigMap with namespacedFilterPolicies and/or clusterScopedFilterPolicy
|
||||
// succeed when old-style resource filters are not set on the spec.
|
||||
func TestPrepareBackupRequest_FilterPoliciesWithNewFilters(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
policyYAML string
|
||||
policyConfigMapName string
|
||||
backup *velerov1api.Backup
|
||||
expectNamespacedPolicies int
|
||||
expectClusterScopedPolicy bool
|
||||
}{
|
||||
{
|
||||
name: "namespacedFilterPolicies only",
|
||||
policyYAML: namespacedFilterPolicyYAML,
|
||||
policyConfigMapName: "my-filter-policy",
|
||||
backup: defaultBackup().StorageLocation("loc-1").Result(),
|
||||
expectNamespacedPolicies: 1,
|
||||
},
|
||||
{
|
||||
name: "clusterScopedFilterPolicy only",
|
||||
policyYAML: clusterScopedFilterPolicyYAML,
|
||||
policyConfigMapName: "my-cluster-filter-policy",
|
||||
backup: defaultBackup().StorageLocation("loc-1").Result(),
|
||||
expectClusterScopedPolicy: true,
|
||||
},
|
||||
{
|
||||
name: "both filter policies",
|
||||
policyYAML: bothFilterPoliciesYAML,
|
||||
policyConfigMapName: "my-combined-filter-policy",
|
||||
backup: defaultBackup().StorageLocation("loc-1").Result(),
|
||||
expectNamespacedPolicies: 1,
|
||||
expectClusterScopedPolicy: true,
|
||||
},
|
||||
{
|
||||
name: "with new-style spec filters",
|
||||
policyYAML: bothFilterPoliciesYAML,
|
||||
policyConfigMapName: "my-combined-filter-policy",
|
||||
backup: defaultBackup().
|
||||
StorageLocation("loc-1").
|
||||
IncludedNamespaceScopedResources("deployments").
|
||||
IncludedClusterScopedResources("clusterroles").
|
||||
Result(),
|
||||
expectNamespacedPolicies: 1,
|
||||
expectClusterScopedPolicy: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
formatFlag := logging.FormatText
|
||||
logger := logging.DefaultLogger(logrus.DebugLevel, formatFlag)
|
||||
|
||||
policyConfigMap := &corev1api.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: test.policyConfigMapName,
|
||||
Namespace: velerov1api.DefaultNamespace,
|
||||
},
|
||||
Data: map[string]string{"policy": test.policyYAML},
|
||||
}
|
||||
|
||||
test.backup.Spec.ResourcePolicy = &corev1api.TypedLocalObjectReference{
|
||||
Kind: "configmap",
|
||||
Name: test.policyConfigMapName,
|
||||
}
|
||||
|
||||
backupLocation := builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "loc-1").
|
||||
Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result()
|
||||
fakeClient := velerotest.NewFakeControllerRuntimeClient(t, backupLocation, policyConfigMap)
|
||||
|
||||
apiServer := velerotest.NewAPIServer(t)
|
||||
discoveryHelper, err := discovery.NewHelper(apiServer.DiscoveryClient, logger)
|
||||
require.NoError(t, err)
|
||||
|
||||
c := &backupReconciler{
|
||||
logger: logger,
|
||||
discoveryHelper: discoveryHelper,
|
||||
kbClient: fakeClient,
|
||||
clock: &clock.RealClock{},
|
||||
formatFlag: formatFlag,
|
||||
}
|
||||
|
||||
res := c.prepareBackupRequest(ctx, test.backup, logger)
|
||||
defer res.WorkerPool.Stop()
|
||||
|
||||
assert.Empty(t, res.Status.ValidationErrors)
|
||||
hasIncompatibilityError := slices.ContainsFunc(res.Status.ValidationErrors, func(e string) bool {
|
||||
return strings.Contains(e, "namespace-scoped or fine-grained global filter policies")
|
||||
})
|
||||
assert.False(t, hasIncompatibilityError)
|
||||
|
||||
require.NotNil(t, res.ResPolicies)
|
||||
assert.Len(t, res.ResPolicies.GetNamespacedFilterPolicies(), test.expectNamespacedPolicies)
|
||||
if test.expectClusterScopedPolicy {
|
||||
assert.NotNil(t, res.ResPolicies.GetClusterScopedFilterPolicy())
|
||||
} else {
|
||||
assert.Nil(t, res.ResPolicies.GetClusterScopedFilterPolicy())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -150,7 +150,7 @@ func (r *DataDownloadReconciler) Reconcile(ctx context.Context, req ctrl.Request
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
|
||||
if !datamover.IsBuiltInUploader(dd.Spec.DataMover) {
|
||||
if !datamover.IsBuiltInDataMover(dd.Spec.DataMover) {
|
||||
log.WithField("data mover", dd.Spec.DataMover).Info("it is not one built-in data mover which is not supported by Velero")
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
@@ -66,25 +66,26 @@ const (
|
||||
|
||||
// DataUploadReconciler reconciles a DataUpload object
|
||||
type DataUploadReconciler struct {
|
||||
client client.Client
|
||||
kubeClient kubernetes.Interface
|
||||
csiSnapshotClient snapshotter.SnapshotV1Interface
|
||||
mgr manager.Manager
|
||||
Clock clocks.WithTickerAndDelayedExecution
|
||||
nodeName string
|
||||
logger logrus.FieldLogger
|
||||
snapshotExposerList map[velerov2alpha1api.SnapshotType]exposer.SnapshotExposer
|
||||
dataPathMgr *datapath.Manager
|
||||
vgdpCounter *exposer.VgdpCounter
|
||||
loadAffinity []*kube.LoadAffinity
|
||||
backupPVCConfig map[string]velerotypes.BackupPVC
|
||||
podResources corev1api.ResourceRequirements
|
||||
preparingTimeout time.Duration
|
||||
metrics *metrics.ServerMetrics
|
||||
cancelledDataUpload map[string]time.Time
|
||||
dataMovePriorityClass string
|
||||
podLabels map[string]string
|
||||
podAnnotations map[string]string
|
||||
client client.Client
|
||||
kubeClient kubernetes.Interface
|
||||
csiSnapshotClient snapshotter.SnapshotV1Interface
|
||||
mgr manager.Manager
|
||||
Clock clocks.WithTickerAndDelayedExecution
|
||||
nodeName string
|
||||
logger logrus.FieldLogger
|
||||
snapshotExposerList map[velerov2alpha1api.SnapshotType]exposer.SnapshotExposer
|
||||
dataPathMgr *datapath.Manager
|
||||
vgdpCounter *exposer.VgdpCounter
|
||||
loadAffinity []*kube.LoadAffinity
|
||||
backupPVCConfig map[string]velerotypes.BackupPVC
|
||||
podResources corev1api.ResourceRequirements
|
||||
preparingTimeout time.Duration
|
||||
metrics *metrics.ServerMetrics
|
||||
cancelledDataUpload map[string]time.Time
|
||||
dataMovePriorityClass string
|
||||
podLabels map[string]string
|
||||
podAnnotations map[string]string
|
||||
snapshotMetadataServiceConfigs *velerotypes.CSISnapshotMetadataService
|
||||
}
|
||||
|
||||
func NewDataUploadReconciler(
|
||||
@@ -105,6 +106,7 @@ func NewDataUploadReconciler(
|
||||
dataMovePriorityClass string,
|
||||
podLabels map[string]string,
|
||||
podAnnotations map[string]string,
|
||||
snapshotMetadataServiceConfigs *velerotypes.CSISnapshotMetadataService,
|
||||
) *DataUploadReconciler {
|
||||
return &DataUploadReconciler{
|
||||
client: client,
|
||||
@@ -121,17 +123,18 @@ func NewDataUploadReconciler(
|
||||
log,
|
||||
),
|
||||
},
|
||||
dataPathMgr: dataPathMgr,
|
||||
vgdpCounter: counter,
|
||||
loadAffinity: loadAffinity,
|
||||
backupPVCConfig: backupPVCConfig,
|
||||
podResources: podResources,
|
||||
preparingTimeout: preparingTimeout,
|
||||
metrics: metrics,
|
||||
cancelledDataUpload: make(map[string]time.Time),
|
||||
dataMovePriorityClass: dataMovePriorityClass,
|
||||
podLabels: podLabels,
|
||||
podAnnotations: podAnnotations,
|
||||
dataPathMgr: dataPathMgr,
|
||||
vgdpCounter: counter,
|
||||
loadAffinity: loadAffinity,
|
||||
backupPVCConfig: backupPVCConfig,
|
||||
podResources: podResources,
|
||||
preparingTimeout: preparingTimeout,
|
||||
metrics: metrics,
|
||||
cancelledDataUpload: make(map[string]time.Time),
|
||||
dataMovePriorityClass: dataMovePriorityClass,
|
||||
podLabels: podLabels,
|
||||
podAnnotations: podAnnotations,
|
||||
snapshotMetadataServiceConfigs: snapshotMetadataServiceConfigs,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -156,7 +159,7 @@ func (r *DataUploadReconciler) Reconcile(ctx context.Context, req ctrl.Request)
|
||||
return ctrl.Result{}, errors.Wrap(err, "getting DataUpload")
|
||||
}
|
||||
|
||||
if !datamover.IsBuiltInUploader(du.Spec.DataMover) {
|
||||
if !datamover.IsBuiltInDataMover(du.Spec.DataMover) {
|
||||
log.WithField("Data mover", du.Spec.DataMover).Debug("it is not one built-in data mover which is not supported by Velero")
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
@@ -936,7 +939,12 @@ func (r *DataUploadReconciler) setupExposeParam(du *velerov2alpha1api.DataUpload
|
||||
return nil, errors.Wrapf(err, "failed to get source PV %s", pvc.Spec.VolumeName)
|
||||
}
|
||||
|
||||
nodeOS := kube.GetPVCAttachingNodeOS(pvc, r.kubeClient.CoreV1(), r.kubeClient.StorageV1(), log)
|
||||
nodeOS := ""
|
||||
if du.Spec.DataMover == datamover.DataMoverTypeVeleroBlock {
|
||||
nodeOS = kube.NodeOSLinux
|
||||
} else {
|
||||
nodeOS = kube.GetPVCAttachingNodeOS(pvc, r.kubeClient.CoreV1(), r.kubeClient.StorageV1(), log)
|
||||
}
|
||||
|
||||
if err := kube.HasNodeWithOS(context.Background(), nodeOS, r.kubeClient.CoreV1()); err != nil {
|
||||
return nil, errors.Wrapf(err, "no appropriate node to run data upload for PVC %s/%s", du.Spec.SourceNamespace, du.Spec.SourcePVC)
|
||||
@@ -993,23 +1001,25 @@ func (r *DataUploadReconciler) setupExposeParam(du *velerov2alpha1api.DataUpload
|
||||
}
|
||||
|
||||
return &exposer.CSISnapshotExposeParam{
|
||||
SnapshotName: du.Spec.CSISnapshot.VolumeSnapshot,
|
||||
SourceNamespace: du.Spec.SourceNamespace,
|
||||
SourcePVCName: pvc.Name,
|
||||
SourcePVName: pv.Name,
|
||||
StorageClass: du.Spec.CSISnapshot.StorageClass,
|
||||
HostingPodLabels: hostingPodLabels,
|
||||
HostingPodAnnotations: hostingPodAnnotation,
|
||||
HostingPodTolerations: hostingPodTolerations,
|
||||
AccessMode: accessMode,
|
||||
OperationTimeout: du.Spec.OperationTimeout.Duration,
|
||||
ExposeTimeout: r.preparingTimeout,
|
||||
VolumeSize: pvc.Spec.Resources.Requests[corev1api.ResourceStorage],
|
||||
Affinity: r.loadAffinity,
|
||||
BackupPVCConfig: r.backupPVCConfig,
|
||||
Resources: r.podResources,
|
||||
NodeOS: nodeOS,
|
||||
PriorityClassName: r.dataMovePriorityClass,
|
||||
SnapshotName: du.Spec.CSISnapshot.VolumeSnapshot,
|
||||
SourceNamespace: du.Spec.SourceNamespace,
|
||||
SourcePVCName: pvc.Name,
|
||||
SourcePVName: pv.Name,
|
||||
StorageClass: du.Spec.CSISnapshot.StorageClass,
|
||||
HostingPodLabels: hostingPodLabels,
|
||||
HostingPodAnnotations: hostingPodAnnotation,
|
||||
HostingPodTolerations: hostingPodTolerations,
|
||||
AccessMode: accessMode,
|
||||
OperationTimeout: du.Spec.OperationTimeout.Duration,
|
||||
ExposeTimeout: r.preparingTimeout,
|
||||
VolumeSize: pvc.Spec.Resources.Requests[corev1api.ResourceStorage],
|
||||
Affinity: r.loadAffinity,
|
||||
BackupPVCConfig: r.backupPVCConfig,
|
||||
Resources: r.podResources,
|
||||
NodeOS: nodeOS,
|
||||
PriorityClassName: r.dataMovePriorityClass,
|
||||
DataMover: du.Spec.DataMover,
|
||||
SnapshotMetadataServiceConfigs: r.snapshotMetadataServiceConfigs,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -251,6 +251,7 @@ func initDataUploaderReconcilerWithError(needError ...error) (*DataUploadReconci
|
||||
"", // dataMovePriorityClass
|
||||
nil, // podLabels
|
||||
nil, // podAnnotations
|
||||
nil,
|
||||
), nil
|
||||
}
|
||||
|
||||
@@ -1513,6 +1514,7 @@ func TestDataUploadSetupExposeParam(t *testing.T) {
|
||||
"upload-priority",
|
||||
tt.args.customLabels,
|
||||
tt.args.customAnnotations,
|
||||
nil,
|
||||
)
|
||||
|
||||
// Act
|
||||
|
||||
@@ -399,6 +399,17 @@ func (r *restoreReconciler) validateAndComplete(restore *api.Restore) (backupInf
|
||||
return backupInfo{}, nil
|
||||
}
|
||||
|
||||
// reject restores from backups that are not in a usable phase
|
||||
switch info.backup.Status.Phase {
|
||||
case api.BackupPhaseCompleted, api.BackupPhasePartiallyFailed:
|
||||
// ok
|
||||
default:
|
||||
restore.Status.ValidationErrors = append(restore.Status.ValidationErrors,
|
||||
fmt.Sprintf("backup %q is in phase %q and cannot be used as a restore source",
|
||||
info.backup.Name, info.backup.Status.Phase))
|
||||
return backupInfo{}, nil
|
||||
}
|
||||
|
||||
// Fill in the ScheduleName so it's easier to consume for metrics.
|
||||
if restore.Spec.ScheduleName == "" {
|
||||
restore.Spec.ScheduleName = info.backup.GetLabels()[api.ScheduleNameLabel]
|
||||
|
||||
@@ -305,7 +305,7 @@ func TestRestoreReconcile(t *testing.T) {
|
||||
name: "restorer throwing an error causes the restore to fail",
|
||||
location: defaultStorageLocation,
|
||||
restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).Result(),
|
||||
backup: defaultBackup().StorageLocation("default").Result(),
|
||||
backup: defaultBackup().StorageLocation("default").Phase(velerov1api.BackupPhaseCompleted).Result(),
|
||||
restorerError: errors.New("blarg"),
|
||||
expectedErr: false,
|
||||
expectedPhase: string(velerov1api.RestorePhaseInProgress),
|
||||
@@ -319,7 +319,7 @@ func TestRestoreReconcile(t *testing.T) {
|
||||
name: "valid restore with none existingresourcepolicy gets executed",
|
||||
location: defaultStorageLocation,
|
||||
restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).ExistingResourcePolicy("none").Result(),
|
||||
backup: defaultBackup().StorageLocation("default").Result(),
|
||||
backup: defaultBackup().StorageLocation("default").Phase(velerov1api.BackupPhaseCompleted).Result(),
|
||||
expectedErr: false,
|
||||
expectedPhase: string(velerov1api.RestorePhaseInProgress),
|
||||
expectedStartTime: ×tamp,
|
||||
@@ -330,7 +330,7 @@ func TestRestoreReconcile(t *testing.T) {
|
||||
name: "valid restore with update existingresourcepolicy gets executed",
|
||||
location: defaultStorageLocation,
|
||||
restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).ExistingResourcePolicy("update").Result(),
|
||||
backup: defaultBackup().StorageLocation("default").Result(),
|
||||
backup: defaultBackup().StorageLocation("default").Phase(velerov1api.BackupPhaseCompleted).Result(),
|
||||
expectedErr: false,
|
||||
expectedPhase: string(velerov1api.RestorePhaseInProgress),
|
||||
expectedStartTime: ×tamp,
|
||||
@@ -352,7 +352,7 @@ func TestRestoreReconcile(t *testing.T) {
|
||||
name: "valid restore gets executed",
|
||||
location: defaultStorageLocation,
|
||||
restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).Result(),
|
||||
backup: defaultBackup().StorageLocation("default").Result(),
|
||||
backup: defaultBackup().StorageLocation("default").Phase(velerov1api.BackupPhaseCompleted).Result(),
|
||||
expectedErr: false,
|
||||
expectedPhase: string(velerov1api.RestorePhaseInProgress),
|
||||
expectedStartTime: ×tamp,
|
||||
@@ -363,7 +363,7 @@ func TestRestoreReconcile(t *testing.T) {
|
||||
name: "valid restore gets executed and only includes pod volume backups from restore namespace",
|
||||
location: defaultStorageLocation,
|
||||
restore: NewRestore("foo", "bar2", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).Result(),
|
||||
backup: defaultBackup().StorageLocation("default").Result(),
|
||||
backup: defaultBackup().StorageLocation("default").Phase(velerov1api.BackupPhaseCompleted).Result(),
|
||||
podVolumeBackups: []*velerov1api.PodVolumeBackup{
|
||||
builder.ForPodVolumeBackup("foo", "pvb-1").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(),
|
||||
builder.ForPodVolumeBackup("other-ns", "pvb-2").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(),
|
||||
@@ -444,7 +444,7 @@ func TestRestoreReconcile(t *testing.T) {
|
||||
expectedStartTime: ×tamp,
|
||||
expectedCompletedTime: ×tamp,
|
||||
backupStoreGetBackupContentsErr: errors.New("Couldn't download backup"),
|
||||
backup: defaultBackup().StorageLocation("default").Result(),
|
||||
backup: defaultBackup().StorageLocation("default").Phase(velerov1api.BackupPhaseCompleted).Result(),
|
||||
},
|
||||
{
|
||||
name: "restore attached with an expected finalizer gets cleaned up successfully",
|
||||
@@ -473,7 +473,7 @@ func TestRestoreReconcile(t *testing.T) {
|
||||
name: "valid restore with empty VolumeInfos",
|
||||
location: defaultStorageLocation,
|
||||
restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).Result(),
|
||||
backup: defaultBackup().StorageLocation("default").Result(),
|
||||
backup: defaultBackup().StorageLocation("default").Phase(velerov1api.BackupPhaseCompleted).Result(),
|
||||
emptyVolumeInfo: true,
|
||||
expectedErr: false,
|
||||
expectedPhase: string(velerov1api.RestorePhaseInProgress),
|
||||
@@ -497,6 +497,44 @@ func TestRestoreReconcile(t *testing.T) {
|
||||
backup: defaultBackup().StorageLocation("default").Result(),
|
||||
expectedErr: true,
|
||||
},
|
||||
{
|
||||
name: "restore from backup in Deleting phase fails validation",
|
||||
location: defaultStorageLocation,
|
||||
restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).Result(),
|
||||
backup: defaultBackup().StorageLocation("default").Phase(velerov1api.BackupPhaseDeleting).Result(),
|
||||
expectedErr: false,
|
||||
expectedPhase: string(velerov1api.RestorePhaseFailedValidation),
|
||||
expectedValidationErrors: []string{`backup "backup-1" is in phase "Deleting" and cannot be used as a restore source`},
|
||||
},
|
||||
{
|
||||
name: "restore from backup in InProgress phase fails validation",
|
||||
location: defaultStorageLocation,
|
||||
restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).Result(),
|
||||
backup: defaultBackup().StorageLocation("default").Phase(velerov1api.BackupPhaseInProgress).Result(),
|
||||
expectedErr: false,
|
||||
expectedPhase: string(velerov1api.RestorePhaseFailedValidation),
|
||||
expectedValidationErrors: []string{`backup "backup-1" is in phase "InProgress" and cannot be used as a restore source`},
|
||||
},
|
||||
{
|
||||
name: "restore from backup in PartiallyFailed phase succeeds",
|
||||
location: defaultStorageLocation,
|
||||
restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).Result(),
|
||||
backup: defaultBackup().StorageLocation("default").Phase(velerov1api.BackupPhasePartiallyFailed).Result(),
|
||||
expectedErr: false,
|
||||
expectedPhase: string(velerov1api.RestorePhaseInProgress),
|
||||
expectedStartTime: ×tamp,
|
||||
expectedCompletedTime: ×tamp,
|
||||
expectedRestorerCall: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseInProgress).Result(),
|
||||
},
|
||||
{
|
||||
name: "restore from backup in Failed phase fails validation",
|
||||
location: defaultStorageLocation,
|
||||
restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).Result(),
|
||||
backup: defaultBackup().StorageLocation("default").Phase(velerov1api.BackupPhaseFailed).Result(),
|
||||
expectedErr: false,
|
||||
expectedPhase: string(velerov1api.RestorePhaseFailedValidation),
|
||||
expectedValidationErrors: []string{`backup "backup-1" is in phase "Failed" and cannot be used as a restore source`},
|
||||
},
|
||||
}
|
||||
|
||||
formatFlag := logging.FormatText
|
||||
|
||||
@@ -170,14 +170,14 @@ func (r *BackupMicroService) RunCancelableDataPath(ctx context.Context) (string,
|
||||
OnProgress: r.OnDataUploadProgress,
|
||||
}
|
||||
|
||||
fsBackup, err := r.dataPathMgr.CreateFileSystemBR(du.Name, dataUploadDownloadRequestor, ctx, r.client, du.Namespace, callbacks, log)
|
||||
dp, err := r.dataPathMgr.CreateGenericDataPath(du.Name, dataUploadDownloadRequestor, ctx, r.client, du.Namespace, callbacks, log)
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "error to create data path")
|
||||
}
|
||||
|
||||
log.Debug("Async fs br created")
|
||||
|
||||
if err := fsBackup.Init(ctx, &datapath.FSBRInitParam{
|
||||
if err := dp.Init(ctx, &datapath.InitParam{
|
||||
BSLName: du.Spec.BackupStorageLocation,
|
||||
SourceNamespace: du.Spec.SourceNamespace,
|
||||
UploaderType: GetUploaderType(du.Spec.DataMover),
|
||||
@@ -195,7 +195,7 @@ func (r *BackupMicroService) RunCancelableDataPath(ctx context.Context) (string,
|
||||
velerov1api.AsyncOperationIDLabel: du.Labels[velerov1api.AsyncOperationIDLabel],
|
||||
}
|
||||
|
||||
if err := fsBackup.StartBackup(r.sourceTargetPath, du.Spec.DataMoverConfig, &datapath.FSBRStartParam{
|
||||
if err := dp.StartBackup(r.sourceTargetPath, du.Spec.DataMoverConfig, &datapath.BackupStartParam{
|
||||
RealSource: GetRealSource(du.Spec.SourceNamespace, du.Spec.SourcePVC),
|
||||
ParentSnapshot: "",
|
||||
ForceFull: false,
|
||||
|
||||
@@ -403,7 +403,7 @@ func TestRunCancelableDataPath(t *testing.T) {
|
||||
bs.dataPathMgr = test.dataPathMgr
|
||||
}
|
||||
|
||||
datapath.FSBRCreator = func(string, string, kbclient.Client, string, datapath.Callbacks, logrus.FieldLogger) datapath.AsyncBR {
|
||||
datapath.VGDPCreator = func(string, string, kbclient.Client, string, datapath.Callbacks, logrus.FieldLogger) datapath.AsyncBR {
|
||||
fsBR := datapathmockes.NewAsyncBR(t)
|
||||
if test.initErr != nil {
|
||||
fsBR.On("Init", mock.Anything, mock.Anything).Return(test.initErr)
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
|
||||
velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
|
||||
velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1"
|
||||
"github.com/vmware-tanzu/velero/pkg/label"
|
||||
"github.com/vmware-tanzu/velero/pkg/plugin/velero"
|
||||
repotypes "github.com/vmware-tanzu/velero/pkg/repository/types"
|
||||
)
|
||||
@@ -35,6 +36,44 @@ func (d *DataUploadDeleteAction) Execute(input *velero.DeleteItemActionExecuteIn
|
||||
if err := runtime.DefaultUnstructuredConverter.FromUnstructured(input.Item.UnstructuredContent(), &du); err != nil {
|
||||
return errors.WithStack(errors.Wrapf(err, "failed to convert input.Item from unstructured"))
|
||||
}
|
||||
// Only create a snapshot-info ConfigMap when the DataUpload's owning
|
||||
// backup (its velero.io/backup-name label) matches the backup currently
|
||||
// being deleted. Two other cases reach this code path and must be
|
||||
// skipped, because the resulting CM would be unmatchable and only adds
|
||||
// etcd churn:
|
||||
//
|
||||
// 1. The label is missing. We have no verifiable owner, so a CM created
|
||||
// with the executing backup's label is a guess that deleteMovedSnapshots
|
||||
// cannot rely on.
|
||||
// 2. The label names a different backup. Velero does not support
|
||||
// self-protection, so this almost always means the velero namespace
|
||||
// was captured in a backup tarball and the DataUpload CR belongs to
|
||||
// an unrelated backup. Creating a CM labeled with the executing
|
||||
// backup mislabels the snapshot and causes the real owning backup's
|
||||
// deleteMovedSnapshots query to miss it, leaking the Kopia snapshot
|
||||
// in the object store.
|
||||
//
|
||||
// Both cases warn so misconfigured installs surface in logs.
|
||||
owner := du.Labels[velerov1.BackupNameLabel]
|
||||
switch {
|
||||
case owner == "":
|
||||
d.logger.Warnf(
|
||||
"DataUpload %q has no %q label, so its owning backup cannot be verified; "+
|
||||
"skipping snapshot-info ConfigMap creation because a CM without a verifiable owner "+
|
||||
"cannot be matched back to its snapshot at backup deletion time.",
|
||||
du.Name, velerov1.BackupNameLabel,
|
||||
)
|
||||
return nil
|
||||
case owner != label.GetValidName(input.Backup.Name):
|
||||
d.logger.Warnf(
|
||||
"DataUpload %q belongs to backup %q but is being deleted under backup %q; "+
|
||||
"this almost always means the velero namespace was included in a backup tarball. "+
|
||||
"Velero does not support self-protection — exclude the velero namespace from your schedules. "+
|
||||
"Skipping snapshot-info ConfigMap creation to avoid mislabeling.",
|
||||
du.Name, owner, input.Backup.Name,
|
||||
)
|
||||
return nil
|
||||
}
|
||||
cm := genConfigmap(input.Backup, *du)
|
||||
if cm == nil {
|
||||
// will not fail the backup deletion
|
||||
@@ -49,7 +88,7 @@ func (d *DataUploadDeleteAction) Execute(input *velero.DeleteItemActionExecuteIn
|
||||
|
||||
// generate the configmap which is to be created and used as a way to communicate the snapshot info to the backup deletion controller
|
||||
func genConfigmap(bak *velerov1.Backup, du velerov2alpha1.DataUpload) *corev1api.ConfigMap {
|
||||
if !IsBuiltInUploader(du.Spec.DataMover) || du.Status.SnapshotID == "" {
|
||||
if !IsBuiltInDataMover(du.Spec.DataMover) || du.Status.SnapshotID == "" {
|
||||
return nil
|
||||
}
|
||||
snapshot := repotypes.SnapshotIdentifier{
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
/*
|
||||
Copyright the Velero contributors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package datamover
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
logrustest "github.com/sirupsen/logrus/hooks/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
corev1api "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
crclient "sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
|
||||
velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1"
|
||||
"github.com/vmware-tanzu/velero/pkg/builder"
|
||||
"github.com/vmware-tanzu/velero/pkg/plugin/velero"
|
||||
velerotest "github.com/vmware-tanzu/velero/pkg/test"
|
||||
)
|
||||
|
||||
func toUnstructured(t *testing.T, du *velerov2alpha1.DataUpload) runtime.Unstructured {
|
||||
t.Helper()
|
||||
m, err := runtime.DefaultUnstructuredConverter.ToUnstructured(du)
|
||||
require.NoError(t, err)
|
||||
return &unstructured.Unstructured{Object: m}
|
||||
}
|
||||
|
||||
func newCompletedDataUpload(name, ownerBackup string) *velerov2alpha1.DataUpload {
|
||||
du := &velerov2alpha1.DataUpload{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
APIVersion: velerov2alpha1.SchemeGroupVersion.String(),
|
||||
Kind: "DataUpload",
|
||||
},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Namespace: "velero",
|
||||
Name: name,
|
||||
},
|
||||
Spec: velerov2alpha1.DataUploadSpec{
|
||||
SnapshotType: velerov2alpha1.SnapshotTypeCSI,
|
||||
SourcePVC: "my-pvc",
|
||||
SourceNamespace: "app",
|
||||
BackupStorageLocation: "default",
|
||||
DataMover: "velero",
|
||||
},
|
||||
Status: velerov2alpha1.DataUploadStatus{
|
||||
Phase: velerov2alpha1.DataUploadPhaseCompleted,
|
||||
SnapshotID: "kopia-snapshot-id",
|
||||
},
|
||||
}
|
||||
if ownerBackup != "" {
|
||||
du.Labels = map[string]string{velerov1.BackupNameLabel: ownerBackup}
|
||||
}
|
||||
return du
|
||||
}
|
||||
|
||||
func TestDataUploadDeleteActionAppliesTo(t *testing.T) {
|
||||
a := NewDataUploadDeleteAction(logrus.StandardLogger(), nil)
|
||||
selector, err := a.AppliesTo()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, velero.ResourceSelector{IncludedResources: []string{"datauploads.velero.io"}}, selector)
|
||||
}
|
||||
|
||||
func TestDataUploadDeleteActionExecute(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
duName string
|
||||
duOwnerBackup string // value placed in velero.io/backup-name label on the DataUpload
|
||||
executingBackup string // name of the Backup being deleted (input.Backup.Name)
|
||||
wantConfigMap bool
|
||||
wantWarnContains string // substring expected in a warn-level log entry; empty means no warn expected
|
||||
}{
|
||||
{
|
||||
name: "DataUpload owned by the executing backup creates a snapshot-info ConfigMap",
|
||||
duName: "daily-backup-abcde",
|
||||
duOwnerBackup: "daily-backup",
|
||||
executingBackup: "daily-backup",
|
||||
wantConfigMap: true,
|
||||
wantWarnContains: "",
|
||||
},
|
||||
{
|
||||
name: "DataUpload owned by a different backup is skipped and a warning is logged",
|
||||
duName: "daily-backup-abcde",
|
||||
duOwnerBackup: "daily-backup",
|
||||
executingBackup: "hourly-backup",
|
||||
wantConfigMap: false,
|
||||
wantWarnContains: "velero namespace",
|
||||
},
|
||||
{
|
||||
name: "DataUpload with no backup-name label is skipped and a warning is logged",
|
||||
duName: "unlabeled-du",
|
||||
duOwnerBackup: "",
|
||||
executingBackup: "some-backup",
|
||||
wantConfigMap: false,
|
||||
wantWarnContains: "cannot be verified",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
crClient := velerotest.NewFakeControllerRuntimeClient(t)
|
||||
logger, hook := logrustest.NewNullLogger()
|
||||
logger.SetLevel(logrus.DebugLevel)
|
||||
action := NewDataUploadDeleteAction(logger, crClient)
|
||||
|
||||
du := newCompletedDataUpload(tc.duName, tc.duOwnerBackup)
|
||||
backup := builder.ForBackup("velero", tc.executingBackup).StorageLocation("default").Result()
|
||||
|
||||
err := action.Execute(&velero.DeleteItemActionExecuteInput{
|
||||
Item: toUnstructured(t, du),
|
||||
Backup: backup,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
cm := &corev1api.ConfigMap{}
|
||||
getErr := crClient.Get(t.Context(), crclient.ObjectKey{
|
||||
Namespace: backup.Namespace,
|
||||
Name: fmt.Sprintf("%s-info", du.Name),
|
||||
}, cm)
|
||||
|
||||
if tc.wantConfigMap {
|
||||
require.NoError(t, getErr, "expected snapshot-info ConfigMap to be created")
|
||||
assert.Equal(t, tc.executingBackup, cm.Labels[velerov1.BackupNameLabel])
|
||||
assert.Equal(t, "true", cm.Labels[velerov1.DataUploadSnapshotInfoLabel])
|
||||
} else {
|
||||
require.Error(t, getErr)
|
||||
assert.True(t, apierrors.IsNotFound(getErr),
|
||||
"expected no ConfigMap to be created for foreign DataUpload, but got: %v", getErr)
|
||||
}
|
||||
|
||||
// The action must surface DataUploads it cannot generate a useful
|
||||
// snapshot-info ConfigMap for as warnings, so operators who
|
||||
// accidentally included the velero namespace in a backup (or
|
||||
// otherwise produced DataUploads without a verifiable owner) can
|
||||
// detect the misconfiguration from logs instead of having the
|
||||
// case silently swallowed.
|
||||
var sawWarn bool
|
||||
for _, entry := range hook.AllEntries() {
|
||||
if entry.Level == logrus.WarnLevel &&
|
||||
strings.Contains(entry.Message, tc.duName) &&
|
||||
(tc.wantWarnContains == "" || strings.Contains(entry.Message, tc.wantWarnContains)) {
|
||||
sawWarn = true
|
||||
break
|
||||
}
|
||||
}
|
||||
assert.Equal(t, tc.wantWarnContains != "", sawWarn,
|
||||
"unexpected warn log presence (wantContains=%q, got=%v); entries=%v",
|
||||
tc.wantWarnContains, sawWarn, hook.AllEntries())
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -159,14 +159,14 @@ func (r *RestoreMicroService) RunCancelableDataPath(ctx context.Context) (string
|
||||
OnProgress: r.OnDataDownloadProgress,
|
||||
}
|
||||
|
||||
fsRestore, err := r.dataPathMgr.CreateFileSystemBR(dd.Name, dataUploadDownloadRequestor, ctx, r.client, dd.Namespace, callbacks, log)
|
||||
dp, err := r.dataPathMgr.CreateGenericDataPath(dd.Name, dataUploadDownloadRequestor, ctx, r.client, dd.Namespace, callbacks, log)
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "error to create data path")
|
||||
}
|
||||
|
||||
log.Debug("Found volume path")
|
||||
if err := fsRestore.Init(ctx,
|
||||
&datapath.FSBRInitParam{
|
||||
if err := dp.Init(ctx,
|
||||
&datapath.InitParam{
|
||||
BSLName: dd.Spec.BackupStorageLocation,
|
||||
SourceNamespace: dd.Spec.SourceNamespace,
|
||||
UploaderType: GetUploaderType(dd.Spec.DataMover),
|
||||
@@ -180,7 +180,7 @@ func (r *RestoreMicroService) RunCancelableDataPath(ctx context.Context) (string
|
||||
}
|
||||
log.Info("fs init")
|
||||
|
||||
if err := fsRestore.StartRestore(dd.Spec.SnapshotID, r.sourceTargetPath, dd.Spec.DataMoverConfig); err != nil {
|
||||
if err := dp.StartRestore(dd.Spec.SnapshotID, r.sourceTargetPath, dd.Spec.DataMoverConfig); err != nil {
|
||||
return "", errors.Wrap(err, "error starting data path restore")
|
||||
}
|
||||
|
||||
|
||||
@@ -347,7 +347,7 @@ func TestRunCancelableRestore(t *testing.T) {
|
||||
rs.dataPathMgr = test.dataPathMgr
|
||||
}
|
||||
|
||||
datapath.FSBRCreator = func(string, string, kbclient.Client, string, datapath.Callbacks, logrus.FieldLogger) datapath.AsyncBR {
|
||||
datapath.VGDPCreator = func(string, string, kbclient.Client, string, datapath.Callbacks, logrus.FieldLogger) datapath.AsyncBR {
|
||||
fsBR := datapathmockes.NewAsyncBR(t)
|
||||
if test.initErr != nil {
|
||||
fsBR.On("Init", mock.Anything, mock.Anything).Return(test.initErr)
|
||||
|
||||
@@ -18,6 +18,11 @@ package datamover
|
||||
|
||||
import "fmt"
|
||||
|
||||
const (
|
||||
DataMoverTypeVeleroFs string = "velero-fs"
|
||||
DataMoverTypeVeleroBlock string = "velero-block"
|
||||
)
|
||||
|
||||
func GetUploaderType(dataMover string) string {
|
||||
if dataMover == "" || dataMover == "velero" {
|
||||
return "kopia"
|
||||
@@ -26,7 +31,7 @@ func GetUploaderType(dataMover string) string {
|
||||
}
|
||||
}
|
||||
|
||||
func IsBuiltInUploader(dataMover string) bool {
|
||||
func IsBuiltInDataMover(dataMover string) bool {
|
||||
return dataMover == "" || dataMover == "velero"
|
||||
}
|
||||
|
||||
|
||||
@@ -30,7 +30,7 @@ func TestIsBuiltInUploader(t *testing.T) {
|
||||
}
|
||||
for _, tc := range testcases {
|
||||
t.Run(tc.name, func(tt *testing.T) {
|
||||
assert.Equal(tt, tc.want, IsBuiltInUploader(tc.dataMover))
|
||||
assert.Equal(tt, tc.want, IsBuiltInDataMover(tc.dataMover))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,8 +34,8 @@ import (
|
||||
"github.com/vmware-tanzu/velero/pkg/util/filesystem"
|
||||
)
|
||||
|
||||
// FSBRInitParam define the input param for FSBR init
|
||||
type FSBRInitParam struct {
|
||||
// InitParam define the input param for data path init
|
||||
type InitParam struct {
|
||||
BSLName string
|
||||
SourceNamespace string
|
||||
UploaderType string
|
||||
@@ -47,15 +47,15 @@ type FSBRInitParam struct {
|
||||
CacheDir string
|
||||
}
|
||||
|
||||
// FSBRStartParam define the input param for FSBR start
|
||||
type FSBRStartParam struct {
|
||||
// BackupStartParam define the input param for backup start
|
||||
type BackupStartParam struct {
|
||||
RealSource string
|
||||
ParentSnapshot string
|
||||
ForceFull bool
|
||||
Tags map[string]string
|
||||
}
|
||||
|
||||
type fileSystemBR struct {
|
||||
type generalDataPath struct {
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
backupRepo *velerov1api.BackupRepository
|
||||
@@ -72,8 +72,8 @@ type fileSystemBR struct {
|
||||
dataPathLock sync.Mutex
|
||||
}
|
||||
|
||||
func newFileSystemBR(jobName string, requestorType string, client client.Client, namespace string, callbacks Callbacks, log logrus.FieldLogger) AsyncBR {
|
||||
fs := &fileSystemBR{
|
||||
func newGeneralDataPath(jobName string, requestorType string, client client.Client, namespace string, callbacks Callbacks, log logrus.FieldLogger) AsyncBR {
|
||||
dp := &generalDataPath{
|
||||
jobName: jobName,
|
||||
requestorType: requestorType,
|
||||
client: client,
|
||||
@@ -83,151 +83,151 @@ func newFileSystemBR(jobName string, requestorType string, client client.Client,
|
||||
log: log,
|
||||
}
|
||||
|
||||
return fs
|
||||
return dp
|
||||
}
|
||||
|
||||
func (fs *fileSystemBR) Init(ctx context.Context, param any) error {
|
||||
initParam := param.(*FSBRInitParam)
|
||||
func (dp *generalDataPath) Init(ctx context.Context, param any) error {
|
||||
initParam := param.(*InitParam)
|
||||
|
||||
var err error
|
||||
defer func() {
|
||||
if err != nil {
|
||||
fs.Close(ctx)
|
||||
dp.Close(ctx)
|
||||
}
|
||||
}()
|
||||
|
||||
fs.ctx, fs.cancel = context.WithCancel(ctx)
|
||||
dp.ctx, dp.cancel = context.WithCancel(ctx)
|
||||
|
||||
backupLocation := &velerov1api.BackupStorageLocation{}
|
||||
if err = fs.client.Get(ctx, client.ObjectKey{
|
||||
Namespace: fs.namespace,
|
||||
if err = dp.client.Get(ctx, client.ObjectKey{
|
||||
Namespace: dp.namespace,
|
||||
Name: initParam.BSLName,
|
||||
}, backupLocation); err != nil {
|
||||
return errors.Wrapf(err, "error getting backup storage location %s", initParam.BSLName)
|
||||
}
|
||||
|
||||
fs.backupLocation = backupLocation
|
||||
dp.backupLocation = backupLocation
|
||||
|
||||
fs.backupRepo, err = initParam.RepositoryEnsurer.EnsureRepo(ctx, fs.namespace, initParam.SourceNamespace, initParam.BSLName, initParam.RepositoryType)
|
||||
dp.backupRepo, err = initParam.RepositoryEnsurer.EnsureRepo(ctx, dp.namespace, initParam.SourceNamespace, initParam.BSLName, initParam.RepositoryType)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "error to ensure backup repository %s-%s-%s", initParam.BSLName, initParam.SourceNamespace, initParam.RepositoryType)
|
||||
}
|
||||
|
||||
err = fs.boostRepoConnect(ctx, initParam.RepositoryType, initParam.CredentialGetter, initParam.CacheDir)
|
||||
err = dp.boostRepoConnect(ctx, initParam.RepositoryType, initParam.CredentialGetter, initParam.CacheDir)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "error to boost backup repository connection %s-%s-%s", initParam.BSLName, initParam.SourceNamespace, initParam.RepositoryType)
|
||||
}
|
||||
|
||||
fs.uploaderProv, err = provider.NewUploaderProvider(ctx, fs.client, initParam.UploaderType, fs.requestorType, initParam.RepoIdentifier,
|
||||
fs.backupLocation, fs.backupRepo, initParam.CredentialGetter, repokey.RepoKeySelector(), fs.log)
|
||||
dp.uploaderProv, err = provider.NewUploaderProvider(ctx, dp.client, initParam.UploaderType, dp.requestorType, initParam.RepoIdentifier,
|
||||
dp.backupLocation, dp.backupRepo, initParam.CredentialGetter, repokey.RepoKeySelector(), dp.log)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "error creating uploader %s", initParam.UploaderType)
|
||||
}
|
||||
|
||||
fs.initialized = true
|
||||
dp.initialized = true
|
||||
|
||||
fs.log.WithFields(
|
||||
dp.log.WithFields(
|
||||
logrus.Fields{
|
||||
"jobName": fs.jobName,
|
||||
"jobName": dp.jobName,
|
||||
"bsl": initParam.BSLName,
|
||||
"source namespace": initParam.SourceNamespace,
|
||||
"uploader": initParam.UploaderType,
|
||||
"repository": initParam.RepositoryType,
|
||||
}).Info("FileSystemBR is initialized")
|
||||
}).Info("Data path is initialized")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (fs *fileSystemBR) Close(ctx context.Context) {
|
||||
if fs.cancel != nil {
|
||||
fs.cancel()
|
||||
func (dp *generalDataPath) Close(ctx context.Context) {
|
||||
if dp.cancel != nil {
|
||||
dp.cancel()
|
||||
}
|
||||
|
||||
fs.log.WithField("user", fs.jobName).Info("Closing FileSystemBR")
|
||||
dp.log.WithField("user", dp.jobName).Info("Closing data path")
|
||||
|
||||
fs.wgDataPath.Wait()
|
||||
dp.wgDataPath.Wait()
|
||||
|
||||
fs.close(ctx)
|
||||
dp.close(ctx)
|
||||
|
||||
fs.log.WithField("user", fs.jobName).Info("FileSystemBR is closed")
|
||||
dp.log.WithField("user", dp.jobName).Info("Data path is closed")
|
||||
}
|
||||
|
||||
func (fs *fileSystemBR) close(ctx context.Context) {
|
||||
fs.dataPathLock.Lock()
|
||||
defer fs.dataPathLock.Unlock()
|
||||
func (dp *generalDataPath) close(ctx context.Context) {
|
||||
dp.dataPathLock.Lock()
|
||||
defer dp.dataPathLock.Unlock()
|
||||
|
||||
if fs.uploaderProv != nil {
|
||||
if err := fs.uploaderProv.Close(ctx); err != nil {
|
||||
fs.log.Errorf("failed to close uploader provider with error %v", err)
|
||||
if dp.uploaderProv != nil {
|
||||
if err := dp.uploaderProv.Close(ctx); err != nil {
|
||||
dp.log.Errorf("failed to close uploader provider with error %v", err)
|
||||
}
|
||||
|
||||
fs.uploaderProv = nil
|
||||
dp.uploaderProv = nil
|
||||
}
|
||||
}
|
||||
|
||||
func (fs *fileSystemBR) StartBackup(source AccessPoint, uploaderConfig map[string]string, param any) error {
|
||||
if !fs.initialized {
|
||||
func (dp *generalDataPath) StartBackup(source AccessPoint, uploaderConfig map[string]string, param any) error {
|
||||
if !dp.initialized {
|
||||
return errors.New("file system data path is not initialized")
|
||||
}
|
||||
|
||||
fs.wgDataPath.Add(1)
|
||||
dp.wgDataPath.Add(1)
|
||||
|
||||
backupParam := param.(*FSBRStartParam)
|
||||
backupParam := param.(*BackupStartParam)
|
||||
|
||||
go func() {
|
||||
fs.log.Info("Start data path backup")
|
||||
dp.log.Info("Start data path backup")
|
||||
|
||||
defer func() {
|
||||
fs.close(context.Background())
|
||||
fs.wgDataPath.Done()
|
||||
dp.close(context.Background())
|
||||
dp.wgDataPath.Done()
|
||||
}()
|
||||
|
||||
snapshotID, emptySnapshot, totalBytes, incrementalBytes, err := fs.uploaderProv.RunBackup(fs.ctx, source.ByPath, backupParam.RealSource, backupParam.Tags, backupParam.ForceFull,
|
||||
backupParam.ParentSnapshot, source.VolMode, uploaderConfig, fs)
|
||||
snapshotID, emptySnapshot, totalBytes, incrementalBytes, err := dp.uploaderProv.RunBackup(dp.ctx, source.ByPath, backupParam.RealSource, backupParam.Tags, backupParam.ForceFull,
|
||||
backupParam.ParentSnapshot, provider.CBTParam{}, source.VolMode, uploaderConfig, dp)
|
||||
|
||||
if err == provider.ErrorCanceled {
|
||||
fs.callbacks.OnCancelled(context.Background(), fs.namespace, fs.jobName)
|
||||
dp.callbacks.OnCancelled(context.Background(), dp.namespace, dp.jobName)
|
||||
} else if err != nil {
|
||||
dataPathErr := DataPathError{
|
||||
snapshotID: snapshotID,
|
||||
err: err,
|
||||
}
|
||||
fs.callbacks.OnFailed(context.Background(), fs.namespace, fs.jobName, dataPathErr)
|
||||
dp.callbacks.OnFailed(context.Background(), dp.namespace, dp.jobName, dataPathErr)
|
||||
} else {
|
||||
fs.callbacks.OnCompleted(context.Background(), fs.namespace, fs.jobName, Result{Backup: BackupResult{snapshotID, emptySnapshot, source, totalBytes, incrementalBytes}})
|
||||
dp.callbacks.OnCompleted(context.Background(), dp.namespace, dp.jobName, Result{Backup: BackupResult{snapshotID, emptySnapshot, source, totalBytes, incrementalBytes}})
|
||||
}
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (fs *fileSystemBR) StartRestore(snapshotID string, target AccessPoint, uploaderConfigs map[string]string) error {
|
||||
if !fs.initialized {
|
||||
return errors.New("file system data path is not initialized")
|
||||
func (dp *generalDataPath) StartRestore(snapshotID string, target AccessPoint, uploaderConfigs map[string]string) error {
|
||||
if !dp.initialized {
|
||||
return errors.New("data path is not initialized")
|
||||
}
|
||||
|
||||
fs.wgDataPath.Add(1)
|
||||
dp.wgDataPath.Add(1)
|
||||
|
||||
go func() {
|
||||
fs.log.Info("Start data path restore")
|
||||
dp.log.Info("Start data path restore")
|
||||
|
||||
defer func() {
|
||||
fs.close(context.Background())
|
||||
fs.wgDataPath.Done()
|
||||
dp.close(context.Background())
|
||||
dp.wgDataPath.Done()
|
||||
}()
|
||||
|
||||
totalBytes, err := fs.uploaderProv.RunRestore(fs.ctx, snapshotID, target.ByPath, target.VolMode, uploaderConfigs, fs)
|
||||
totalBytes, err := dp.uploaderProv.RunRestore(dp.ctx, snapshotID, target.ByPath, target.VolMode, uploaderConfigs, dp)
|
||||
|
||||
if err == provider.ErrorCanceled {
|
||||
fs.callbacks.OnCancelled(context.Background(), fs.namespace, fs.jobName)
|
||||
dp.callbacks.OnCancelled(context.Background(), dp.namespace, dp.jobName)
|
||||
} else if err != nil {
|
||||
dataPathErr := DataPathError{
|
||||
snapshotID: snapshotID,
|
||||
err: err,
|
||||
}
|
||||
fs.callbacks.OnFailed(context.Background(), fs.namespace, fs.jobName, dataPathErr)
|
||||
dp.callbacks.OnFailed(context.Background(), dp.namespace, dp.jobName, dataPathErr)
|
||||
} else {
|
||||
fs.callbacks.OnCompleted(context.Background(), fs.namespace, fs.jobName, Result{Restore: RestoreResult{Target: target, TotalBytes: totalBytes}})
|
||||
dp.callbacks.OnCompleted(context.Background(), dp.namespace, dp.jobName, Result{Restore: RestoreResult{Target: target, TotalBytes: totalBytes}})
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -235,20 +235,20 @@ func (fs *fileSystemBR) StartRestore(snapshotID string, target AccessPoint, uplo
|
||||
}
|
||||
|
||||
// UpdateProgress which implement ProgressUpdater interface to update progress status
|
||||
func (fs *fileSystemBR) UpdateProgress(p *uploader.Progress) {
|
||||
if fs.callbacks.OnProgress != nil {
|
||||
fs.callbacks.OnProgress(context.Background(), fs.namespace, fs.jobName, &uploader.Progress{TotalBytes: p.TotalBytes, BytesDone: p.BytesDone})
|
||||
func (dp *generalDataPath) UpdateProgress(p *uploader.Progress) {
|
||||
if dp.callbacks.OnProgress != nil {
|
||||
dp.callbacks.OnProgress(context.Background(), dp.namespace, dp.jobName, &uploader.Progress{TotalBytes: p.TotalBytes, BytesDone: p.BytesDone})
|
||||
}
|
||||
}
|
||||
|
||||
func (fs *fileSystemBR) Cancel() {
|
||||
fs.cancel()
|
||||
fs.log.WithField("user", fs.jobName).Info("FileSystemBR is canceled")
|
||||
func (dp *generalDataPath) Cancel() {
|
||||
dp.cancel()
|
||||
dp.log.WithField("user", dp.jobName).Info("FileSystemBR is canceled")
|
||||
}
|
||||
|
||||
func (fs *fileSystemBR) boostRepoConnect(ctx context.Context, repositoryType string, credentialGetter *credentials.CredentialGetter, cacheDir string) error {
|
||||
func (dp *generalDataPath) boostRepoConnect(ctx context.Context, repositoryType string, credentialGetter *credentials.CredentialGetter, cacheDir string) error {
|
||||
if repositoryType == velerov1api.BackupRepositoryTypeKopia {
|
||||
if err := repoProvider.NewUnifiedRepoProvider(*credentialGetter, repositoryType, fs.log).BoostRepoConnect(ctx, repoProvider.RepoParam{BackupLocation: fs.backupLocation, BackupRepo: fs.backupRepo, CacheDir: cacheDir}); err != nil {
|
||||
if err := repoProvider.NewUnifiedRepoProvider(*credentialGetter, repositoryType, dp.log).BoostRepoConnect(ctx, repoProvider.RepoParam{BackupLocation: dp.backupLocation, BackupRepo: dp.backupRepo, CacheDir: cacheDir}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -94,21 +94,21 @@ func TestAsyncBackup(t *testing.T) {
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
fs := newFileSystemBR("job-1", "test", nil, "velero", Callbacks{}, velerotest.NewLogger()).(*fileSystemBR)
|
||||
dp := newGeneralDataPath("job-1", "test", nil, "velero", Callbacks{}, velerotest.NewLogger()).(*generalDataPath)
|
||||
mockProvider := providerMock.NewProvider(t)
|
||||
mockProvider.On("RunBackup", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(test.result.Backup.SnapshotID, test.result.Backup.EmptySnapshot, test.result.Backup.TotalBytes, test.result.Backup.IncrementalBytes, test.err)
|
||||
mockProvider.On("RunBackup", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(test.result.Backup.SnapshotID, test.result.Backup.EmptySnapshot, test.result.Backup.TotalBytes, test.result.Backup.IncrementalBytes, test.err)
|
||||
mockProvider.On("Close", mock.Anything).Return(nil)
|
||||
fs.uploaderProv = mockProvider
|
||||
fs.initialized = true
|
||||
fs.callbacks = test.callbacks
|
||||
dp.uploaderProv = mockProvider
|
||||
dp.initialized = true
|
||||
dp.callbacks = test.callbacks
|
||||
|
||||
err := fs.StartBackup(AccessPoint{ByPath: test.path}, map[string]string{}, &FSBRStartParam{})
|
||||
err := dp.StartBackup(AccessPoint{ByPath: test.path}, map[string]string{}, &BackupStartParam{})
|
||||
require.NoError(t, err)
|
||||
|
||||
<-finish
|
||||
|
||||
// Ensure the goroutine finishes so deferred fs.close executes, satisfying mock expectations.
|
||||
fs.wgDataPath.Wait()
|
||||
dp.wgDataPath.Wait()
|
||||
|
||||
assert.Equal(t, test.err, asyncErr)
|
||||
assert.Equal(t, test.result, asyncResult)
|
||||
@@ -182,21 +182,21 @@ func TestAsyncRestore(t *testing.T) {
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
fs := newFileSystemBR("job-1", "test", nil, "velero", Callbacks{}, velerotest.NewLogger()).(*fileSystemBR)
|
||||
dp := newGeneralDataPath("job-1", "test", nil, "velero", Callbacks{}, velerotest.NewLogger()).(*generalDataPath)
|
||||
mockProvider := providerMock.NewProvider(t)
|
||||
mockProvider.On("RunRestore", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(test.result.Restore.TotalBytes, test.err)
|
||||
mockProvider.On("Close", mock.Anything).Return(nil)
|
||||
fs.uploaderProv = mockProvider
|
||||
fs.initialized = true
|
||||
fs.callbacks = test.callbacks
|
||||
dp.uploaderProv = mockProvider
|
||||
dp.initialized = true
|
||||
dp.callbacks = test.callbacks
|
||||
|
||||
err := fs.StartRestore(test.snapshot, AccessPoint{ByPath: test.path}, map[string]string{})
|
||||
err := dp.StartRestore(test.snapshot, AccessPoint{ByPath: test.path}, map[string]string{})
|
||||
require.NoError(t, err)
|
||||
|
||||
<-finish
|
||||
|
||||
// Ensure the goroutine finishes so deferred fs.close executes, satisfying mock expectations.
|
||||
fs.wgDataPath.Wait()
|
||||
dp.wgDataPath.Wait()
|
||||
|
||||
assert.Equal(t, asyncErr, test.err)
|
||||
assert.Equal(t, asyncResult, test.result)
|
||||
@@ -28,7 +28,7 @@ import (
|
||||
)
|
||||
|
||||
var ConcurrentLimitExceed error = errors.New("Concurrent number exceeds")
|
||||
var FSBRCreator = newFileSystemBR
|
||||
var VGDPCreator = newGeneralDataPath
|
||||
var MicroServiceBRWatcherCreator = newMicroServiceBRWatcher
|
||||
|
||||
type Manager struct {
|
||||
@@ -45,8 +45,8 @@ func NewManager(cocurrentNum int) *Manager {
|
||||
}
|
||||
}
|
||||
|
||||
// CreateFileSystemBR creates a new file system backup/restore data path instance
|
||||
func (m *Manager) CreateFileSystemBR(jobName string, requestorType string, ctx context.Context, client client.Client, namespace string, callbacks Callbacks, log logrus.FieldLogger) (AsyncBR, error) {
|
||||
// CreateGenericDataPath creates a new generic data path instance
|
||||
func (m *Manager) CreateGenericDataPath(jobName string, requestorType string, ctx context.Context, client client.Client, namespace string, callbacks Callbacks, log logrus.FieldLogger) (AsyncBR, error) {
|
||||
m.trackerLock.Lock()
|
||||
defer m.trackerLock.Unlock()
|
||||
|
||||
@@ -54,7 +54,7 @@ func (m *Manager) CreateFileSystemBR(jobName string, requestorType string, ctx c
|
||||
return nil, ConcurrentLimitExceed
|
||||
}
|
||||
|
||||
m.tracker[jobName] = FSBRCreator(jobName, requestorType, client, namespace, callbacks, log)
|
||||
m.tracker[jobName] = VGDPCreator(jobName, requestorType, client, namespace, callbacks, log)
|
||||
|
||||
return m.tracker[jobName], nil
|
||||
}
|
||||
|
||||
@@ -23,16 +23,16 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestCreateFileSystemBR(t *testing.T) {
|
||||
func TestCreateGenericDataPath(t *testing.T) {
|
||||
m := NewManager(2)
|
||||
|
||||
async_job_1, err := m.CreateFileSystemBR("job-1", "test", t.Context(), nil, "velero", Callbacks{}, nil)
|
||||
async_job_1, err := m.CreateGenericDataPath("job-1", "test", t.Context(), nil, "velero", Callbacks{}, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = m.CreateFileSystemBR("job-2", "test", t.Context(), nil, "velero", Callbacks{}, nil)
|
||||
_, err = m.CreateGenericDataPath("job-2", "test", t.Context(), nil, "velero", Callbacks{}, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = m.CreateFileSystemBR("job-3", "test", t.Context(), nil, "velero", Callbacks{}, nil)
|
||||
_, err = m.CreateGenericDataPath("job-3", "test", t.Context(), nil, "velero", Callbacks{}, nil)
|
||||
assert.Equal(t, ConcurrentLimitExceed, err)
|
||||
|
||||
ret := m.GetAsyncBR("job-0")
|
||||
|
||||
@@ -19,6 +19,7 @@ package exposer
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"maps"
|
||||
"time"
|
||||
|
||||
snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1"
|
||||
@@ -33,6 +34,7 @@ import (
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
"github.com/vmware-tanzu/velero/pkg/datamover"
|
||||
"github.com/vmware-tanzu/velero/pkg/nodeagent"
|
||||
velerotypes "github.com/vmware-tanzu/velero/pkg/types"
|
||||
"github.com/vmware-tanzu/velero/pkg/util"
|
||||
@@ -93,6 +95,12 @@ type CSISnapshotExposeParam struct {
|
||||
|
||||
// PriorityClassName is the priority class name for the data mover pod
|
||||
PriorityClassName string
|
||||
|
||||
// DataMover is the data mover type, e.g., velero-fs, velero-block
|
||||
DataMover string
|
||||
|
||||
// SnapshotMetadataServiceConfigs is the config for CSI snapshot metadata service
|
||||
SnapshotMetadataServiceConfigs *velerotypes.CSISnapshotMetadataService
|
||||
}
|
||||
|
||||
// CSISnapshotExposeWaitParam define the input param for WaitExposed of CSI snapshots
|
||||
@@ -234,7 +242,7 @@ func (e *csiSnapshotExposer) Expose(ctx context.Context, ownerObject corev1api.O
|
||||
}
|
||||
}
|
||||
|
||||
backupPVC, err := e.createBackupPVC(ctx, ownerObject, backupVS.Name, backupPVCStorageClass, csiExposeParam.AccessMode, volumeSize, backupPVCReadOnly, backupPVCAnnotations)
|
||||
backupPVC, err := e.createBackupPVC(ctx, ownerObject, backupVS.Name, backupPVCStorageClass, csiExposeParam.AccessMode, volumeSize, backupPVCReadOnly, backupPVCAnnotations, csiExposeParam.DataMover)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "error to create backup pvc")
|
||||
}
|
||||
@@ -264,6 +272,7 @@ func (e *csiSnapshotExposer) Expose(ctx context.Context, ownerObject corev1api.O
|
||||
csiExposeParam.PriorityClassName,
|
||||
intoleratableNodes,
|
||||
volumeTopology,
|
||||
csiExposeParam.SnapshotMetadataServiceConfigs,
|
||||
)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "error to create backup pod")
|
||||
@@ -450,7 +459,11 @@ func (e *csiSnapshotExposer) CleanUp(ctx context.Context, ownerObject corev1api.
|
||||
csi.DeleteVolumeSnapshotIfAny(ctx, e.csiSnapshotClient, vsName, sourceNamespace, e.log)
|
||||
}
|
||||
|
||||
func getVolumeModeByAccessMode(accessMode string) (corev1api.PersistentVolumeMode, error) {
|
||||
func getVolumeModeByAccessMode(accessMode string, dataMover string) (corev1api.PersistentVolumeMode, error) {
|
||||
if dataMover == datamover.DataMoverTypeVeleroBlock {
|
||||
return corev1api.PersistentVolumeBlock, nil
|
||||
}
|
||||
|
||||
switch accessMode {
|
||||
case AccessModeFileSystem:
|
||||
return corev1api.PersistentVolumeFilesystem, nil
|
||||
@@ -488,10 +501,14 @@ func (e *csiSnapshotExposer) createBackupVS(ctx context.Context, ownerObject cor
|
||||
func (e *csiSnapshotExposer) createBackupVSC(ctx context.Context, ownerObject corev1api.ObjectReference, snapshotVSC *snapshotv1api.VolumeSnapshotContent, vs *snapshotv1api.VolumeSnapshot) (*snapshotv1api.VolumeSnapshotContent, error) {
|
||||
backupVSCName := ownerObject.Name
|
||||
|
||||
anno := make(map[string]string)
|
||||
maps.Copy(anno, snapshotVSC.Annotations)
|
||||
anno[kube.KubeAnnAllowVolumeModeChange] = "true"
|
||||
|
||||
vsc := &snapshotv1api.VolumeSnapshotContent{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: backupVSCName,
|
||||
Annotations: snapshotVSC.Annotations,
|
||||
Annotations: anno,
|
||||
Labels: map[string]string{},
|
||||
},
|
||||
Spec: snapshotv1api.VolumeSnapshotContentSpec{
|
||||
@@ -524,10 +541,10 @@ func (e *csiSnapshotExposer) createBackupVSC(ctx context.Context, ownerObject co
|
||||
return e.csiSnapshotClient.VolumeSnapshotContents().Create(ctx, vsc, metav1.CreateOptions{})
|
||||
}
|
||||
|
||||
func (e *csiSnapshotExposer) createBackupPVC(ctx context.Context, ownerObject corev1api.ObjectReference, backupVS, storageClass, accessMode string, resource resource.Quantity, readOnly bool, annotations map[string]string) (*corev1api.PersistentVolumeClaim, error) {
|
||||
func (e *csiSnapshotExposer) createBackupPVC(ctx context.Context, ownerObject corev1api.ObjectReference, backupVS, storageClass, accessMode string, resource resource.Quantity, readOnly bool, annotations map[string]string, dataMover string) (*corev1api.PersistentVolumeClaim, error) {
|
||||
backupPVCName := ownerObject.Name
|
||||
|
||||
volumeMode, err := getVolumeModeByAccessMode(accessMode)
|
||||
volumeMode, err := getVolumeModeByAccessMode(accessMode, dataMover)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -600,6 +617,7 @@ func (e *csiSnapshotExposer) createBackupPod(
|
||||
priorityClassName string,
|
||||
intoleratableNodes []string,
|
||||
volumeTopology *corev1api.NodeSelector,
|
||||
csiSnapshotMetadataServiceConfigs *velerotypes.CSISnapshotMetadataService,
|
||||
) (*corev1api.Pod, error) {
|
||||
podName := ownerObject.Name
|
||||
|
||||
@@ -655,6 +673,12 @@ func (e *csiSnapshotExposer) createBackupPod(
|
||||
args = append(args, podInfo.logFormatArgs...)
|
||||
args = append(args, podInfo.logLevelArgs...)
|
||||
|
||||
if csiSnapshotMetadataServiceConfigs != nil {
|
||||
if csiSnapshotMetadataServiceConfigs.SAName != "" {
|
||||
args = append(args, fmt.Sprintf("--csi-snapshot-metadata-service-sa=%s", csiSnapshotMetadataServiceConfigs.SAName))
|
||||
}
|
||||
}
|
||||
|
||||
if affinity == nil {
|
||||
affinity = &kube.LoadAffinity{}
|
||||
}
|
||||
|
||||
@@ -155,6 +155,7 @@ func TestCreateBackupPodWithPriorityClass(t *testing.T) {
|
||||
tc.expectedPriorityClass,
|
||||
nil,
|
||||
nil,
|
||||
nil,
|
||||
)
|
||||
|
||||
require.NoError(t, err, tc.description)
|
||||
@@ -241,6 +242,7 @@ func TestCreateBackupPodWithMissingConfigMap(t *testing.T) {
|
||||
"", // empty priority class since config map is missing
|
||||
nil,
|
||||
nil,
|
||||
nil,
|
||||
)
|
||||
|
||||
// Should succeed even when config map is missing
|
||||
|
||||
@@ -18,6 +18,7 @@ package exposer
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"maps"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -1056,21 +1057,25 @@ func TestExpose(t *testing.T) {
|
||||
backupPVC, err := exposer.kubeClient.CoreV1().PersistentVolumeClaims(ownerObject.Namespace).Get(t.Context(), ownerObject.Name, metav1.GetOptions{})
|
||||
require.NoError(t, err)
|
||||
|
||||
expectedVS, err := exposer.csiSnapshotClient.VolumeSnapshots(ownerObject.Namespace).Get(t.Context(), ownerObject.Name, metav1.GetOptions{})
|
||||
backupVS, err := exposer.csiSnapshotClient.VolumeSnapshots(ownerObject.Namespace).Get(t.Context(), ownerObject.Name, metav1.GetOptions{})
|
||||
require.NoError(t, err)
|
||||
|
||||
expectedVSC, err := exposer.csiSnapshotClient.VolumeSnapshotContents().Get(t.Context(), ownerObject.Name, metav1.GetOptions{})
|
||||
backupVSC, err := exposer.csiSnapshotClient.VolumeSnapshotContents().Get(t.Context(), ownerObject.Name, metav1.GetOptions{})
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, expectedVS.Annotations, vsObject.Annotations)
|
||||
assert.Equal(t, *expectedVS.Spec.VolumeSnapshotClassName, *vsObject.Spec.VolumeSnapshotClassName)
|
||||
assert.Equal(t, expectedVSC.Name, *expectedVS.Spec.Source.VolumeSnapshotContentName)
|
||||
assert.Equal(t, vsObject.Annotations, backupVS.Annotations)
|
||||
assert.Equal(t, *vsObject.Spec.VolumeSnapshotClassName, *backupVS.Spec.VolumeSnapshotClassName)
|
||||
assert.Equal(t, *backupVS.Spec.Source.VolumeSnapshotContentName, backupVSC.Name)
|
||||
|
||||
assert.Equal(t, expectedVSC.Annotations, vscObj.Annotations)
|
||||
assert.Equal(t, expectedVSC.Labels, vscObj.Labels)
|
||||
assert.Equal(t, expectedVSC.Spec.DeletionPolicy, vscObj.Spec.DeletionPolicy)
|
||||
assert.Equal(t, expectedVSC.Spec.Driver, vscObj.Spec.Driver)
|
||||
assert.Equal(t, *expectedVSC.Spec.VolumeSnapshotClassName, *vscObj.Spec.VolumeSnapshotClassName)
|
||||
anno := make(map[string]string)
|
||||
maps.Copy(anno, vscObj.Annotations)
|
||||
anno[kube.KubeAnnAllowVolumeModeChange] = "true"
|
||||
|
||||
assert.Equal(t, anno, backupVSC.Annotations)
|
||||
assert.Equal(t, vscObj.Labels, backupVSC.Labels)
|
||||
assert.Equal(t, vscObj.Spec.DeletionPolicy, backupVSC.Spec.DeletionPolicy)
|
||||
assert.Equal(t, vscObj.Spec.Driver, backupVSC.Spec.Driver)
|
||||
assert.Equal(t, *vscObj.Spec.VolumeSnapshotClassName, *backupVSC.Spec.VolumeSnapshotClassName)
|
||||
|
||||
if test.expectedVolumeSize != nil {
|
||||
assert.Equal(t, *test.expectedVolumeSize, backupPVC.Spec.Resources.Requests[corev1api.ResourceStorage])
|
||||
@@ -1514,7 +1519,7 @@ func Test_csiSnapshotExposer_createBackupPVC(t *testing.T) {
|
||||
APIVersion: tt.ownerBackup.APIVersion,
|
||||
}
|
||||
}
|
||||
got, err := e.createBackupPVC(t.Context(), ownerObject, tt.backupVS, tt.storageClass, tt.accessMode, tt.resource, tt.readOnly, map[string]string{})
|
||||
got, err := e.createBackupPVC(t.Context(), ownerObject, tt.backupVS, tt.storageClass, tt.accessMode, tt.resource, tt.readOnly, map[string]string{}, "")
|
||||
if !tt.wantErr(t, err, fmt.Sprintf("createBackupPVC(%v, %v, %v, %v, %v, %v)", ownerObject, tt.backupVS, tt.storageClass, tt.accessMode, tt.resource, tt.readOnly)) {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -169,14 +169,14 @@ func (r *BackupMicroService) RunCancelableDataPath(ctx context.Context) (string,
|
||||
OnProgress: r.OnDataPathProgress,
|
||||
}
|
||||
|
||||
fsBackup, err := r.dataPathMgr.CreateFileSystemBR(pvb.Name, podVolumeRequestor, ctx, r.client, pvb.Namespace, callbacks, log)
|
||||
fsBackup, err := r.dataPathMgr.CreateGenericDataPath(pvb.Name, podVolumeRequestor, ctx, r.client, pvb.Namespace, callbacks, log)
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "error to create data path")
|
||||
}
|
||||
|
||||
log.Debug("Async fs br created")
|
||||
|
||||
if err := fsBackup.Init(ctx, &datapath.FSBRInitParam{
|
||||
if err := fsBackup.Init(ctx, &datapath.InitParam{
|
||||
BSLName: pvb.Spec.BackupStorageLocation,
|
||||
SourceNamespace: pvb.Spec.Pod.Namespace,
|
||||
UploaderType: pvb.Spec.UploaderType,
|
||||
@@ -192,7 +192,7 @@ func (r *BackupMicroService) RunCancelableDataPath(ctx context.Context) (string,
|
||||
|
||||
tags := map[string]string{}
|
||||
|
||||
if err := fsBackup.StartBackup(r.sourceTargetPath, pvb.Spec.UploaderSettings, &datapath.FSBRStartParam{
|
||||
if err := fsBackup.StartBackup(r.sourceTargetPath, pvb.Spec.UploaderSettings, &datapath.BackupStartParam{
|
||||
RealSource: GetRealSource(pvb),
|
||||
ParentSnapshot: "",
|
||||
ForceFull: false,
|
||||
|
||||
@@ -402,7 +402,7 @@ func TestRunCancelableDataPath(t *testing.T) {
|
||||
bs.dataPathMgr = test.dataPathMgr
|
||||
}
|
||||
|
||||
datapath.FSBRCreator = func(string, string, kbclient.Client, string, datapath.Callbacks, logrus.FieldLogger) datapath.AsyncBR {
|
||||
datapath.VGDPCreator = func(string, string, kbclient.Client, string, datapath.Callbacks, logrus.FieldLogger) datapath.AsyncBR {
|
||||
fsBR := datapathmockes.NewAsyncBR(t)
|
||||
if test.initErr != nil {
|
||||
fsBR.On("Init", mock.Anything, mock.Anything).Return(test.initErr)
|
||||
|
||||
@@ -396,7 +396,7 @@ func TestBackupPodVolumes(t *testing.T) {
|
||||
},
|
||||
uploaderType: "fake-uploader-type",
|
||||
errs: []string{
|
||||
"invalid uploader type 'fake-uploader-type', valid type: 'kopia'",
|
||||
"invalid uploader type 'fake-uploader-type', valid types: 'kopia', 'velero-block'",
|
||||
},
|
||||
},
|
||||
{
|
||||
|
||||
@@ -161,7 +161,7 @@ func (r *RestoreMicroService) RunCancelableDataPath(ctx context.Context) (string
|
||||
OnProgress: r.OnPvrProgress,
|
||||
}
|
||||
|
||||
fsRestore, err := r.dataPathMgr.CreateFileSystemBR(pvr.Name, podVolumeRequestor, ctx, r.client, pvr.Namespace, callbacks, log)
|
||||
fsRestore, err := r.dataPathMgr.CreateGenericDataPath(pvr.Name, podVolumeRequestor, ctx, r.client, pvr.Namespace, callbacks, log)
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "error to create data path")
|
||||
}
|
||||
@@ -169,7 +169,7 @@ func (r *RestoreMicroService) RunCancelableDataPath(ctx context.Context) (string
|
||||
log.Debug("Async fs br created")
|
||||
|
||||
if err := fsRestore.Init(ctx,
|
||||
&datapath.FSBRInitParam{
|
||||
&datapath.InitParam{
|
||||
BSLName: pvr.Spec.BackupStorageLocation,
|
||||
SourceNamespace: pvr.Spec.SourceNamespace,
|
||||
UploaderType: pvr.Spec.UploaderType,
|
||||
|
||||
@@ -428,7 +428,7 @@ func TestRunCancelableDataPathRestore(t *testing.T) {
|
||||
rs.dataPathMgr = test.dataPathMgr
|
||||
}
|
||||
|
||||
datapath.FSBRCreator = func(string, string, kbclient.Client, string, datapath.Callbacks, logrus.FieldLogger) datapath.AsyncBR {
|
||||
datapath.VGDPCreator = func(string, string, kbclient.Client, string, datapath.Callbacks, logrus.FieldLogger) datapath.AsyncBR {
|
||||
fsBR := datapathmockes.NewAsyncBR(t)
|
||||
if test.initErr != nil {
|
||||
fsBR.On("Init", mock.Anything, mock.Anything).Return(test.initErr)
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
/*
|
||||
Copyright the Velero contributors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package freelist
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
)
|
||||
|
||||
type FreeList struct {
|
||||
chunks chan []byte
|
||||
memory []byte
|
||||
chunkSize int
|
||||
}
|
||||
|
||||
func New(size, chunkSize int) *FreeList {
|
||||
memory := make([]byte, size)
|
||||
numChunks := size / chunkSize
|
||||
chunks := make(chan []byte, numChunks)
|
||||
|
||||
for i := range numChunks {
|
||||
start := i * chunkSize
|
||||
end := start + chunkSize
|
||||
|
||||
chunks <- memory[start:end:end]
|
||||
}
|
||||
|
||||
return &FreeList{
|
||||
chunks: chunks,
|
||||
memory: memory,
|
||||
chunkSize: chunkSize,
|
||||
}
|
||||
}
|
||||
|
||||
func (f *FreeList) Chunks() <-chan []byte {
|
||||
return f.chunks
|
||||
}
|
||||
|
||||
func (f *FreeList) Get() []byte {
|
||||
return <-f.chunks
|
||||
}
|
||||
|
||||
func (f *FreeList) Return(chunk []byte) {
|
||||
if cap(chunk) != f.chunkSize {
|
||||
panic(fmt.Sprintf("chunk (%v) is not allocated by me", cap(chunk)))
|
||||
}
|
||||
|
||||
chunk = chunk[:cap(chunk)]
|
||||
f.chunks <- chunk
|
||||
}
|
||||
|
||||
func (f *FreeList) Capacity() int {
|
||||
return len(f.chunks)
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
/*
|
||||
Copyright the Velero contributors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package freelist
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestNew(t *testing.T) {
|
||||
size := 1024
|
||||
chunkSize := 256
|
||||
numChunks := size / chunkSize
|
||||
|
||||
fl := New(size, chunkSize)
|
||||
|
||||
assert.NotNil(t, fl)
|
||||
assert.Equal(t, chunkSize, fl.chunkSize)
|
||||
assert.Len(t, fl.memory, size)
|
||||
assert.Equal(t, numChunks, cap(fl.chunks))
|
||||
assert.Len(t, fl.chunks, numChunks)
|
||||
assert.Equal(t, numChunks, fl.Capacity())
|
||||
}
|
||||
|
||||
func TestGetAndReturn(t *testing.T) {
|
||||
size := 1024
|
||||
chunkSize := 256
|
||||
|
||||
fl := New(size, chunkSize)
|
||||
|
||||
chunk := fl.Get()
|
||||
assert.Equal(t, chunkSize, cap(chunk))
|
||||
assert.Len(t, chunk, chunkSize)
|
||||
assert.Equal(t, 3, fl.Capacity())
|
||||
|
||||
fl.Return(chunk)
|
||||
assert.Equal(t, 4, fl.Capacity())
|
||||
}
|
||||
|
||||
func TestReturnPanic(t *testing.T) {
|
||||
fl := New(1024, 256)
|
||||
|
||||
invalidChunk := make([]byte, 128)
|
||||
assert.PanicsWithValue(t, "chunk (128) is not allocated by me", func() {
|
||||
fl.Return(invalidChunk)
|
||||
})
|
||||
}
|
||||
|
||||
func TestChunks(t *testing.T) {
|
||||
fl := New(1024, 256)
|
||||
|
||||
chunks := fl.Chunks()
|
||||
assert.Len(t, chunks, 4)
|
||||
assert.Equal(t, cap(fl.chunks), cap(chunks))
|
||||
}
|
||||
|
||||
func TestCapacity(t *testing.T) {
|
||||
fl := New(1024, 256)
|
||||
|
||||
assert.Equal(t, 4, fl.Capacity())
|
||||
|
||||
fl.Get()
|
||||
assert.Equal(t, 3, fl.Capacity())
|
||||
}
|
||||
|
||||
func TestConcurrentAccess(t *testing.T) {
|
||||
size := 1024 * 10
|
||||
chunkSize := 256
|
||||
numChunks := size / chunkSize
|
||||
|
||||
fl := New(size, chunkSize)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < numChunks; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
chunk := fl.Get()
|
||||
assert.Equal(t, chunkSize, cap(chunk))
|
||||
assert.Len(t, chunk, chunkSize)
|
||||
|
||||
for j := 0; j < len(chunk); j++ {
|
||||
chunk[j] = byte(j % 256)
|
||||
}
|
||||
|
||||
fl.Return(chunk)
|
||||
}()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
assert.Equal(t, numChunks, fl.Capacity())
|
||||
}
|
||||
@@ -19,18 +19,24 @@ package kopialib
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/kopia/kopia/fs"
|
||||
"github.com/kopia/kopia/repo"
|
||||
"github.com/kopia/kopia/repo/compression"
|
||||
"github.com/kopia/kopia/repo/content"
|
||||
"github.com/kopia/kopia/repo/content/index"
|
||||
"github.com/kopia/kopia/repo/maintenance"
|
||||
"github.com/kopia/kopia/repo/manifest"
|
||||
"github.com/kopia/kopia/repo/object"
|
||||
"github.com/kopia/kopia/snapshot"
|
||||
"github.com/kopia/kopia/snapshot/snapshotfs"
|
||||
"github.com/kopia/kopia/snapshot/snapshotmaintenance"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/sirupsen/logrus"
|
||||
@@ -38,6 +44,7 @@ import (
|
||||
"github.com/vmware-tanzu/velero/pkg/kopia"
|
||||
"github.com/vmware-tanzu/velero/pkg/repository/udmrepo"
|
||||
"github.com/vmware-tanzu/velero/pkg/repository/udmrepo/kopialib/backend"
|
||||
"github.com/vmware-tanzu/velero/pkg/repository/udmrepo/kopialib/freelist"
|
||||
)
|
||||
|
||||
type kopiaRepoService struct {
|
||||
@@ -75,6 +82,24 @@ type kopiaObjectWriter struct {
|
||||
rawWriter object.Writer
|
||||
}
|
||||
|
||||
type kopiaObjectWriterEx struct {
|
||||
ctx context.Context
|
||||
rawRepoWriter repo.RepositoryWriter
|
||||
parentEntries []object.IndirectObjectEntry
|
||||
entries []object.IndirectObjectEntry
|
||||
entryLock sync.Mutex
|
||||
blockSize int64
|
||||
description string
|
||||
compressor compression.Name
|
||||
splitter string
|
||||
writeLock sync.Mutex
|
||||
asyncWritesSem chan struct{}
|
||||
asyncWritesGroup sync.WaitGroup
|
||||
asyncBuffer *freelist.FreeList
|
||||
writeError atomic.Value
|
||||
logger logrus.FieldLogger
|
||||
}
|
||||
|
||||
type openOptions struct {
|
||||
repoLogger io.Writer
|
||||
}
|
||||
@@ -85,6 +110,9 @@ const (
|
||||
overwriteFullMaintainInterval = time.Duration(0)
|
||||
overwriteQuickMaintainInterval = time.Duration(0)
|
||||
repoBackend = "kopia"
|
||||
fixedSplitter1M = "FIXED-1M"
|
||||
fixedSplitter128K = "FIXED-128K"
|
||||
fixedBlockSize = 1 << 20
|
||||
)
|
||||
|
||||
var kopiaRepoOpen = repo.Open
|
||||
@@ -388,36 +416,165 @@ func (kr *kopiaRepository) Close(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (kr *kopiaRepository) ContentInfo(ctx context.Context, contentID content.ID) (content.Info, error) {
|
||||
return kr.rawRepo.ContentInfo(kopia.SetupKopiaLog(ctx, kr.logger), contentID)
|
||||
}
|
||||
|
||||
func (kr *kopiaRepository) GetContent(ctx context.Context, contentID content.ID) ([]byte, error) {
|
||||
directRepo, ok := kr.rawRepo.(repo.DirectRepository)
|
||||
if !ok {
|
||||
return nil, errors.New("invalid repo interface")
|
||||
}
|
||||
|
||||
return directRepo.ContentReader().GetContent(kopia.SetupKopiaLog(ctx, kr.logger), contentID)
|
||||
}
|
||||
|
||||
func (kr *kopiaRepository) PrefetchContents(ctx context.Context, contentIDs []content.ID, prefetchHint string) []content.ID {
|
||||
return kr.rawRepo.PrefetchContents(kopia.SetupKopiaLog(ctx, kr.logger), contentIDs, prefetchHint)
|
||||
}
|
||||
|
||||
func (kr *kopiaRepository) getFlattenedEntries(ctx context.Context, rawID object.ID) ([]object.IndirectObjectEntry, error) {
|
||||
indexObjectID, ok := rawID.IndexObjectID()
|
||||
if !ok {
|
||||
return nil, errors.Errorf("object is not an indirect object, %v", rawID)
|
||||
}
|
||||
|
||||
return object.LoadIndexObject(kopia.SetupKopiaLog(ctx, kr.logger), kr, indexObjectID)
|
||||
}
|
||||
|
||||
func (kr *kopiaRepository) NewObjectWriter(ctx context.Context, opt udmrepo.ObjectWriteOptions) (udmrepo.ObjectWriter, error) {
|
||||
if kr.rawWriter == nil {
|
||||
return nil, errors.New("repo writer is closed or not open")
|
||||
}
|
||||
|
||||
writer := kr.rawWriter.NewObjectWriter(kopia.SetupKopiaLog(ctx, kr.logger), object.WriterOptions{
|
||||
Description: opt.Description,
|
||||
Prefix: index.IDPrefix(opt.Prefix),
|
||||
AsyncWrites: opt.AsyncWrites,
|
||||
Compressor: getCompressorForObject(opt),
|
||||
MetadataCompressor: getMetadataCompressor(),
|
||||
})
|
||||
var parentEntries []object.IndirectObjectEntry
|
||||
if opt.AccessMode == udmrepo.ObjectDataAccessModeBlock {
|
||||
if opt.ParentObject != "" {
|
||||
kr.logger.Infof("Write object %s in block mode with parent %s", opt.Description, opt.ParentObject)
|
||||
|
||||
if writer == nil {
|
||||
return nil, errors.Errorf("error creating writer for object %s", opt.Description)
|
||||
rawID, err := object.ParseID(string(opt.ParentObject))
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "error parsing parent object ID from %v", opt.ParentObject)
|
||||
}
|
||||
|
||||
parentEntries, err = kr.getFlattenedEntries(ctx, rawID)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "error getting parent object entries from %v", opt.ParentObject)
|
||||
}
|
||||
} else {
|
||||
kr.logger.Infof("Write object %s in block mode without parent", opt.Description)
|
||||
}
|
||||
|
||||
var asyncWritesSem chan struct{}
|
||||
var asyncBuffer *freelist.FreeList
|
||||
if opt.AsyncWrites > 0 {
|
||||
asyncWritesSem = make(chan struct{}, opt.AsyncWrites)
|
||||
asyncBuffer = freelist.New(opt.AsyncWrites*fixedBlockSize, fixedBlockSize)
|
||||
}
|
||||
|
||||
return &kopiaObjectWriterEx{
|
||||
ctx: ctx,
|
||||
rawRepoWriter: kr.rawWriter,
|
||||
parentEntries: parentEntries,
|
||||
description: opt.Description,
|
||||
compressor: getCompressorForObject(opt),
|
||||
blockSize: fixedBlockSize,
|
||||
splitter: fixedSplitter1M,
|
||||
asyncWritesSem: asyncWritesSem,
|
||||
asyncBuffer: asyncBuffer,
|
||||
logger: kr.logger,
|
||||
}, nil
|
||||
} else {
|
||||
if opt.ParentObject != "" {
|
||||
return nil, errors.Errorf("parent object is only supported for block mode")
|
||||
}
|
||||
|
||||
writer := kr.rawWriter.NewObjectWriter(kopia.SetupKopiaLog(ctx, kr.logger), object.WriterOptions{
|
||||
Description: opt.Description,
|
||||
Prefix: index.IDPrefix(opt.Prefix),
|
||||
AsyncWrites: opt.AsyncWrites,
|
||||
Compressor: getCompressorForObject(opt),
|
||||
MetadataCompressor: getMetadataCompressor(),
|
||||
})
|
||||
|
||||
if writer == nil {
|
||||
return nil, errors.Errorf("error creating writer for object %s", opt.Description)
|
||||
}
|
||||
|
||||
return &kopiaObjectWriter{
|
||||
rawWriter: writer,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
const kopiaDirStreamType = "kopia:directory"
|
||||
|
||||
func (kr *kopiaRepository) WriteMetadata(ctx context.Context, meta *udmrepo.Metadata, opt udmrepo.ObjectWriteOptions) (udmrepo.ID, error) {
|
||||
if kr.rawWriter == nil {
|
||||
return "", errors.New("repo writer is closed or not open")
|
||||
}
|
||||
|
||||
return &kopiaObjectWriter{
|
||||
rawWriter: writer,
|
||||
}, nil
|
||||
dirEntries := []*snapshot.DirEntry{}
|
||||
if meta.SubObjects != nil {
|
||||
for _, sub := range meta.SubObjects {
|
||||
rawID, err := object.ParseID(string(sub.ID))
|
||||
if err != nil {
|
||||
return "", errors.Wrapf(err, "error parsing object ID from %v", sub)
|
||||
}
|
||||
|
||||
dirEntries = append(dirEntries, &snapshot.DirEntry{
|
||||
Name: sub.Name,
|
||||
ObjectID: rawID,
|
||||
Type: getKopiaObjectType(sub.Type),
|
||||
FileSize: sub.Size,
|
||||
Permissions: snapshot.Permissions(sub.Permissions),
|
||||
ModTime: fs.UTCTimestampFromTime(sub.ModTime),
|
||||
UserID: sub.UserID,
|
||||
GroupID: sub.GroupID,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
dirManifest := snapshot.DirManifest{
|
||||
StreamType: kopiaDirStreamType,
|
||||
Entries: dirEntries,
|
||||
}
|
||||
|
||||
oid, err := snapshotfs.WriteDirManifest(ctx, kr.rawWriter, opt.Description, &dirManifest, getMetadataCompressor())
|
||||
if err != nil {
|
||||
return "", errors.Wrapf(err, "error writing dir manifest: %v", opt.Description)
|
||||
}
|
||||
|
||||
return udmrepo.ID(oid.String()), nil
|
||||
}
|
||||
|
||||
// TODO add implementation in following PRs
|
||||
func (kr *kopiaRepository) WriteMetadata(ctx context.Context, meta *udmrepo.Metadata, opt udmrepo.ObjectWriteOptions) (udmrepo.ID, error) {
|
||||
return "", errors.New("not supported")
|
||||
}
|
||||
|
||||
// TODO add implementation in following PRs
|
||||
func (kr *kopiaRepository) ReadMetadata(ctx context.Context, id udmrepo.ID) (*udmrepo.Metadata, error) {
|
||||
return nil, errors.New("not supported")
|
||||
reader, err := kr.OpenObject(ctx, id)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "error to open metadata object %v", id)
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
dirManifest := snapshot.DirManifest{}
|
||||
if err := json.NewDecoder(reader).Decode(&dirManifest); err != nil {
|
||||
return nil, errors.Wrap(err, "unable to parse directory object")
|
||||
}
|
||||
|
||||
meta := udmrepo.Metadata{}
|
||||
for _, sub := range dirManifest.Entries {
|
||||
meta.SubObjects = append(meta.SubObjects, udmrepo.ObjectMetadata{
|
||||
ID: udmrepo.ID(sub.ObjectID.String()),
|
||||
Name: sub.Name,
|
||||
Type: getObjectDataType(sub.Type),
|
||||
Size: sub.FileSize,
|
||||
ModTime: sub.ModTime.ToTime(),
|
||||
Permissions: int(sub.Permissions),
|
||||
UserID: sub.UserID,
|
||||
GroupID: sub.GroupID,
|
||||
})
|
||||
}
|
||||
|
||||
return &meta, nil
|
||||
}
|
||||
|
||||
func (kr *kopiaRepository) PutManifest(ctx context.Context, manifest udmrepo.RepoManifest) (udmrepo.ID, error) {
|
||||
@@ -446,19 +603,124 @@ func (kr *kopiaRepository) DeleteManifest(ctx context.Context, id udmrepo.ID) er
|
||||
return nil
|
||||
}
|
||||
|
||||
// TODO add implementation in following PRs
|
||||
func (kr *kopiaRepository) SaveSnapshot(ctx context.Context, snap udmrepo.Snapshot) (udmrepo.ID, error) {
|
||||
return "", errors.New("not supported")
|
||||
if kr.rawWriter == nil {
|
||||
return "", errors.New("repo writer is closed or not open")
|
||||
}
|
||||
|
||||
if snap.Source == "" {
|
||||
return "", errors.New("invalid snapshot source")
|
||||
}
|
||||
|
||||
rootObj, err := object.ParseID(string(snap.RootObject.ID))
|
||||
if err != nil {
|
||||
return "", errors.Wrapf(err, "error parsing root object ID %v", snap.RootObject.ID)
|
||||
}
|
||||
|
||||
manifest := snapshot.Manifest{
|
||||
Source: snapshot.SourceInfo{
|
||||
UserName: udmrepo.GetRepoUser(),
|
||||
Host: udmrepo.GetRepoDomain(),
|
||||
Path: snap.Source,
|
||||
},
|
||||
Description: snap.Description,
|
||||
StartTime: fs.UTCTimestampFromTime(snap.StartTime),
|
||||
EndTime: fs.UTCTimestampFromTime(snap.EndTime),
|
||||
Stats: snapshot.Stats{
|
||||
TotalFileSize: snap.TotalSize,
|
||||
},
|
||||
RootEntry: &snapshot.DirEntry{
|
||||
Type: snapshot.EntryTypeDirectory,
|
||||
ObjectID: rootObj,
|
||||
ModTime: fs.UTCTimestampFromTime(snap.RootObject.ModTime),
|
||||
Permissions: snapshot.Permissions(snap.RootObject.Permissions),
|
||||
FileSize: snap.RootObject.Size,
|
||||
UserID: snap.RootObject.UserID,
|
||||
GroupID: snap.RootObject.GroupID,
|
||||
DirSummary: &fs.DirectorySummary{
|
||||
TotalFileSize: snap.TotalSize,
|
||||
},
|
||||
},
|
||||
Tags: snap.Tags,
|
||||
Pins: []string{"velero-pin"}, // pins are meant to prevent snapshot from automatic expiration/deletion.
|
||||
}
|
||||
|
||||
id, err := snapshot.SaveSnapshot(ctx, kr.rawWriter, &manifest)
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "error saving snapshot")
|
||||
}
|
||||
|
||||
return udmrepo.ID(id), nil
|
||||
}
|
||||
|
||||
// TODO add implementation in following PRs
|
||||
func (kr *kopiaRepository) GetSnapshot(ctx context.Context, id udmrepo.ID) (udmrepo.Snapshot, error) {
|
||||
return udmrepo.Snapshot{}, errors.New("not supported")
|
||||
snap, err := snapshot.LoadSnapshot(ctx, kr.rawRepo, manifest.ID(id))
|
||||
if err != nil {
|
||||
return udmrepo.Snapshot{}, errors.Wrap(err, "error getting snapshot manifest")
|
||||
}
|
||||
|
||||
if snap.RootEntry == nil {
|
||||
return udmrepo.Snapshot{}, errors.New("invalid snapshot root entry")
|
||||
}
|
||||
|
||||
return udmrepo.Snapshot{
|
||||
Source: snap.Source.Path,
|
||||
Description: snap.Description,
|
||||
StartTime: snap.StartTime.ToTime(),
|
||||
EndTime: snap.EndTime.ToTime(),
|
||||
Tags: snap.Tags,
|
||||
TotalSize: snap.Stats.TotalFileSize,
|
||||
RootObject: udmrepo.ObjectMetadata{
|
||||
ID: udmrepo.ID(snap.RootEntry.ObjectID.String()),
|
||||
Type: udmrepo.ObjectDataTypeMetadata,
|
||||
Size: snap.RootEntry.FileSize,
|
||||
ModTime: snap.RootEntry.ModTime.ToTime(),
|
||||
Permissions: int(snap.RootEntry.Permissions),
|
||||
UserID: snap.RootEntry.UserID,
|
||||
GroupID: snap.RootEntry.GroupID,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// TODO add implementation in following PRs
|
||||
func (kr *kopiaRepository) DeleteSnapshot(ctx context.Context, id udmrepo.ID) error {
|
||||
return errors.New("not supported")
|
||||
if _, err := kr.GetSnapshot(ctx, id); err != nil {
|
||||
return errors.Wrap(err, "error getting snapshot")
|
||||
}
|
||||
|
||||
return kr.DeleteManifest(ctx, id)
|
||||
}
|
||||
|
||||
func (kr *kopiaRepository) ListSnapshot(ctx context.Context, source string) ([]udmrepo.Snapshot, error) {
|
||||
mani, err := snapshot.ListSnapshots(ctx, kr.rawRepo, snapshot.SourceInfo{
|
||||
Host: udmrepo.GetRepoDomain(),
|
||||
UserName: udmrepo.GetRepoUser(),
|
||||
Path: source,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "error listing snapshot manifest for source %s", source)
|
||||
}
|
||||
|
||||
snapshots := []udmrepo.Snapshot{}
|
||||
for _, snap := range mani {
|
||||
snapshots = append(snapshots, udmrepo.Snapshot{
|
||||
Source: snap.Source.Path,
|
||||
Description: snap.Description,
|
||||
StartTime: snap.StartTime.ToTime(),
|
||||
EndTime: snap.EndTime.ToTime(),
|
||||
Tags: snap.Tags,
|
||||
RootObject: udmrepo.ObjectMetadata{
|
||||
ID: udmrepo.ID(snap.RootEntry.ObjectID.String()),
|
||||
Type: udmrepo.ObjectDataTypeMetadata,
|
||||
Size: snap.RootEntry.FileSize,
|
||||
ModTime: snap.RootEntry.ModTime.ToTime(),
|
||||
Permissions: int(snap.RootEntry.Permissions),
|
||||
UserID: snap.RootEntry.UserID,
|
||||
GroupID: snap.RootEntry.GroupID,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
return snapshots, nil
|
||||
}
|
||||
|
||||
func (kr *kopiaRepository) Flush(ctx context.Context) error {
|
||||
@@ -571,7 +833,6 @@ func (kow *kopiaObjectWriter) Write(p []byte) (int, error) {
|
||||
return kow.rawWriter.Write(p)
|
||||
}
|
||||
|
||||
// TODO add implementation in following PRs
|
||||
func (kow *kopiaObjectWriter) WriteAt(p []byte, offset int64) (int, error) {
|
||||
return 0, errors.New("not supported")
|
||||
}
|
||||
@@ -617,6 +878,203 @@ func (kow *kopiaObjectWriter) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (kow *kopiaObjectWriterEx) Write(p []byte) (int, error) {
|
||||
kow.writeLock.Lock()
|
||||
defer kow.writeLock.Unlock()
|
||||
|
||||
if kow.rawRepoWriter == nil {
|
||||
return 0, errors.New("object writer is closed or not open")
|
||||
}
|
||||
|
||||
if err := kow.getWriteError(); err != nil {
|
||||
return 0, errors.Wrapf(err, "error happened during writing object")
|
||||
}
|
||||
|
||||
length := len(p)
|
||||
if int64(length)%kow.blockSize != 0 {
|
||||
return 0, errors.Errorf("invalid length %v", length)
|
||||
}
|
||||
|
||||
kow.entryLock.Lock()
|
||||
curPos := int64(len(kow.entries)) * kow.blockSize
|
||||
kow.entryLock.Unlock()
|
||||
|
||||
offset := curPos
|
||||
entryID := 0
|
||||
for curPos < offset+int64(length) {
|
||||
kow.entryLock.Lock()
|
||||
entryID = len(kow.entries)
|
||||
kow.entries = append(kow.entries, object.IndirectObjectEntry{
|
||||
Start: curPos,
|
||||
Length: kow.blockSize,
|
||||
})
|
||||
kow.entryLock.Unlock()
|
||||
|
||||
buffOffset := curPos - offset
|
||||
objName := fmt.Sprintf("%s-b%v", kow.description, entryID)
|
||||
kow.writeObjectAsync(objName, entryID, p[buffOffset:buffOffset+kow.blockSize])
|
||||
|
||||
curPos += kow.blockSize
|
||||
}
|
||||
|
||||
return length, nil
|
||||
}
|
||||
|
||||
func (kow *kopiaObjectWriterEx) writeObject(objName string, p []byte) (object.ID, error) {
|
||||
writer := kow.rawRepoWriter.NewObjectWriter(kopia.SetupKopiaLog(kow.ctx, kow.logger), object.WriterOptions{
|
||||
Description: objName,
|
||||
Compressor: kow.compressor,
|
||||
Splitter: kow.splitter,
|
||||
})
|
||||
|
||||
if writer == nil {
|
||||
return object.EmptyID, errors.Errorf("error opening writer for %s", objName)
|
||||
}
|
||||
|
||||
defer writer.Close()
|
||||
|
||||
written, err := writer.Write(p)
|
||||
if err != nil {
|
||||
return object.EmptyID, errors.Wrapf(err, "error writing for %s", objName)
|
||||
}
|
||||
|
||||
if written != len(p) {
|
||||
return object.EmptyID, errors.Errorf("short write for %s", objName)
|
||||
}
|
||||
|
||||
objID, err := writer.Result()
|
||||
if err != nil {
|
||||
return object.EmptyID, errors.Wrapf(err, "error flushing data for %s", objName)
|
||||
}
|
||||
|
||||
return objID, nil
|
||||
}
|
||||
|
||||
func (kow *kopiaObjectWriterEx) writeObjectSync(objName string, entry int, p []byte) error {
|
||||
objID, err := kow.writeObject(objName, p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
kow.entryLock.Lock()
|
||||
kow.entries[entry].Object = objID
|
||||
kow.entryLock.Unlock()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (kow *kopiaObjectWriterEx) writeObjectAsync(objName string, entryID int, p []byte) {
|
||||
if kow.asyncWritesSem == nil {
|
||||
if err := kow.writeObjectSync(objName, entryID, p); err != nil {
|
||||
kow.saveWriteError(errors.Wrapf(err, "error writing object for %s", objName))
|
||||
}
|
||||
} else {
|
||||
kow.asyncWritesSem <- struct{}{}
|
||||
|
||||
buffer := kow.asyncBuffer.Get()
|
||||
copy(buffer, p)
|
||||
|
||||
kow.asyncWritesGroup.Go(func() {
|
||||
if err := kow.writeObjectSync(objName, entryID, buffer); err != nil {
|
||||
kow.saveWriteError(errors.Wrapf(err, "error writing object for %s", objName))
|
||||
}
|
||||
|
||||
kow.asyncBuffer.Return(buffer)
|
||||
<-kow.asyncWritesSem
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TODO add implementation in following PRs
|
||||
func (kow *kopiaObjectWriterEx) WriteAt(p []byte, offset int64) (int, error) {
|
||||
return 0, errors.New("not implemented")
|
||||
}
|
||||
|
||||
func (kow *kopiaObjectWriterEx) Checkpoint() (udmrepo.ID, error) {
|
||||
return udmrepo.ID(""), errors.New("not supported")
|
||||
}
|
||||
|
||||
type indirectObject struct {
|
||||
StreamID string `json:"stream"`
|
||||
Entries []object.IndirectObjectEntry `json:"entries"`
|
||||
}
|
||||
|
||||
const kopiaIndirectStreamType = "kopia:indirect"
|
||||
|
||||
func (kow *kopiaObjectWriterEx) writeIndirectObject() (object.ID, error) {
|
||||
if kow.rawRepoWriter == nil {
|
||||
return object.EmptyID, errors.New("object writer is closed or not open")
|
||||
}
|
||||
|
||||
writer := kow.rawRepoWriter.NewObjectWriter(kopia.SetupKopiaLog(kow.ctx, kow.logger), object.WriterOptions{
|
||||
Description: "LIST(" + kow.description + ")",
|
||||
Prefix: "x",
|
||||
Compressor: getMetadataCompressor(),
|
||||
Splitter: fixedSplitter128K,
|
||||
})
|
||||
if writer == nil {
|
||||
return object.EmptyID, errors.New("unable to create writer for indirect object")
|
||||
}
|
||||
|
||||
defer writer.Close()
|
||||
|
||||
ind := indirectObject{
|
||||
StreamID: kopiaIndirectStreamType,
|
||||
Entries: kow.entries,
|
||||
}
|
||||
|
||||
if err := json.NewEncoder(writer).Encode(ind); err != nil {
|
||||
return object.EmptyID, errors.Wrap(err, "unable to write indirect object index")
|
||||
}
|
||||
|
||||
return writer.Result()
|
||||
}
|
||||
|
||||
func (kow *kopiaObjectWriterEx) saveWriteError(err error) {
|
||||
if err != nil {
|
||||
kow.writeError.Store(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (kow *kopiaObjectWriterEx) getWriteError() error {
|
||||
if v := kow.writeError.Load(); v != nil {
|
||||
return v.(error)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (kow *kopiaObjectWriterEx) Result() (udmrepo.ID, error) {
|
||||
kow.writeLock.Lock()
|
||||
defer kow.writeLock.Unlock()
|
||||
|
||||
kow.asyncWritesGroup.Wait()
|
||||
|
||||
if err := kow.getWriteError(); err != nil {
|
||||
return udmrepo.ID(""), errors.Wrap(err, "error happened during writing object")
|
||||
}
|
||||
|
||||
id, err := kow.writeIndirectObject()
|
||||
if err != nil {
|
||||
return udmrepo.ID(""), errors.Wrap(err, "error to write indirect object")
|
||||
}
|
||||
|
||||
objectID := "I" + udmrepo.ID(id.String())
|
||||
|
||||
return objectID, nil
|
||||
}
|
||||
|
||||
func (kow *kopiaObjectWriterEx) Close() error {
|
||||
kow.writeLock.Lock()
|
||||
defer kow.writeLock.Unlock()
|
||||
|
||||
kow.asyncWritesGroup.Wait()
|
||||
|
||||
kow.rawRepoWriter = nil
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// getCompressorForObject returns the compressor for an object, at present, we don't support compression
|
||||
func getCompressorForObject(_ udmrepo.ObjectWriteOptions) compression.Name {
|
||||
return ""
|
||||
@@ -676,3 +1134,25 @@ func openKopiaRepo(ctx context.Context, configFile string, password string, opti
|
||||
|
||||
return r, nil
|
||||
}
|
||||
|
||||
func getKopiaObjectType(tp int) snapshot.EntryType {
|
||||
switch tp {
|
||||
case udmrepo.ObjectDataTypeMetadata:
|
||||
return snapshot.EntryTypeDirectory
|
||||
case udmrepo.ObjectDataTypeData:
|
||||
return snapshot.EntryTypeFile
|
||||
default:
|
||||
return snapshot.EntryTypeUnknown
|
||||
}
|
||||
}
|
||||
|
||||
func getObjectDataType(tp snapshot.EntryType) int {
|
||||
switch tp {
|
||||
case snapshot.EntryTypeDirectory:
|
||||
return udmrepo.ObjectDataTypeMetadata
|
||||
case snapshot.EntryTypeFile:
|
||||
return udmrepo.ObjectDataTypeData
|
||||
default:
|
||||
return udmrepo.ObjectDataTypeUnknown
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,550 @@
|
||||
package kopialib
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/kopia/kopia/repo"
|
||||
"github.com/kopia/kopia/repo/content"
|
||||
"github.com/kopia/kopia/repo/object"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/mock"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/vmware-tanzu/velero/pkg/repository/udmrepo"
|
||||
repomocks "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/kopialib/backend/mocks"
|
||||
"github.com/vmware-tanzu/velero/pkg/repository/udmrepo/kopialib/freelist"
|
||||
velerotest "github.com/vmware-tanzu/velero/pkg/test"
|
||||
)
|
||||
|
||||
type mockDirectRepository struct {
|
||||
repo.DirectRepository
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
func (m *mockDirectRepository) ContentInfo(ctx context.Context, contentID content.ID) (content.Info, error) {
|
||||
args := m.Called(ctx, contentID)
|
||||
return args.Get(0).(content.Info), args.Error(1)
|
||||
}
|
||||
|
||||
func (m *mockDirectRepository) ContentReader() content.Reader {
|
||||
args := m.Called()
|
||||
return args.Get(0).(content.Reader)
|
||||
}
|
||||
|
||||
type mockContentReader struct {
|
||||
content.Reader
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
func (m *mockContentReader) GetContent(ctx context.Context, contentID content.ID) ([]byte, error) {
|
||||
args := m.Called(ctx, contentID)
|
||||
if args.Get(0) == nil {
|
||||
return nil, args.Error(1)
|
||||
}
|
||||
return args.Get(0).([]byte), args.Error(1)
|
||||
}
|
||||
|
||||
func TestContentInfo(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
rawRepo repo.Repository
|
||||
contentID content.ID
|
||||
expectedErr string
|
||||
}{
|
||||
{
|
||||
name: "success",
|
||||
rawRepo: func() repo.Repository {
|
||||
m := repomocks.NewMockRepository(t)
|
||||
m.On("ContentInfo", mock.Anything, mock.Anything).Return(content.Info{}, nil)
|
||||
return m
|
||||
}(),
|
||||
},
|
||||
{
|
||||
name: "error",
|
||||
rawRepo: func() repo.Repository {
|
||||
m := repomocks.NewMockRepository(t)
|
||||
m.On("ContentInfo", mock.Anything, mock.Anything).Return(content.Info{}, assert.AnError)
|
||||
return m
|
||||
}(),
|
||||
expectedErr: assert.AnError.Error(),
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
kr := &kopiaRepository{rawRepo: tc.rawRepo, logger: velerotest.NewLogger()}
|
||||
_, err := kr.ContentInfo(context.Background(), tc.contentID)
|
||||
if tc.expectedErr != "" {
|
||||
assert.EqualError(t, err, tc.expectedErr)
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetContent(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
rawRepo repo.Repository
|
||||
contentID content.ID
|
||||
expectedErr string
|
||||
}{
|
||||
{
|
||||
name: "invalid repo interface",
|
||||
rawRepo: repomocks.NewMockRepository(t),
|
||||
expectedErr: "invalid repo interface",
|
||||
},
|
||||
{
|
||||
name: "success",
|
||||
rawRepo: func() repo.Repository {
|
||||
m := &mockDirectRepository{}
|
||||
cr := &mockContentReader{}
|
||||
cr.On("GetContent", mock.Anything, mock.Anything).Return([]byte("test"), nil)
|
||||
m.On("ContentReader").Return(cr)
|
||||
return m
|
||||
}(),
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
kr := &kopiaRepository{rawRepo: tc.rawRepo, logger: velerotest.NewLogger()}
|
||||
_, err := kr.GetContent(context.Background(), tc.contentID)
|
||||
if tc.expectedErr != "" {
|
||||
assert.EqualError(t, err, tc.expectedErr)
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrefetchContents(t *testing.T) {
|
||||
mockRepo := repomocks.NewMockRepository(t)
|
||||
id, _ := content.ParseID("123")
|
||||
mockRepo.On("PrefetchContents", mock.Anything, mock.Anything, mock.Anything).Return([]content.ID{id})
|
||||
kr := &kopiaRepository{rawRepo: mockRepo, logger: velerotest.NewLogger()}
|
||||
res := kr.PrefetchContents(context.Background(), []content.ID{id}, "hint")
|
||||
assert.Equal(t, []content.ID{id}, res)
|
||||
}
|
||||
|
||||
func TestGetFlattenedEntries(t *testing.T) {
|
||||
kr := &kopiaRepository{logger: velerotest.NewLogger()}
|
||||
rawID := object.ID{}
|
||||
_, err := kr.getFlattenedEntries(context.Background(), rawID)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "object is not an indirect object")
|
||||
}
|
||||
|
||||
func TestNewObjectWriterEx(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
opt udmrepo.ObjectWriteOptions
|
||||
rawWriter *repomocks.MockRepositoryWriter
|
||||
expectedErr string
|
||||
}{
|
||||
{
|
||||
name: "block mode success without parent",
|
||||
opt: udmrepo.ObjectWriteOptions{
|
||||
AccessMode: udmrepo.ObjectDataAccessModeBlock,
|
||||
},
|
||||
rawWriter: repomocks.NewMockRepositoryWriter(t),
|
||||
},
|
||||
{
|
||||
name: "block mode with parent, invalid parent ID",
|
||||
opt: udmrepo.ObjectWriteOptions{
|
||||
AccessMode: udmrepo.ObjectDataAccessModeBlock,
|
||||
ParentObject: udmrepo.ID("invalid-parent"),
|
||||
},
|
||||
rawWriter: repomocks.NewMockRepositoryWriter(t),
|
||||
expectedErr: "error parsing parent object ID from invalid-parent: malformed content ID: \"invalid-parent\": invalid content hash: encoding/hex: invalid byte: U+0069 'i'",
|
||||
},
|
||||
{
|
||||
name: "block mode with parent, valid ID but failed to load index",
|
||||
opt: udmrepo.ObjectWriteOptions{
|
||||
AccessMode: udmrepo.ObjectDataAccessModeBlock,
|
||||
ParentObject: udmrepo.ID("I0123456789abcdef"),
|
||||
},
|
||||
rawWriter: repomocks.NewMockRepositoryWriter(t),
|
||||
expectedErr: "error getting parent object entries from I0123456789abcdef: unexpected content error: invalid repo interface",
|
||||
},
|
||||
{
|
||||
name: "file mode with parent",
|
||||
opt: udmrepo.ObjectWriteOptions{
|
||||
AccessMode: udmrepo.ObjectDataAccessModeFile,
|
||||
ParentObject: udmrepo.ID("some-parent"),
|
||||
},
|
||||
rawWriter: repomocks.NewMockRepositoryWriter(t),
|
||||
expectedErr: "parent object is only supported for block mode",
|
||||
},
|
||||
{
|
||||
name: "block mode success with async writes",
|
||||
opt: udmrepo.ObjectWriteOptions{
|
||||
AccessMode: udmrepo.ObjectDataAccessModeBlock,
|
||||
AsyncWrites: 4,
|
||||
},
|
||||
rawWriter: repomocks.NewMockRepositoryWriter(t),
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
kr := &kopiaRepository{logger: velerotest.NewLogger()}
|
||||
if tc.rawWriter != nil {
|
||||
kr.rawWriter = tc.rawWriter
|
||||
}
|
||||
|
||||
_, err := kr.NewObjectWriter(context.Background(), tc.opt)
|
||||
|
||||
if tc.expectedErr == "" {
|
||||
require.NoError(t, err)
|
||||
} else {
|
||||
require.EqualError(t, err, tc.expectedErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestKopiaObjectWriterEx_Write(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
setupWriter func(t *testing.T) *kopiaObjectWriterEx
|
||||
inputData []byte
|
||||
expectedErr string
|
||||
expectedLen int
|
||||
}{
|
||||
{
|
||||
name: "writer is closed",
|
||||
setupWriter: func(t *testing.T) *kopiaObjectWriterEx {
|
||||
t.Helper()
|
||||
return &kopiaObjectWriterEx{
|
||||
rawRepoWriter: nil,
|
||||
}
|
||||
},
|
||||
inputData: make([]byte, 1024),
|
||||
expectedErr: "object writer is closed or not open",
|
||||
},
|
||||
{
|
||||
name: "write error exists",
|
||||
setupWriter: func(t *testing.T) *kopiaObjectWriterEx {
|
||||
t.Helper()
|
||||
kow := &kopiaObjectWriterEx{
|
||||
rawRepoWriter: repomocks.NewMockRepositoryWriter(t),
|
||||
blockSize: 1024,
|
||||
}
|
||||
kow.saveWriteError(errors.New("previous error"))
|
||||
return kow
|
||||
},
|
||||
inputData: make([]byte, 1024),
|
||||
expectedErr: "error happened during writing object: previous error",
|
||||
},
|
||||
{
|
||||
name: "invalid length",
|
||||
setupWriter: func(t *testing.T) *kopiaObjectWriterEx {
|
||||
t.Helper()
|
||||
return &kopiaObjectWriterEx{
|
||||
rawRepoWriter: repomocks.NewMockRepositoryWriter(t),
|
||||
blockSize: 1024,
|
||||
}
|
||||
},
|
||||
inputData: make([]byte, 1023),
|
||||
expectedErr: "invalid length 1023",
|
||||
},
|
||||
{
|
||||
name: "success sync write",
|
||||
setupWriter: func(t *testing.T) *kopiaObjectWriterEx {
|
||||
t.Helper()
|
||||
mockRepoWriter := repomocks.NewMockRepositoryWriter(t)
|
||||
mockWriter := repomocks.NewWriter(t)
|
||||
|
||||
mockWriter.On("Write", mock.Anything).Return(1024, nil)
|
||||
mockWriter.On("Close").Return(nil)
|
||||
|
||||
id, _ := object.ParseID("I12345")
|
||||
mockWriter.On("Result").Return(id, nil)
|
||||
|
||||
mockRepoWriter.On("NewObjectWriter", mock.Anything, mock.Anything).Return(mockWriter)
|
||||
|
||||
return &kopiaObjectWriterEx{
|
||||
ctx: context.Background(),
|
||||
rawRepoWriter: mockRepoWriter,
|
||||
blockSize: 1024,
|
||||
logger: velerotest.NewLogger(),
|
||||
}
|
||||
},
|
||||
inputData: make([]byte, 1024),
|
||||
expectedLen: 1024,
|
||||
},
|
||||
{
|
||||
name: "success async write",
|
||||
setupWriter: func(t *testing.T) *kopiaObjectWriterEx {
|
||||
t.Helper()
|
||||
mockRepoWriter := repomocks.NewMockRepositoryWriter(t)
|
||||
mockWriter := repomocks.NewWriter(t)
|
||||
|
||||
mockWriter.On("Write", mock.Anything).Return(1024, nil)
|
||||
mockWriter.On("Close").Return(nil)
|
||||
|
||||
id, _ := object.ParseID("I12345")
|
||||
mockWriter.On("Result").Return(id, nil)
|
||||
|
||||
mockRepoWriter.On("NewObjectWriter", mock.Anything, mock.Anything).Return(mockWriter)
|
||||
|
||||
sem := make(chan struct{}, 1)
|
||||
buf := freelist.New(1024, 1024)
|
||||
|
||||
return &kopiaObjectWriterEx{
|
||||
ctx: context.Background(),
|
||||
rawRepoWriter: mockRepoWriter,
|
||||
blockSize: 1024,
|
||||
asyncWritesSem: sem,
|
||||
asyncBuffer: buf,
|
||||
logger: velerotest.NewLogger(),
|
||||
}
|
||||
},
|
||||
inputData: make([]byte, 1024),
|
||||
expectedLen: 1024,
|
||||
},
|
||||
{
|
||||
name: "success multiple blocks in one write",
|
||||
setupWriter: func(t *testing.T) *kopiaObjectWriterEx {
|
||||
t.Helper()
|
||||
mockRepoWriter := repomocks.NewMockRepositoryWriter(t)
|
||||
mockWriter := repomocks.NewWriter(t)
|
||||
|
||||
mockWriter.On("Write", mock.Anything).Return(1024, nil)
|
||||
mockWriter.On("Close").Return(nil)
|
||||
|
||||
id, _ := object.ParseID("I12345")
|
||||
mockWriter.On("Result").Return(id, nil)
|
||||
|
||||
mockRepoWriter.On("NewObjectWriter", mock.Anything, mock.Anything).Return(mockWriter)
|
||||
|
||||
return &kopiaObjectWriterEx{
|
||||
ctx: context.Background(),
|
||||
rawRepoWriter: mockRepoWriter,
|
||||
blockSize: 1024,
|
||||
logger: velerotest.NewLogger(),
|
||||
}
|
||||
},
|
||||
inputData: make([]byte, 2048),
|
||||
expectedLen: 2048,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
kow := tc.setupWriter(t)
|
||||
l, err := kow.Write(tc.inputData)
|
||||
|
||||
if kow.asyncWritesSem != nil {
|
||||
kow.asyncWritesGroup.Wait()
|
||||
}
|
||||
|
||||
if tc.expectedErr != "" {
|
||||
assert.EqualError(t, err, tc.expectedErr)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.expectedLen, l)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestKopiaObjectWriterEx_Result(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
setupWriter func(t *testing.T) *kopiaObjectWriterEx
|
||||
expectedErr string
|
||||
expectedID udmrepo.ID
|
||||
}{
|
||||
{
|
||||
name: "write error exists",
|
||||
setupWriter: func(t *testing.T) *kopiaObjectWriterEx {
|
||||
t.Helper()
|
||||
kow := &kopiaObjectWriterEx{}
|
||||
kow.saveWriteError(errors.New("async write failed"))
|
||||
return kow
|
||||
},
|
||||
expectedErr: "error happened during writing object: async write failed",
|
||||
},
|
||||
{
|
||||
name: "writer closed",
|
||||
setupWriter: func(t *testing.T) *kopiaObjectWriterEx {
|
||||
t.Helper()
|
||||
kow := &kopiaObjectWriterEx{
|
||||
rawRepoWriter: nil,
|
||||
}
|
||||
return kow
|
||||
},
|
||||
expectedErr: "error to write indirect object: object writer is closed or not open",
|
||||
},
|
||||
{
|
||||
name: "success",
|
||||
setupWriter: func(t *testing.T) *kopiaObjectWriterEx {
|
||||
t.Helper()
|
||||
mockRepoWriter := repomocks.NewMockRepositoryWriter(t)
|
||||
mockWriter := repomocks.NewWriter(t)
|
||||
|
||||
mockWriter.On("Write", mock.Anything).Return(100, nil)
|
||||
mockWriter.On("Close").Return(nil)
|
||||
|
||||
id, _ := object.ParseID("Iabcdef")
|
||||
mockWriter.On("Result").Return(id, nil)
|
||||
|
||||
mockRepoWriter.On("NewObjectWriter", mock.Anything, mock.Anything).Return(mockWriter)
|
||||
|
||||
return &kopiaObjectWriterEx{
|
||||
ctx: context.Background(),
|
||||
rawRepoWriter: mockRepoWriter,
|
||||
logger: velerotest.NewLogger(),
|
||||
}
|
||||
},
|
||||
expectedID: udmrepo.ID("IIabcdef"),
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
kow := tc.setupWriter(t)
|
||||
id, err := kow.Result()
|
||||
|
||||
if tc.expectedErr != "" {
|
||||
assert.EqualError(t, err, tc.expectedErr)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.expectedID, id)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestKopiaObjectWriterEx_Close(t *testing.T) {
|
||||
kow := &kopiaObjectWriterEx{}
|
||||
err := kow.Close()
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestKopiaObjectWriterEx_ConcurrentWrite(t *testing.T) {
|
||||
mockRepoWriter := repomocks.NewMockRepositoryWriter(t)
|
||||
mockWriter := repomocks.NewWriter(t)
|
||||
|
||||
mockWriter.On("Write", mock.Anything).Return(1024, nil)
|
||||
mockWriter.On("Close").Return(nil)
|
||||
|
||||
id, _ := object.ParseID("I12345")
|
||||
mockWriter.On("Result").Return(id, nil)
|
||||
|
||||
mockRepoWriter.On("NewObjectWriter", mock.Anything, mock.Anything).Return(mockWriter)
|
||||
|
||||
kow := &kopiaObjectWriterEx{
|
||||
ctx: context.Background(),
|
||||
rawRepoWriter: mockRepoWriter,
|
||||
blockSize: 1024,
|
||||
logger: velerotest.NewLogger(),
|
||||
}
|
||||
|
||||
numGoroutines := 10
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for i := 0; i < numGoroutines; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
data := make([]byte, 1024)
|
||||
l, err := kow.Write(data)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 1024, l)
|
||||
}()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
assert.Len(t, kow.entries, numGoroutines)
|
||||
}
|
||||
|
||||
func TestKopiaObjectWriterEx_ConcurrentAsyncWrite(t *testing.T) {
|
||||
mockRepoWriter := repomocks.NewMockRepositoryWriter(t)
|
||||
mockWriter := repomocks.NewWriter(t)
|
||||
|
||||
mockWriter.On("Write", mock.Anything).Return(1024, nil)
|
||||
mockWriter.On("Close").Return(nil)
|
||||
|
||||
id, _ := object.ParseID("I12345")
|
||||
mockWriter.On("Result").Return(id, nil)
|
||||
|
||||
mockRepoWriter.On("NewObjectWriter", mock.Anything, mock.Anything).Return(mockWriter)
|
||||
|
||||
sem := make(chan struct{}, 5)
|
||||
buf := freelist.New(5*1024, 1024)
|
||||
|
||||
kow := &kopiaObjectWriterEx{
|
||||
ctx: context.Background(),
|
||||
rawRepoWriter: mockRepoWriter,
|
||||
blockSize: 1024,
|
||||
asyncWritesSem: sem,
|
||||
asyncBuffer: buf,
|
||||
logger: velerotest.NewLogger(),
|
||||
}
|
||||
|
||||
numGoroutines := 10
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for i := 0; i < numGoroutines; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
data := make([]byte, 1024)
|
||||
l, err := kow.Write(data)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 1024, l)
|
||||
}()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
kow.asyncWritesGroup.Wait()
|
||||
|
||||
assert.Len(t, kow.entries, numGoroutines)
|
||||
}
|
||||
|
||||
func TestKopiaObjectWriterEx_MultipleWrites(t *testing.T) {
|
||||
mockRepoWriter := repomocks.NewMockRepositoryWriter(t)
|
||||
mockWriter := repomocks.NewWriter(t)
|
||||
|
||||
// Since we are writing 3 blocks, Write should be called 3 times and Close 3 times
|
||||
mockWriter.On("Write", mock.Anything).Return(1024, nil)
|
||||
mockWriter.On("Close").Return(nil)
|
||||
|
||||
id, _ := object.ParseID("I12345")
|
||||
mockWriter.On("Result").Return(id, nil)
|
||||
|
||||
mockRepoWriter.On("NewObjectWriter", mock.Anything, mock.Anything).Return(mockWriter)
|
||||
|
||||
kow := &kopiaObjectWriterEx{
|
||||
ctx: context.Background(),
|
||||
rawRepoWriter: mockRepoWriter,
|
||||
blockSize: 1024,
|
||||
logger: velerotest.NewLogger(),
|
||||
}
|
||||
|
||||
// Write 1st block
|
||||
l, err := kow.Write(make([]byte, 1024))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1024, l)
|
||||
|
||||
// Write 2nd and 3rd block
|
||||
l, err = kow.Write(make([]byte, 2048))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 2048, l)
|
||||
|
||||
// In the end we expect 3 blocks to be tracked in `kow.entries`
|
||||
assert.Len(t, kow.entries, 3)
|
||||
assert.Equal(t, int64(0), kow.entries[0].Start)
|
||||
assert.Equal(t, int64(1024), kow.entries[1].Start)
|
||||
assert.Equal(t, int64(2048), kow.entries[2].Start)
|
||||
}
|
||||
@@ -17,15 +17,19 @@ limitations under the License.
|
||||
package kopialib
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"math"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kopia/kopia/fs"
|
||||
"github.com/kopia/kopia/repo"
|
||||
"github.com/kopia/kopia/repo/manifest"
|
||||
"github.com/kopia/kopia/repo/object"
|
||||
"github.com/kopia/kopia/snapshot"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -1282,3 +1286,549 @@ func TestIsReady(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type fakeObjectReader struct {
|
||||
*bytes.Reader
|
||||
}
|
||||
|
||||
func (f *fakeObjectReader) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeObjectReader) Length() int64 {
|
||||
return int64(f.Reader.Len())
|
||||
}
|
||||
|
||||
func TestWriteMetadata(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
rawWriter *repomocks.MockRepositoryWriter
|
||||
rawObjWriter *repomocks.Writer
|
||||
meta *udmrepo.Metadata
|
||||
rawWriterRetErr error
|
||||
expectedErr string
|
||||
}{
|
||||
{
|
||||
name: "raw writer is nil",
|
||||
expectedErr: "repo writer is closed or not open",
|
||||
},
|
||||
{
|
||||
name: "invalid object id",
|
||||
rawWriter: repomocks.NewMockRepositoryWriter(t),
|
||||
meta: &udmrepo.Metadata{
|
||||
SubObjects: []udmrepo.ObjectMetadata{
|
||||
{
|
||||
ID: "fake-id",
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedErr: "error parsing object ID from {fake-id 0 0 0001-01-01 00:00:00 +0000 UTC 0 0 0}: malformed content ID: \"fake-id\": invalid content prefix",
|
||||
},
|
||||
{
|
||||
name: "write dir manifest fail",
|
||||
rawWriter: repomocks.NewMockRepositoryWriter(t),
|
||||
rawObjWriter: repomocks.NewWriter(t),
|
||||
meta: &udmrepo.Metadata{
|
||||
SubObjects: []udmrepo.ObjectMetadata{
|
||||
{
|
||||
ID: "I123456",
|
||||
},
|
||||
},
|
||||
},
|
||||
rawWriterRetErr: errors.New("fake-write-error"),
|
||||
expectedErr: "error writing dir manifest: : unable to encode directory JSON: fake-write-error",
|
||||
},
|
||||
{
|
||||
name: "succeed",
|
||||
rawWriter: repomocks.NewMockRepositoryWriter(t),
|
||||
rawObjWriter: repomocks.NewWriter(t),
|
||||
meta: &udmrepo.Metadata{
|
||||
SubObjects: []udmrepo.ObjectMetadata{
|
||||
{
|
||||
ID: "I123456",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
kr := &kopiaRepository{}
|
||||
|
||||
if tc.rawWriter != nil {
|
||||
if tc.rawObjWriter != nil {
|
||||
tc.rawWriter.On("NewObjectWriter", mock.Anything, mock.Anything).Return(tc.rawObjWriter)
|
||||
if tc.rawWriterRetErr != nil {
|
||||
tc.rawObjWriter.On("Write", mock.Anything).Return(0, tc.rawWriterRetErr)
|
||||
tc.rawObjWriter.On("Close").Return(nil)
|
||||
} else {
|
||||
tc.rawObjWriter.On("Write", mock.Anything).Return(10, nil)
|
||||
tc.rawObjWriter.On("Result").Return(object.ID{}, nil)
|
||||
tc.rawObjWriter.On("Close").Return(nil)
|
||||
}
|
||||
}
|
||||
kr.rawWriter = tc.rawWriter
|
||||
}
|
||||
|
||||
_, err := kr.WriteMetadata(t.Context(), tc.meta, udmrepo.ObjectWriteOptions{})
|
||||
|
||||
if tc.expectedErr == "" {
|
||||
assert.NoError(t, err)
|
||||
} else {
|
||||
assert.EqualError(t, err, tc.expectedErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadMetadata(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
rawRepo *repomocks.MockRepository
|
||||
objectID udmrepo.ID
|
||||
openErr error
|
||||
readData []byte
|
||||
expectedErr string
|
||||
expected *udmrepo.Metadata
|
||||
}{
|
||||
{
|
||||
name: "open object fail",
|
||||
rawRepo: repomocks.NewMockRepository(t),
|
||||
objectID: "I123456",
|
||||
openErr: errors.New("fake-open-error"),
|
||||
expectedErr: "error to open metadata object I123456: error to open object: fake-open-error",
|
||||
},
|
||||
{
|
||||
name: "invalid json",
|
||||
rawRepo: repomocks.NewMockRepository(t),
|
||||
objectID: "I123456",
|
||||
readData: []byte("invalid json"),
|
||||
expectedErr: "unable to parse directory object: invalid character 'i' looking for beginning of value",
|
||||
},
|
||||
{
|
||||
name: "succeed",
|
||||
rawRepo: repomocks.NewMockRepository(t),
|
||||
objectID: "I123456",
|
||||
readData: []byte(`{"stream":"kopia:directory","entries":[{"name":"file1","type":"f","mode":"0644","size":100,"uid":1000,"gid":1000,"mtime":"2023-01-01T00:00:00Z","obj":"I123456"}]}`),
|
||||
expected: &udmrepo.Metadata{
|
||||
SubObjects: []udmrepo.ObjectMetadata{
|
||||
{
|
||||
ID: "I123456",
|
||||
Name: "file1",
|
||||
Type: udmrepo.ObjectDataTypeData,
|
||||
Size: 100,
|
||||
ModTime: time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC).Local(),
|
||||
Permissions: 420,
|
||||
UserID: 1000,
|
||||
GroupID: 1000,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
kr := &kopiaRepository{}
|
||||
|
||||
if tc.rawRepo != nil {
|
||||
if tc.openErr != nil {
|
||||
tc.rawRepo.On("OpenObject", mock.Anything, mock.Anything).Return(nil, tc.openErr)
|
||||
} else {
|
||||
reader := &fakeObjectReader{Reader: bytes.NewReader(tc.readData)}
|
||||
tc.rawRepo.On("OpenObject", mock.Anything, mock.Anything).Return(reader, nil)
|
||||
}
|
||||
kr.rawRepo = tc.rawRepo
|
||||
}
|
||||
|
||||
meta, err := kr.ReadMetadata(t.Context(), tc.objectID)
|
||||
|
||||
if tc.expectedErr == "" {
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.expected, meta)
|
||||
} else {
|
||||
assert.EqualError(t, err, tc.expectedErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSaveSnapshot(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
rawWriter *repomocks.MockRepositoryWriter
|
||||
snap udmrepo.Snapshot
|
||||
rawWriterRetErr error
|
||||
rawWriterRetID manifest.ID
|
||||
setWriterMock bool
|
||||
expectedErr string
|
||||
expectedID udmrepo.ID
|
||||
}{
|
||||
{
|
||||
name: "raw writer is nil",
|
||||
expectedErr: "repo writer is closed or not open",
|
||||
},
|
||||
{
|
||||
name: "invalid snapshot source",
|
||||
rawWriter: repomocks.NewMockRepositoryWriter(t),
|
||||
snap: udmrepo.Snapshot{
|
||||
Source: "",
|
||||
},
|
||||
expectedErr: "invalid snapshot source",
|
||||
},
|
||||
{
|
||||
name: "invalid root object id",
|
||||
rawWriter: repomocks.NewMockRepositoryWriter(t),
|
||||
snap: udmrepo.Snapshot{
|
||||
Source: "fake-source",
|
||||
RootObject: udmrepo.ObjectMetadata{ID: "fake-id"},
|
||||
},
|
||||
expectedErr: "error parsing root object ID fake-id: malformed content ID: \"fake-id\": invalid content prefix",
|
||||
},
|
||||
{
|
||||
name: "save snapshot fail",
|
||||
rawWriter: repomocks.NewMockRepositoryWriter(t),
|
||||
snap: udmrepo.Snapshot{
|
||||
Source: "fake-source",
|
||||
RootObject: udmrepo.ObjectMetadata{ID: "I123456"},
|
||||
},
|
||||
rawWriterRetErr: errors.New("fake-save-error"),
|
||||
setWriterMock: true,
|
||||
expectedErr: "error saving snapshot: error putting manifest: fake-save-error",
|
||||
},
|
||||
{
|
||||
name: "succeed",
|
||||
rawWriter: repomocks.NewMockRepositoryWriter(t),
|
||||
snap: udmrepo.Snapshot{
|
||||
Source: "fake-source",
|
||||
RootObject: udmrepo.ObjectMetadata{ID: "I123456"},
|
||||
},
|
||||
rawWriterRetID: manifest.ID("fake-manifest-id"),
|
||||
setWriterMock: true,
|
||||
expectedID: udmrepo.ID("fake-manifest-id"),
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
kr := &kopiaRepository{}
|
||||
|
||||
if tc.rawWriter != nil {
|
||||
if tc.setWriterMock {
|
||||
tc.rawWriter.On("PutManifest", mock.Anything, mock.Anything, mock.Anything).Return(tc.rawWriterRetID, tc.rawWriterRetErr)
|
||||
}
|
||||
kr.rawWriter = tc.rawWriter
|
||||
}
|
||||
|
||||
id, err := kr.SaveSnapshot(t.Context(), tc.snap)
|
||||
|
||||
if tc.expectedErr == "" {
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.expectedID, id)
|
||||
} else {
|
||||
assert.EqualError(t, err, tc.expectedErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetSnapshot(t *testing.T) {
|
||||
expectedTime := time.Now()
|
||||
rawObjID, _ := object.ParseID("I123456")
|
||||
|
||||
mockMani := &snapshot.Manifest{
|
||||
Source: snapshot.SourceInfo{Path: "fake-source"},
|
||||
Description: "fake-desc",
|
||||
StartTime: fs.UTCTimestampFromTime(expectedTime),
|
||||
EndTime: fs.UTCTimestampFromTime(expectedTime.Add(time.Minute)),
|
||||
RootEntry: &snapshot.DirEntry{
|
||||
ObjectID: rawObjID,
|
||||
},
|
||||
Tags: map[string]string{"tag1": "val1"},
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
rawRepo *repomocks.MockRepository
|
||||
snapshotID udmrepo.ID
|
||||
rawRepoRetErr error
|
||||
setRepoMock bool
|
||||
expectedErr string
|
||||
expectedSnap udmrepo.Snapshot
|
||||
}{
|
||||
{
|
||||
name: "get snapshot fail",
|
||||
rawRepo: repomocks.NewMockRepository(t),
|
||||
snapshotID: udmrepo.ID("fake-id"),
|
||||
rawRepoRetErr: errors.New("fake-get-error"),
|
||||
setRepoMock: true,
|
||||
expectedErr: "error getting snapshot manifest: unable to find manifest entries: fake-get-error",
|
||||
},
|
||||
{
|
||||
name: "succeed",
|
||||
rawRepo: repomocks.NewMockRepository(t),
|
||||
snapshotID: udmrepo.ID("fake-id"),
|
||||
setRepoMock: true,
|
||||
expectedSnap: udmrepo.Snapshot{
|
||||
Source: "fake-source",
|
||||
Description: "fake-desc",
|
||||
StartTime: mockMani.StartTime.ToTime(),
|
||||
EndTime: mockMani.EndTime.ToTime(),
|
||||
RootObject: udmrepo.ObjectMetadata{
|
||||
ID: udmrepo.ID("I123456"),
|
||||
Type: udmrepo.ObjectDataTypeMetadata,
|
||||
Size: mockMani.RootEntry.FileSize,
|
||||
ModTime: mockMani.RootEntry.ModTime.ToTime(),
|
||||
Permissions: int(mockMani.RootEntry.Permissions),
|
||||
UserID: mockMani.RootEntry.UserID,
|
||||
GroupID: mockMani.RootEntry.GroupID,
|
||||
},
|
||||
Tags: map[string]string{"tag1": "val1"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
kr := &kopiaRepository{}
|
||||
|
||||
if tc.rawRepo != nil {
|
||||
if tc.setRepoMock {
|
||||
tc.rawRepo.On("GetManifest", mock.Anything, mock.Anything, mock.Anything).Return(&manifest.EntryMetadata{
|
||||
Labels: map[string]string{
|
||||
manifest.TypeLabelKey: snapshot.ManifestType,
|
||||
},
|
||||
}, tc.rawRepoRetErr).Run(func(args mock.Arguments) {
|
||||
if tc.rawRepoRetErr == nil {
|
||||
payload := args.Get(2)
|
||||
if ptr, ok := payload.(*snapshot.Manifest); ok {
|
||||
*ptr = *mockMani
|
||||
} else {
|
||||
b, _ := json.Marshal(mockMani)
|
||||
json.Unmarshal(b, payload)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
kr.rawRepo = tc.rawRepo
|
||||
}
|
||||
|
||||
snap, err := kr.GetSnapshot(t.Context(), tc.snapshotID)
|
||||
|
||||
if tc.expectedErr == "" {
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.expectedSnap, snap)
|
||||
} else {
|
||||
assert.EqualError(t, err, tc.expectedErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteSnapshot(t *testing.T) {
|
||||
expectedTime := time.Now()
|
||||
rawObjID, _ := object.ParseID("I123456")
|
||||
|
||||
mockMani := &snapshot.Manifest{
|
||||
Source: snapshot.SourceInfo{Path: "fake-source"},
|
||||
Description: "fake-desc",
|
||||
StartTime: fs.UTCTimestampFromTime(expectedTime),
|
||||
EndTime: fs.UTCTimestampFromTime(expectedTime.Add(time.Minute)),
|
||||
RootEntry: &snapshot.DirEntry{
|
||||
ObjectID: rawObjID,
|
||||
},
|
||||
Tags: map[string]string{"tag1": "val1"},
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
rawRepo *repomocks.MockRepository
|
||||
rawWriter *repomocks.MockRepositoryWriter
|
||||
snapshotID udmrepo.ID
|
||||
rawRepoRetErr error
|
||||
rawWriterRetErr error
|
||||
setRepoMock bool
|
||||
setWriterMock bool
|
||||
expectedErr string
|
||||
}{
|
||||
{
|
||||
name: "get snapshot fail",
|
||||
rawRepo: repomocks.NewMockRepository(t),
|
||||
snapshotID: udmrepo.ID("fake-id"),
|
||||
rawRepoRetErr: errors.New("fake-get-error"),
|
||||
setRepoMock: true,
|
||||
expectedErr: "error getting snapshot: error getting snapshot manifest: unable to find manifest entries: fake-get-error",
|
||||
},
|
||||
{
|
||||
name: "delete manifest fail",
|
||||
rawRepo: repomocks.NewMockRepository(t),
|
||||
rawWriter: repomocks.NewMockRepositoryWriter(t),
|
||||
snapshotID: udmrepo.ID("fake-id"),
|
||||
rawWriterRetErr: errors.New("fake-delete-error"),
|
||||
setRepoMock: true,
|
||||
setWriterMock: true,
|
||||
expectedErr: "error to delete manifest: fake-delete-error",
|
||||
},
|
||||
{
|
||||
name: "succeed",
|
||||
rawRepo: repomocks.NewMockRepository(t),
|
||||
rawWriter: repomocks.NewMockRepositoryWriter(t),
|
||||
snapshotID: udmrepo.ID("fake-id"),
|
||||
setRepoMock: true,
|
||||
setWriterMock: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
kr := &kopiaRepository{}
|
||||
|
||||
if tc.rawRepo != nil {
|
||||
if tc.setRepoMock {
|
||||
tc.rawRepo.On("GetManifest", mock.Anything, mock.Anything, mock.Anything).Return(&manifest.EntryMetadata{
|
||||
Labels: map[string]string{
|
||||
manifest.TypeLabelKey: snapshot.ManifestType,
|
||||
},
|
||||
}, tc.rawRepoRetErr).Run(func(args mock.Arguments) {
|
||||
if tc.rawRepoRetErr == nil {
|
||||
payload := args.Get(2)
|
||||
if ptr, ok := payload.(*snapshot.Manifest); ok {
|
||||
*ptr = *mockMani
|
||||
} else {
|
||||
b, _ := json.Marshal(mockMani)
|
||||
json.Unmarshal(b, payload)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
kr.rawRepo = tc.rawRepo
|
||||
}
|
||||
|
||||
if tc.rawWriter != nil {
|
||||
if tc.setWriterMock {
|
||||
tc.rawWriter.On("DeleteManifest", mock.Anything, mock.Anything).Return(tc.rawWriterRetErr)
|
||||
}
|
||||
kr.rawWriter = tc.rawWriter
|
||||
}
|
||||
|
||||
err := kr.DeleteSnapshot(t.Context(), tc.snapshotID)
|
||||
|
||||
if tc.expectedErr == "" {
|
||||
require.NoError(t, err)
|
||||
} else {
|
||||
assert.EqualError(t, err, tc.expectedErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestListSnapshot(t *testing.T) {
|
||||
expectedTime := time.Now()
|
||||
rawObjID, _ := object.ParseID("I123456")
|
||||
|
||||
mockMani := &snapshot.Manifest{
|
||||
Source: snapshot.SourceInfo{Path: "fake-source"},
|
||||
Description: "fake-desc",
|
||||
StartTime: fs.UTCTimestampFromTime(expectedTime),
|
||||
EndTime: fs.UTCTimestampFromTime(expectedTime.Add(time.Minute)),
|
||||
RootEntry: &snapshot.DirEntry{
|
||||
ObjectID: rawObjID,
|
||||
FileSize: 100,
|
||||
ModTime: fs.UTCTimestampFromTime(expectedTime),
|
||||
Permissions: 0o644,
|
||||
UserID: 1000,
|
||||
GroupID: 1000,
|
||||
},
|
||||
Tags: map[string]string{"tag1": "val1"},
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
rawRepo *repomocks.MockRepository
|
||||
source string
|
||||
findRetErr error
|
||||
setRepoMock bool
|
||||
expectedErr string
|
||||
expectedSnaps []udmrepo.Snapshot
|
||||
}{
|
||||
{
|
||||
name: "find manifest fail",
|
||||
rawRepo: repomocks.NewMockRepository(t),
|
||||
source: "fake-source",
|
||||
findRetErr: errors.New("fake-find-error"),
|
||||
setRepoMock: true,
|
||||
expectedErr: "error listing snapshot manifest for source fake-source: unable to find manifest entries: fake-find-error",
|
||||
},
|
||||
{
|
||||
name: "succeed",
|
||||
rawRepo: repomocks.NewMockRepository(t),
|
||||
source: "fake-source",
|
||||
setRepoMock: true,
|
||||
expectedSnaps: []udmrepo.Snapshot{
|
||||
{
|
||||
Source: "fake-source",
|
||||
Description: "fake-desc",
|
||||
StartTime: mockMani.StartTime.ToTime(),
|
||||
EndTime: mockMani.EndTime.ToTime(),
|
||||
RootObject: udmrepo.ObjectMetadata{
|
||||
ID: udmrepo.ID("I123456"),
|
||||
Type: udmrepo.ObjectDataTypeMetadata,
|
||||
Size: mockMani.RootEntry.FileSize,
|
||||
ModTime: mockMani.RootEntry.ModTime.ToTime(),
|
||||
Permissions: int(mockMani.RootEntry.Permissions),
|
||||
UserID: mockMani.RootEntry.UserID,
|
||||
GroupID: mockMani.RootEntry.GroupID,
|
||||
},
|
||||
Tags: map[string]string{"tag1": "val1"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
kr := &kopiaRepository{}
|
||||
|
||||
if tc.rawRepo != nil {
|
||||
if tc.setRepoMock {
|
||||
tc.rawRepo.On("FindManifests", mock.Anything, mock.Anything).Return([]*manifest.EntryMetadata{
|
||||
{
|
||||
ID: "fake-id",
|
||||
Labels: map[string]string{
|
||||
manifest.TypeLabelKey: snapshot.ManifestType,
|
||||
"hostname": udmrepo.GetRepoDomain(),
|
||||
"username": udmrepo.GetRepoUser(),
|
||||
"path": tc.source,
|
||||
},
|
||||
},
|
||||
}, tc.findRetErr)
|
||||
|
||||
tc.rawRepo.On("GetManifest", mock.Anything, mock.Anything, mock.Anything).Return(&manifest.EntryMetadata{
|
||||
Labels: map[string]string{
|
||||
manifest.TypeLabelKey: snapshot.ManifestType,
|
||||
},
|
||||
}, nil).Run(func(args mock.Arguments) {
|
||||
payload := args.Get(2)
|
||||
if ptr, ok := payload.(*snapshot.Manifest); ok {
|
||||
*ptr = *mockMani
|
||||
} else {
|
||||
b, _ := json.Marshal(mockMani)
|
||||
json.Unmarshal(b, payload)
|
||||
}
|
||||
}).Maybe()
|
||||
}
|
||||
kr.rawRepo = tc.rawRepo
|
||||
}
|
||||
|
||||
snaps, err := kr.ListSnapshot(t.Context(), tc.source)
|
||||
|
||||
if tc.expectedErr == "" {
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.expectedSnaps, snaps)
|
||||
} else {
|
||||
assert.EqualError(t, err, tc.expectedErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -562,6 +562,74 @@ func (_c *BackupRepo_GetSnapshot_Call) RunAndReturn(run func(ctx context.Context
|
||||
return _c
|
||||
}
|
||||
|
||||
// ListSnapshot provides a mock function for the type BackupRepo
|
||||
func (_mock *BackupRepo) ListSnapshot(ctx context.Context, source string) ([]udmrepo.Snapshot, error) {
|
||||
ret := _mock.Called(ctx, source)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for ListSnapshot")
|
||||
}
|
||||
|
||||
var r0 []udmrepo.Snapshot
|
||||
var r1 error
|
||||
if returnFunc, ok := ret.Get(0).(func(context.Context, string) ([]udmrepo.Snapshot, error)); ok {
|
||||
return returnFunc(ctx, source)
|
||||
}
|
||||
if returnFunc, ok := ret.Get(0).(func(context.Context, string) []udmrepo.Snapshot); ok {
|
||||
r0 = returnFunc(ctx, source)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).([]udmrepo.Snapshot)
|
||||
}
|
||||
}
|
||||
if returnFunc, ok := ret.Get(1).(func(context.Context, string) error); ok {
|
||||
r1 = returnFunc(ctx, source)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// BackupRepo_ListSnapshot_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListSnapshot'
|
||||
type BackupRepo_ListSnapshot_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// ListSnapshot is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - source string
|
||||
func (_e *BackupRepo_Expecter) ListSnapshot(ctx interface{}, source interface{}) *BackupRepo_ListSnapshot_Call {
|
||||
return &BackupRepo_ListSnapshot_Call{Call: _e.mock.On("ListSnapshot", ctx, source)}
|
||||
}
|
||||
|
||||
func (_c *BackupRepo_ListSnapshot_Call) Run(run func(ctx context.Context, source string)) *BackupRepo_ListSnapshot_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
var arg0 context.Context
|
||||
if args[0] != nil {
|
||||
arg0 = args[0].(context.Context)
|
||||
}
|
||||
var arg1 string
|
||||
if args[1] != nil {
|
||||
arg1 = args[1].(string)
|
||||
}
|
||||
run(
|
||||
arg0,
|
||||
arg1,
|
||||
)
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *BackupRepo_ListSnapshot_Call) Return(snapshots []udmrepo.Snapshot, err error) *BackupRepo_ListSnapshot_Call {
|
||||
_c.Call.Return(snapshots, err)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *BackupRepo_ListSnapshot_Call) RunAndReturn(run func(ctx context.Context, source string) ([]udmrepo.Snapshot, error)) *BackupRepo_ListSnapshot_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// NewObjectWriter provides a mock function for the type BackupRepo
|
||||
func (_mock *BackupRepo) NewObjectWriter(ctx context.Context, opt udmrepo.ObjectWriteOptions) (udmrepo.ObjectWriter, error) {
|
||||
ret := _mock.Called(ctx, opt)
|
||||
|
||||
@@ -77,15 +77,19 @@ type AdvancedFeatureInfo struct {
|
||||
}
|
||||
|
||||
type ObjectMetadata struct {
|
||||
ID ID
|
||||
Type int // OBJECT_DATA_TYPE_*
|
||||
Size int64
|
||||
ID ID
|
||||
Name string
|
||||
Type int // OBJECT_DATA_TYPE_*
|
||||
Size int64
|
||||
ModTime time.Time
|
||||
Permissions int
|
||||
UserID uint32
|
||||
GroupID uint32
|
||||
}
|
||||
|
||||
type Metadata struct {
|
||||
SubObjects []ObjectMetadata // For dir metadata only, the sub objects in this dir.
|
||||
ExtraDataLen int // Extra data associated to this metadata.
|
||||
ExtraData []byte
|
||||
SubObjects []ObjectMetadata
|
||||
Summary string
|
||||
}
|
||||
|
||||
type Snapshot struct {
|
||||
@@ -94,7 +98,8 @@ type Snapshot struct {
|
||||
StartTime time.Time
|
||||
EndTime time.Time
|
||||
Tags map[string]string
|
||||
RootObject ID
|
||||
TotalSize int64
|
||||
RootObject ObjectMetadata
|
||||
}
|
||||
|
||||
// BackupRepoService is used to initialize, open or maintain a backup repository
|
||||
@@ -177,6 +182,9 @@ type BackupRepo interface {
|
||||
// DeleteSnapshot deletes a repo snapshot
|
||||
DeleteSnapshot(ctx context.Context, id ID) error
|
||||
|
||||
// ListSnapshot lists all snapshots in repo for the given source
|
||||
ListSnapshot(ctx context.Context, source string) ([]Snapshot, error)
|
||||
|
||||
// Close closes the backup repository
|
||||
Close(ctx context.Context) error
|
||||
}
|
||||
|
||||
@@ -1,104 +0,0 @@
|
||||
/*
|
||||
Copyright 2020 the Velero contributors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package restic
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Command represents a restic command.
|
||||
type Command struct {
|
||||
Command string
|
||||
RepoIdentifier string
|
||||
PasswordFile string
|
||||
CACertFile string
|
||||
Dir string
|
||||
Args []string
|
||||
ExtraFlags []string
|
||||
Env []string
|
||||
}
|
||||
|
||||
func (c *Command) RepoName() string {
|
||||
if c.RepoIdentifier == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
return c.RepoIdentifier[strings.LastIndex(c.RepoIdentifier, "/")+1:]
|
||||
}
|
||||
|
||||
// StringSlice returns the command as a slice of strings.
|
||||
func (c *Command) StringSlice() []string {
|
||||
res := []string{"restic"}
|
||||
|
||||
res = append(res, c.Command, repoFlag(c.RepoIdentifier))
|
||||
if c.PasswordFile != "" {
|
||||
res = append(res, passwordFlag(c.PasswordFile))
|
||||
}
|
||||
if c.CACertFile != "" {
|
||||
res = append(res, cacertFlag(c.CACertFile))
|
||||
}
|
||||
|
||||
// If VELERO_SCRATCH_DIR is defined, put the restic cache within it. If not,
|
||||
// allow restic to choose the location. This makes running either in-cluster
|
||||
// or local (dev) work properly.
|
||||
if scratch := os.Getenv("VELERO_SCRATCH_DIR"); scratch != "" {
|
||||
res = append(res, cacheDirFlag(filepath.Join(scratch, ".cache", "restic")))
|
||||
}
|
||||
|
||||
res = append(res, c.Args...)
|
||||
res = append(res, c.ExtraFlags...)
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
// String returns the command as a string.
|
||||
func (c *Command) String() string {
|
||||
return strings.Join(c.StringSlice(), " ")
|
||||
}
|
||||
|
||||
// Cmd returns an exec.Cmd for the command.
|
||||
func (c *Command) Cmd() *exec.Cmd {
|
||||
parts := c.StringSlice()
|
||||
cmd := exec.Command(parts[0], parts[1:]...) //nolint:gosec,noctx // Internal call. No need to check the parameter. No to add context for deprecated Restic.
|
||||
cmd.Dir = c.Dir
|
||||
|
||||
if len(c.Env) > 0 {
|
||||
cmd.Env = c.Env
|
||||
}
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
func repoFlag(repoIdentifier string) string {
|
||||
return fmt.Sprintf("--repo=%s", repoIdentifier)
|
||||
}
|
||||
|
||||
func passwordFlag(file string) string {
|
||||
return fmt.Sprintf("--password-file=%s", file)
|
||||
}
|
||||
|
||||
func cacheDirFlag(dir string) string {
|
||||
return fmt.Sprintf("--cache-dir=%s", dir)
|
||||
}
|
||||
|
||||
func cacertFlag(path string) string {
|
||||
return fmt.Sprintf("--cacert=%s", path)
|
||||
}
|
||||
@@ -1,125 +0,0 @@
|
||||
/*
|
||||
Copyright 2018, 2019 the Velero contributors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package restic
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// BackupCommand returns a Command for running a restic backup.
|
||||
func BackupCommand(repoIdentifier, passwordFile, path string, tags map[string]string) *Command {
|
||||
// --host flag is provided with a generic value because restic uses the host
|
||||
// to find a parent snapshot, and by default it will be the name of the daemonset pod
|
||||
// where the `restic backup` command is run. If this pod is recreated, we want to continue
|
||||
// taking incremental backups rather than triggering a full one due to a new pod name.
|
||||
|
||||
return &Command{
|
||||
Command: "backup",
|
||||
RepoIdentifier: repoIdentifier,
|
||||
PasswordFile: passwordFile,
|
||||
Dir: path,
|
||||
Args: []string{"."},
|
||||
ExtraFlags: append(backupTagFlags(tags), "--host=velero", "--json"),
|
||||
}
|
||||
}
|
||||
|
||||
func backupTagFlags(tags map[string]string) []string {
|
||||
var flags []string
|
||||
for k, v := range tags {
|
||||
flags = append(flags, fmt.Sprintf("--tag=%s=%s", k, v))
|
||||
}
|
||||
return flags
|
||||
}
|
||||
|
||||
// RestoreCommand returns a Command for running a restic restore.
|
||||
func RestoreCommand(repoIdentifier, passwordFile, snapshotID, target string) *Command {
|
||||
return &Command{
|
||||
Command: "restore",
|
||||
RepoIdentifier: repoIdentifier,
|
||||
PasswordFile: passwordFile,
|
||||
Dir: target,
|
||||
Args: []string{snapshotID},
|
||||
ExtraFlags: []string{"--target=."},
|
||||
}
|
||||
}
|
||||
|
||||
// GetSnapshotCommand returns a Command for running a restic (get) snapshots.
|
||||
func GetSnapshotCommand(repoIdentifier, passwordFile string, tags map[string]string) *Command {
|
||||
return &Command{
|
||||
Command: "snapshots",
|
||||
RepoIdentifier: repoIdentifier,
|
||||
PasswordFile: passwordFile,
|
||||
// "--last" is replaced by "--latest=1" in restic v0.12.1
|
||||
ExtraFlags: []string{"--json", "--latest=1", getSnapshotTagFlag(tags)},
|
||||
}
|
||||
}
|
||||
|
||||
func getSnapshotTagFlag(tags map[string]string) string {
|
||||
var tagFilters []string
|
||||
for k, v := range tags {
|
||||
tagFilters = append(tagFilters, fmt.Sprintf("%s=%s", k, v))
|
||||
}
|
||||
|
||||
return fmt.Sprintf("--tag=%s", strings.Join(tagFilters, ","))
|
||||
}
|
||||
|
||||
func InitCommand(repoIdentifier string) *Command {
|
||||
return &Command{
|
||||
Command: "init",
|
||||
RepoIdentifier: repoIdentifier,
|
||||
}
|
||||
}
|
||||
|
||||
func SnapshotsCommand(repoIdentifier string) *Command {
|
||||
return &Command{
|
||||
Command: "snapshots",
|
||||
RepoIdentifier: repoIdentifier,
|
||||
}
|
||||
}
|
||||
|
||||
func PruneCommand(repoIdentifier string) *Command {
|
||||
return &Command{
|
||||
Command: "prune",
|
||||
RepoIdentifier: repoIdentifier,
|
||||
}
|
||||
}
|
||||
|
||||
func ForgetCommand(repoIdentifier, snapshotID string) *Command {
|
||||
return &Command{
|
||||
Command: "forget",
|
||||
RepoIdentifier: repoIdentifier,
|
||||
Args: []string{snapshotID},
|
||||
}
|
||||
}
|
||||
|
||||
func UnlockCommand(repoIdentifier string) *Command {
|
||||
return &Command{
|
||||
Command: "unlock",
|
||||
RepoIdentifier: repoIdentifier,
|
||||
}
|
||||
}
|
||||
|
||||
func StatsCommand(repoIdentifier, passwordFile, snapshotID string) *Command {
|
||||
return &Command{
|
||||
Command: "stats",
|
||||
RepoIdentifier: repoIdentifier,
|
||||
PasswordFile: passwordFile,
|
||||
Args: []string{snapshotID},
|
||||
ExtraFlags: []string{"--json"},
|
||||
}
|
||||
}
|
||||
@@ -1,131 +0,0 @@
|
||||
/*
|
||||
Copyright 2018 the Velero contributors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
package restic
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestBackupCommand(t *testing.T) {
|
||||
c := BackupCommand("repo-id", "password-file", "path", map[string]string{"foo": "bar", "c": "d"})
|
||||
|
||||
assert.Equal(t, "backup", c.Command)
|
||||
assert.Equal(t, "repo-id", c.RepoIdentifier)
|
||||
assert.Equal(t, "password-file", c.PasswordFile)
|
||||
assert.Equal(t, "path", c.Dir)
|
||||
assert.Equal(t, []string{"."}, c.Args)
|
||||
|
||||
expected := []string{"--tag=foo=bar", "--tag=c=d", "--host=velero", "--json"}
|
||||
sort.Strings(expected)
|
||||
sort.Strings(c.ExtraFlags)
|
||||
assert.Equal(t, expected, c.ExtraFlags)
|
||||
}
|
||||
|
||||
func TestRestoreCommand(t *testing.T) {
|
||||
c := RestoreCommand("repo-id", "password-file", "snapshot-id", "target")
|
||||
|
||||
assert.Equal(t, "restore", c.Command)
|
||||
assert.Equal(t, "repo-id", c.RepoIdentifier)
|
||||
assert.Equal(t, "password-file", c.PasswordFile)
|
||||
assert.Equal(t, "target", c.Dir)
|
||||
assert.Equal(t, []string{"snapshot-id"}, c.Args)
|
||||
assert.Equal(t, []string{"--target=."}, c.ExtraFlags)
|
||||
}
|
||||
|
||||
func TestGetSnapshotCommand(t *testing.T) {
|
||||
expectedTags := map[string]string{"foo": "bar", "c": "d"}
|
||||
c := GetSnapshotCommand("repo-id", "password-file", expectedTags)
|
||||
|
||||
assert.Equal(t, "snapshots", c.Command)
|
||||
assert.Equal(t, "repo-id", c.RepoIdentifier)
|
||||
assert.Equal(t, "password-file", c.PasswordFile)
|
||||
|
||||
// set up expected flag names
|
||||
expectedFlags := []string{"--json", "--latest=1", "--tag"}
|
||||
// for tracking actual flag names
|
||||
actualFlags := []string{}
|
||||
// for tracking actual --tag values as a map
|
||||
actualTags := make(map[string]string)
|
||||
|
||||
// loop through actual flags
|
||||
for _, flag := range c.ExtraFlags {
|
||||
// split into 2 parts from the first = sign (if any)
|
||||
parts := strings.SplitN(flag, "=", 2)
|
||||
|
||||
// convert --tag data to a map
|
||||
if parts[0] == "--tag" {
|
||||
actualFlags = append(actualFlags, parts[0])
|
||||
|
||||
// split based on ,
|
||||
tags := strings.Split(parts[1], ",")
|
||||
// loop through each key-value tag pair
|
||||
for _, tag := range tags {
|
||||
// split the pair on =
|
||||
kvs := strings.Split(tag, "=")
|
||||
// record actual key & value
|
||||
actualTags[kvs[0]] = kvs[1]
|
||||
}
|
||||
} else {
|
||||
actualFlags = append(actualFlags, flag)
|
||||
}
|
||||
}
|
||||
|
||||
assert.Equal(t, expectedFlags, actualFlags)
|
||||
assert.Equal(t, expectedTags, actualTags)
|
||||
}
|
||||
|
||||
func TestInitCommand(t *testing.T) {
|
||||
c := InitCommand("repo-id")
|
||||
|
||||
assert.Equal(t, "init", c.Command)
|
||||
assert.Equal(t, "repo-id", c.RepoIdentifier)
|
||||
}
|
||||
|
||||
func TestSnapshotsCommand(t *testing.T) {
|
||||
c := SnapshotsCommand("repo-id")
|
||||
|
||||
assert.Equal(t, "snapshots", c.Command)
|
||||
assert.Equal(t, "repo-id", c.RepoIdentifier)
|
||||
}
|
||||
|
||||
func TestPruneCommand(t *testing.T) {
|
||||
c := PruneCommand("repo-id")
|
||||
|
||||
assert.Equal(t, "prune", c.Command)
|
||||
assert.Equal(t, "repo-id", c.RepoIdentifier)
|
||||
}
|
||||
|
||||
func TestForgetCommand(t *testing.T) {
|
||||
c := ForgetCommand("repo-id", "snapshot-id")
|
||||
|
||||
assert.Equal(t, "forget", c.Command)
|
||||
assert.Equal(t, "repo-id", c.RepoIdentifier)
|
||||
assert.Equal(t, []string{"snapshot-id"}, c.Args)
|
||||
}
|
||||
|
||||
func TestStatsCommand(t *testing.T) {
|
||||
c := StatsCommand("repo-id", "password-file", "snapshot-id")
|
||||
|
||||
assert.Equal(t, "stats", c.Command)
|
||||
assert.Equal(t, "repo-id", c.RepoIdentifier)
|
||||
assert.Equal(t, "password-file", c.PasswordFile)
|
||||
assert.Equal(t, []string{"snapshot-id"}, c.Args)
|
||||
assert.Equal(t, []string{"--json"}, c.ExtraFlags)
|
||||
}
|
||||
@@ -1,106 +0,0 @@
|
||||
/*
|
||||
Copyright 2020 the Velero contributors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package restic
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestRepoName(t *testing.T) {
|
||||
c := &Command{RepoIdentifier: ""}
|
||||
assert.Empty(t, c.RepoName())
|
||||
|
||||
c.RepoIdentifier = "s3:s3.amazonaws.com/bucket/prefix/repo"
|
||||
assert.Equal(t, "repo", c.RepoName())
|
||||
|
||||
c.RepoIdentifier = "azure:bucket:/repo"
|
||||
assert.Equal(t, "repo", c.RepoName())
|
||||
|
||||
c.RepoIdentifier = "gs:bucket:/prefix/repo"
|
||||
assert.Equal(t, "repo", c.RepoName())
|
||||
}
|
||||
|
||||
func TestStringSlice(t *testing.T) {
|
||||
c := &Command{
|
||||
Command: "cmd",
|
||||
RepoIdentifier: "repo-id",
|
||||
PasswordFile: "/path/to/password-file",
|
||||
Dir: "/some/pwd",
|
||||
Args: []string{"arg-1", "arg-2"},
|
||||
ExtraFlags: []string{"--foo=bar"},
|
||||
}
|
||||
|
||||
require.NoError(t, os.Unsetenv("VELERO_SCRATCH_DIR"))
|
||||
assert.Equal(t, []string{
|
||||
"restic",
|
||||
"cmd",
|
||||
"--repo=repo-id",
|
||||
"--password-file=/path/to/password-file",
|
||||
"arg-1",
|
||||
"arg-2",
|
||||
"--foo=bar",
|
||||
}, c.StringSlice())
|
||||
|
||||
os.Setenv("VELERO_SCRATCH_DIR", "/foo")
|
||||
assert.Equal(t, []string{
|
||||
"restic",
|
||||
"cmd",
|
||||
"--repo=repo-id",
|
||||
"--password-file=/path/to/password-file",
|
||||
"--cache-dir=/foo/.cache/restic",
|
||||
"arg-1",
|
||||
"arg-2",
|
||||
"--foo=bar",
|
||||
}, c.StringSlice())
|
||||
|
||||
require.NoError(t, os.Unsetenv("VELERO_SCRATCH_DIR"))
|
||||
}
|
||||
|
||||
func TestString(t *testing.T) {
|
||||
c := &Command{
|
||||
Command: "cmd",
|
||||
RepoIdentifier: "repo-id",
|
||||
PasswordFile: "/path/to/password-file",
|
||||
Dir: "/some/pwd",
|
||||
Args: []string{"arg-1", "arg-2"},
|
||||
ExtraFlags: []string{"--foo=bar"},
|
||||
}
|
||||
|
||||
require.NoError(t, os.Unsetenv("VELERO_SCRATCH_DIR"))
|
||||
assert.Equal(t, "restic cmd --repo=repo-id --password-file=/path/to/password-file arg-1 arg-2 --foo=bar", c.String())
|
||||
}
|
||||
|
||||
func TestCmd(t *testing.T) {
|
||||
c := &Command{
|
||||
Command: "cmd",
|
||||
RepoIdentifier: "repo-id",
|
||||
PasswordFile: "/path/to/password-file",
|
||||
Dir: "/some/pwd",
|
||||
Args: []string{"arg-1", "arg-2"},
|
||||
ExtraFlags: []string{"--foo=bar"},
|
||||
}
|
||||
|
||||
require.NoError(t, os.Unsetenv("VELERO_SCRATCH_DIR"))
|
||||
execCmd := c.Cmd()
|
||||
|
||||
assert.Equal(t, c.StringSlice(), execCmd.Args)
|
||||
assert.Equal(t, c.Dir, execCmd.Dir)
|
||||
}
|
||||
@@ -1,159 +0,0 @@
|
||||
/*
|
||||
Copyright the Velero contributors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package restic
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/vmware-tanzu/velero/internal/credentials"
|
||||
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
|
||||
repoconfig "github.com/vmware-tanzu/velero/pkg/repository/config"
|
||||
"github.com/vmware-tanzu/velero/pkg/util/filesystem"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
// DefaultMaintenanceFrequency is the default time interval
|
||||
// at which restic prune is run.
|
||||
DefaultMaintenanceFrequency = 7 * 24 * time.Hour
|
||||
|
||||
// insecureSkipTLSVerifyKey is the flag in BackupStorageLocation's config
|
||||
// to indicate whether to skip TLS verify to setup insecure HTTPS connection.
|
||||
insecureSkipTLSVerifyKey = "insecureSkipTLSVerify"
|
||||
|
||||
// resticInsecureTLSFlag is the flag for Restic command line to indicate
|
||||
// skip TLS verify on https connection.
|
||||
resticInsecureTLSFlag = "--insecure-tls"
|
||||
)
|
||||
|
||||
// TempCACertFile creates a temp file containing a CA bundle
|
||||
// and returns its path. The caller should generally call os.Remove()
|
||||
// to remove the file when done with it.
|
||||
func TempCACertFile(caCert []byte, bsl string, fs filesystem.Interface) (string, error) {
|
||||
file, err := fs.TempFile("", fmt.Sprintf("cacert-%s", bsl))
|
||||
if err != nil {
|
||||
return "", errors.WithStack(err)
|
||||
}
|
||||
|
||||
if _, err := file.Write(caCert); err != nil {
|
||||
// nothing we can do about an error closing the file here, and we're
|
||||
// already returning an error about the write failing.
|
||||
file.Close()
|
||||
return "", errors.WithStack(err)
|
||||
}
|
||||
|
||||
name := file.Name()
|
||||
|
||||
if err := file.Close(); err != nil {
|
||||
return "", errors.WithStack(err)
|
||||
}
|
||||
|
||||
return name, nil
|
||||
}
|
||||
|
||||
// environ is a slice of strings representing the environment, in the form "key=value".
|
||||
type environ []string
|
||||
|
||||
// Unset a single environment variable.
|
||||
func (e *environ) Unset(key string) {
|
||||
for i := range *e {
|
||||
if strings.HasPrefix((*e)[i], key+"=") {
|
||||
(*e)[i] = (*e)[len(*e)-1]
|
||||
*e = (*e)[:len(*e)-1]
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// CmdEnv returns a list of environment variables (in the format var=val) that
|
||||
// should be used when running a restic command for a particular backend provider.
|
||||
// This list is the current environment, plus any provider-specific variables restic needs.
|
||||
func CmdEnv(backupLocation *velerov1api.BackupStorageLocation, credentialFileStore credentials.FileStore) ([]string, error) {
|
||||
var env environ
|
||||
env = os.Environ()
|
||||
customEnv := map[string]string{}
|
||||
var err error
|
||||
|
||||
config := backupLocation.Spec.Config
|
||||
if config == nil {
|
||||
config = map[string]string{}
|
||||
}
|
||||
|
||||
if backupLocation.Spec.Credential != nil {
|
||||
credsFile, err := credentialFileStore.Path(backupLocation.Spec.Credential)
|
||||
if err != nil {
|
||||
return []string{}, errors.WithStack(err)
|
||||
}
|
||||
config[repoconfig.CredentialsFileKey] = credsFile
|
||||
}
|
||||
|
||||
backendType := repoconfig.GetBackendType(backupLocation.Spec.Provider, backupLocation.Spec.Config)
|
||||
|
||||
switch backendType {
|
||||
case repoconfig.AWSBackend:
|
||||
customEnv, err = repoconfig.GetS3ResticEnvVars(config)
|
||||
if err != nil {
|
||||
return []string{}, err
|
||||
}
|
||||
case repoconfig.AzureBackend:
|
||||
customEnv, err = repoconfig.GetAzureResticEnvVars(config)
|
||||
if err != nil {
|
||||
return []string{}, err
|
||||
}
|
||||
case repoconfig.GCPBackend:
|
||||
customEnv, err = repoconfig.GetGCPResticEnvVars(config)
|
||||
if err != nil {
|
||||
return []string{}, err
|
||||
}
|
||||
}
|
||||
|
||||
for k, v := range customEnv {
|
||||
env.Unset(k)
|
||||
if v == "" {
|
||||
continue
|
||||
}
|
||||
env = append(env, fmt.Sprintf("%s=%s", k, v))
|
||||
}
|
||||
|
||||
return env, nil
|
||||
}
|
||||
|
||||
// GetInsecureSkipTLSVerifyFromBSL get insecureSkipTLSVerify flag from BSL configuration,
|
||||
// Then return --insecure-tls flag with boolean value as result.
|
||||
func GetInsecureSkipTLSVerifyFromBSL(backupLocation *velerov1api.BackupStorageLocation, logger logrus.FieldLogger) string {
|
||||
result := ""
|
||||
|
||||
if backupLocation == nil {
|
||||
logger.Info("bsl is nil. return empty.")
|
||||
return result
|
||||
}
|
||||
|
||||
if insecure, _ := strconv.ParseBool(backupLocation.Spec.Config[insecureSkipTLSVerifyKey]); insecure {
|
||||
logger.Debugf("set --insecure-tls=true for Restic command according to BSL %s config", backupLocation.Name)
|
||||
result = resticInsecureTLSFlag + "=true"
|
||||
return result
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
@@ -1,141 +0,0 @@
|
||||
/*
|
||||
Copyright the Velero contributors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package restic
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
|
||||
velerotest "github.com/vmware-tanzu/velero/pkg/test"
|
||||
)
|
||||
|
||||
func TestTempCACertFile(t *testing.T) {
|
||||
var (
|
||||
fs = velerotest.NewFakeFileSystem()
|
||||
caCertData = []byte("cacert")
|
||||
)
|
||||
|
||||
fileName, err := TempCACertFile(caCertData, "default", fs)
|
||||
require.NoError(t, err)
|
||||
|
||||
contents, err := fs.ReadFile(fileName)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, string(caCertData), string(contents))
|
||||
|
||||
os.Remove(fileName)
|
||||
}
|
||||
|
||||
func TestGetInsecureSkipTLSVerifyFromBSL(t *testing.T) {
|
||||
log := logrus.StandardLogger()
|
||||
tests := []struct {
|
||||
name string
|
||||
backupLocation *velerov1api.BackupStorageLocation
|
||||
logger logrus.FieldLogger
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
"Test with nil BSL. Should return empty string.",
|
||||
nil,
|
||||
log,
|
||||
"",
|
||||
},
|
||||
{
|
||||
"Test BSL with no configuration. Should return empty string.",
|
||||
&velerov1api.BackupStorageLocation{
|
||||
Spec: velerov1api.BackupStorageLocationSpec{
|
||||
Provider: "azure",
|
||||
},
|
||||
},
|
||||
log,
|
||||
"",
|
||||
},
|
||||
{
|
||||
"Test with AWS BSL's insecureSkipTLSVerify set to false.",
|
||||
&velerov1api.BackupStorageLocation{
|
||||
Spec: velerov1api.BackupStorageLocationSpec{
|
||||
Provider: "aws",
|
||||
Config: map[string]string{
|
||||
"insecureSkipTLSVerify": "false",
|
||||
},
|
||||
},
|
||||
},
|
||||
log,
|
||||
"",
|
||||
},
|
||||
{
|
||||
"Test with AWS BSL's insecureSkipTLSVerify set to true.",
|
||||
&velerov1api.BackupStorageLocation{
|
||||
Spec: velerov1api.BackupStorageLocationSpec{
|
||||
Provider: "aws",
|
||||
Config: map[string]string{
|
||||
"insecureSkipTLSVerify": "true",
|
||||
},
|
||||
},
|
||||
},
|
||||
log,
|
||||
"--insecure-tls=true",
|
||||
},
|
||||
{
|
||||
"Test with Azure BSL's insecureSkipTLSVerify set to invalid.",
|
||||
&velerov1api.BackupStorageLocation{
|
||||
Spec: velerov1api.BackupStorageLocationSpec{
|
||||
Provider: "azure",
|
||||
Config: map[string]string{
|
||||
"insecureSkipTLSVerify": "invalid",
|
||||
},
|
||||
},
|
||||
},
|
||||
log,
|
||||
"",
|
||||
},
|
||||
{
|
||||
"Test with GCP without insecureSkipTLSVerify.",
|
||||
&velerov1api.BackupStorageLocation{
|
||||
Spec: velerov1api.BackupStorageLocationSpec{
|
||||
Provider: "gcp",
|
||||
Config: map[string]string{},
|
||||
},
|
||||
},
|
||||
log,
|
||||
"",
|
||||
},
|
||||
{
|
||||
"Test with AWS without config.",
|
||||
&velerov1api.BackupStorageLocation{
|
||||
Spec: velerov1api.BackupStorageLocationSpec{
|
||||
Provider: "aws",
|
||||
},
|
||||
},
|
||||
log,
|
||||
"",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
res := GetInsecureSkipTLSVerifyFromBSL(test.backupLocation, test.logger)
|
||||
|
||||
assert.Equal(t, test.expected, res)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,292 +0,0 @@
|
||||
/*
|
||||
Copyright The Velero Contributors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package restic
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/vmware-tanzu/velero/pkg/uploader"
|
||||
"github.com/vmware-tanzu/velero/pkg/util/exec"
|
||||
"github.com/vmware-tanzu/velero/pkg/util/filesystem"
|
||||
)
|
||||
|
||||
const restoreProgressCheckInterval = 10 * time.Second
|
||||
const backupProgressCheckInterval = 10 * time.Second
|
||||
|
||||
var fileSystem = filesystem.NewFileSystem()
|
||||
|
||||
type backupStatusLine struct {
|
||||
MessageType string `json:"message_type"`
|
||||
// seen in status lines
|
||||
TotalBytes int64 `json:"total_bytes"`
|
||||
BytesDone int64 `json:"bytes_done"`
|
||||
// seen in summary line at the end
|
||||
TotalBytesProcessed int64 `json:"total_bytes_processed"`
|
||||
}
|
||||
|
||||
// GetSnapshotID runs provided 'restic snapshots' command to get the ID of a snapshot
|
||||
// and an error if a unique snapshot cannot be identified.
|
||||
func GetSnapshotID(snapshotIDCmd *Command) (string, error) {
|
||||
stdout, stderr, err := exec.RunCommand(snapshotIDCmd.Cmd())
|
||||
if err != nil {
|
||||
return "", errors.Wrapf(err, "error running command, stderr=%s", stderr)
|
||||
}
|
||||
|
||||
type snapshotID struct {
|
||||
ShortID string `json:"short_id"`
|
||||
}
|
||||
|
||||
var snapshots []snapshotID
|
||||
if err := json.Unmarshal([]byte(stdout), &snapshots); err != nil {
|
||||
return "", errors.Wrap(err, "error unmarshaling restic snapshots result")
|
||||
}
|
||||
|
||||
if len(snapshots) != 1 {
|
||||
return "", errors.Errorf("expected one matching snapshot by command: %s, got %d", snapshotIDCmd.String(), len(snapshots))
|
||||
}
|
||||
|
||||
return snapshots[0].ShortID, nil
|
||||
}
|
||||
|
||||
// RunBackup runs a `restic backup` command and watches the output to provide
|
||||
// progress updates to the caller.
|
||||
func RunBackup(backupCmd *Command, log logrus.FieldLogger, updater uploader.ProgressUpdater) (string, string, error) {
|
||||
// buffers for copying command stdout/err output into
|
||||
stdoutBuf := new(bytes.Buffer)
|
||||
stderrBuf := new(bytes.Buffer)
|
||||
|
||||
// create a channel to signal when to end the goroutine scanning for progress
|
||||
// updates
|
||||
quit := make(chan struct{})
|
||||
|
||||
cmd := backupCmd.Cmd()
|
||||
cmd.Stdout = stdoutBuf
|
||||
cmd.Stderr = stderrBuf
|
||||
|
||||
err := cmd.Start()
|
||||
if err != nil {
|
||||
exec.LogErrorAsExitCode(err, log)
|
||||
return stdoutBuf.String(), stderrBuf.String(), err
|
||||
}
|
||||
|
||||
go func() {
|
||||
ticker := time.NewTicker(backupProgressCheckInterval)
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
lastLine := getLastLine(stdoutBuf.Bytes())
|
||||
if len(lastLine) > 0 {
|
||||
stat, err := decodeBackupStatusLine(lastLine)
|
||||
if err != nil {
|
||||
log.WithError(err).Errorf("error getting restic backup progress")
|
||||
}
|
||||
|
||||
// if the line contains a non-empty bytes_done field, we can update the
|
||||
// caller with the progress
|
||||
if stat.BytesDone != 0 {
|
||||
updater.UpdateProgress(&uploader.Progress{
|
||||
TotalBytes: stat.TotalBytes,
|
||||
BytesDone: stat.BytesDone,
|
||||
})
|
||||
}
|
||||
}
|
||||
case <-quit:
|
||||
ticker.Stop()
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
err = cmd.Wait()
|
||||
if err != nil {
|
||||
exec.LogErrorAsExitCode(err, log)
|
||||
return stdoutBuf.String(), stderrBuf.String(), err
|
||||
}
|
||||
quit <- struct{}{}
|
||||
|
||||
summary, err := getSummaryLine(stdoutBuf.Bytes())
|
||||
if err != nil {
|
||||
return stdoutBuf.String(), stderrBuf.String(), err
|
||||
}
|
||||
stat, err := decodeBackupStatusLine(summary)
|
||||
if err != nil {
|
||||
return stdoutBuf.String(), stderrBuf.String(), err
|
||||
}
|
||||
if stat.MessageType != "summary" {
|
||||
return stdoutBuf.String(), stderrBuf.String(), errors.WithStack(fmt.Errorf("error getting restic backup summary: %s", string(summary)))
|
||||
}
|
||||
|
||||
// update progress to 100%
|
||||
updater.UpdateProgress(&uploader.Progress{
|
||||
TotalBytes: stat.TotalBytesProcessed,
|
||||
BytesDone: stat.TotalBytesProcessed,
|
||||
})
|
||||
|
||||
return string(summary), stderrBuf.String(), nil
|
||||
}
|
||||
|
||||
func decodeBackupStatusLine(lastLine []byte) (backupStatusLine, error) {
|
||||
var stat backupStatusLine
|
||||
if err := json.Unmarshal(lastLine, &stat); err != nil {
|
||||
return stat, errors.Wrapf(err, "unable to decode backup JSON line: %s", string(lastLine))
|
||||
}
|
||||
return stat, nil
|
||||
}
|
||||
|
||||
// getLastLine returns the last line of a byte array. The string is assumed to
|
||||
// have a newline at the end of it, so this returns the substring between the
|
||||
// last two newlines.
|
||||
func getLastLine(b []byte) []byte {
|
||||
if len(b) == 0 {
|
||||
return []byte("")
|
||||
}
|
||||
// subslice the byte array to ignore the newline at the end of the string
|
||||
lastNewLineIdx := bytes.LastIndex(b[:len(b)-1], []byte("\n"))
|
||||
return b[lastNewLineIdx+1 : len(b)-1]
|
||||
}
|
||||
|
||||
// getSummaryLine looks for the summary JSON line
|
||||
// (`{"message_type:"summary",...`) in the restic backup command output. Due to
|
||||
// an issue in Restic, this might not always be the last line
|
||||
// (https://github.com/restic/restic/issues/2389). It returns an error if it
|
||||
// can't be found.
|
||||
func getSummaryLine(b []byte) ([]byte, error) {
|
||||
summaryLineIdx := bytes.LastIndex(b, []byte(`{"message_type":"summary"`))
|
||||
if summaryLineIdx < 0 {
|
||||
return nil, errors.New("unable to find summary in restic backup command output")
|
||||
}
|
||||
// find the end of the summary line
|
||||
newLineIdx := bytes.Index(b[summaryLineIdx:], []byte("\n"))
|
||||
if newLineIdx < 0 {
|
||||
return nil, errors.New("unable to get summary line from restic backup command output")
|
||||
}
|
||||
return b[summaryLineIdx : summaryLineIdx+newLineIdx], nil
|
||||
}
|
||||
|
||||
// RunRestore runs a `restic restore` command and monitors the volume size to
|
||||
// provide progress updates to the caller.
|
||||
func RunRestore(restoreCmd *Command, log logrus.FieldLogger, updater uploader.ProgressUpdater) (string, string, error) {
|
||||
insecureTLSFlag := ""
|
||||
|
||||
for _, extraFlag := range restoreCmd.ExtraFlags {
|
||||
if strings.Contains(extraFlag, resticInsecureTLSFlag) {
|
||||
insecureTLSFlag = extraFlag
|
||||
}
|
||||
}
|
||||
|
||||
snapshotSize, err := getSnapshotSize(restoreCmd.RepoIdentifier, restoreCmd.PasswordFile, restoreCmd.CACertFile, restoreCmd.Args[0], restoreCmd.Env, insecureTLSFlag)
|
||||
if err != nil {
|
||||
return "", "", errors.Wrap(err, "error getting snapshot size")
|
||||
}
|
||||
|
||||
updater.UpdateProgress(&uploader.Progress{
|
||||
TotalBytes: snapshotSize,
|
||||
})
|
||||
|
||||
// create a channel to signal when to end the goroutine scanning for progress
|
||||
// updates
|
||||
quit := make(chan struct{})
|
||||
|
||||
go func() {
|
||||
ticker := time.NewTicker(restoreProgressCheckInterval)
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
volumeSize, err := getVolumeSize(restoreCmd.Dir)
|
||||
if err != nil {
|
||||
log.WithError(err).Errorf("error getting restic restore progress")
|
||||
}
|
||||
|
||||
if volumeSize != 0 {
|
||||
updater.UpdateProgress(&uploader.Progress{
|
||||
TotalBytes: snapshotSize,
|
||||
BytesDone: volumeSize,
|
||||
})
|
||||
}
|
||||
case <-quit:
|
||||
ticker.Stop()
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
stdout, stderr, err := exec.RunCommandWithLog(restoreCmd.Cmd(), log)
|
||||
quit <- struct{}{}
|
||||
|
||||
// update progress to 100%
|
||||
updater.UpdateProgress(&uploader.Progress{
|
||||
TotalBytes: snapshotSize,
|
||||
BytesDone: snapshotSize,
|
||||
})
|
||||
|
||||
return stdout, stderr, err
|
||||
}
|
||||
|
||||
func getSnapshotSize(repoIdentifier, passwordFile, caCertFile, snapshotID string, env []string, insecureTLS string) (int64, error) {
|
||||
cmd := StatsCommand(repoIdentifier, passwordFile, snapshotID)
|
||||
cmd.Env = env
|
||||
cmd.CACertFile = caCertFile
|
||||
|
||||
if len(insecureTLS) > 0 {
|
||||
cmd.ExtraFlags = append(cmd.ExtraFlags, insecureTLS)
|
||||
}
|
||||
|
||||
stdout, stderr, err := exec.RunCommand(cmd.Cmd())
|
||||
if err != nil {
|
||||
return 0, errors.Wrapf(err, "error running command, stderr=%s", stderr)
|
||||
}
|
||||
|
||||
var snapshotStats struct {
|
||||
TotalSize int64 `json:"total_size"`
|
||||
}
|
||||
|
||||
if err := json.Unmarshal([]byte(stdout), &snapshotStats); err != nil {
|
||||
return 0, errors.Wrapf(err, "error unmarshaling restic stats result, stdout=%s", stdout)
|
||||
}
|
||||
|
||||
return snapshotStats.TotalSize, nil
|
||||
}
|
||||
|
||||
func getVolumeSize(path string) (int64, error) {
|
||||
var size int64
|
||||
|
||||
files, err := fileSystem.ReadDir(path)
|
||||
if err != nil {
|
||||
return 0, errors.Wrapf(err, "error reading directory %s", path)
|
||||
}
|
||||
|
||||
for _, file := range files {
|
||||
if file.IsDir() {
|
||||
s, err := getVolumeSize(fmt.Sprintf("%s/%s", path, file.Name()))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
size += s
|
||||
} else {
|
||||
size += file.Size()
|
||||
}
|
||||
}
|
||||
|
||||
return size, nil
|
||||
}
|
||||
@@ -1,111 +0,0 @@
|
||||
/*
|
||||
Copyright 2019 the Velero contributors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package restic
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/vmware-tanzu/velero/pkg/test"
|
||||
"github.com/vmware-tanzu/velero/pkg/util/filesystem"
|
||||
)
|
||||
|
||||
func Test_getSummaryLine(t *testing.T) {
|
||||
summaryLine := `{"message_type":"summary","files_new":0,"files_changed":0,"files_unmodified":3,"dirs_new":0,"dirs_changed":0,"dirs_unmodified":0,"data_blobs":0,"tree_blobs":0,"data_added":0,"total_files_processed":3,"total_bytes_processed":13238272000,"total_duration":0.319265105,"snapshot_id":"38515bb5"}`
|
||||
tests := []struct {
|
||||
name string
|
||||
output string
|
||||
wantErr bool
|
||||
}{
|
||||
{"no summary", `{"message_type":"status","percent_done":0,"total_files":1,"total_bytes":10485760000}
|
||||
{"message_type":"status","percent_done":0,"total_files":3,"files_done":1,"total_bytes":13238272000}
|
||||
`, true},
|
||||
{"no newline after summary", `{"message_type":"status","percent_done":0,"total_files":1,"total_bytes":10485760000}
|
||||
{"message_type":"status","percent_done":0,"total_files":3,"files_done":1,"total_bytes":13238272000}
|
||||
{"message_type":"summary","files_new":0,"files_changed":0,"files_unmodified":3,"dirs_new":0`, true},
|
||||
{"summary at end", `{"message_type":"status","percent_done":0,"total_files":1,"total_bytes":10485760000}
|
||||
{"message_type":"status","percent_done":0,"total_files":3,"files_done":1,"total_bytes":13238272000}
|
||||
{"message_type":"status","percent_done":1,"total_files":3,"files_done":3,"total_bytes":13238272000,"bytes_done":13238272000}
|
||||
{"message_type":"summary","files_new":0,"files_changed":0,"files_unmodified":3,"dirs_new":0,"dirs_changed":0,"dirs_unmodified":0,"data_blobs":0,"tree_blobs":0,"data_added":0,"total_files_processed":3,"total_bytes_processed":13238272000,"total_duration":0.319265105,"snapshot_id":"38515bb5"}
|
||||
`, false},
|
||||
{"summary before status", `{"message_type":"status","percent_done":0,"total_files":1,"total_bytes":10485760000}
|
||||
{"message_type":"status","percent_done":0,"total_files":3,"files_done":1,"total_bytes":13238272000}
|
||||
{"message_type":"summary","files_new":0,"files_changed":0,"files_unmodified":3,"dirs_new":0,"dirs_changed":0,"dirs_unmodified":0,"data_blobs":0,"tree_blobs":0,"data_added":0,"total_files_processed":3,"total_bytes_processed":13238272000,"total_duration":0.319265105,"snapshot_id":"38515bb5"}
|
||||
{"message_type":"status","percent_done":1,"total_files":3,"files_done":3,"total_bytes":13238272000,"bytes_done":13238272000}
|
||||
`, false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
summary, err := getSummaryLine([]byte(tt.output))
|
||||
if tt.wantErr {
|
||||
assert.Error(t, err)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, summaryLine, string(summary))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_getLastLine(t *testing.T) {
|
||||
tests := []struct {
|
||||
output []byte
|
||||
want string
|
||||
}{
|
||||
{[]byte(`last line
|
||||
`), "last line"},
|
||||
{[]byte(`first line
|
||||
second line
|
||||
third line
|
||||
`), "third line"},
|
||||
{[]byte(""), ""},
|
||||
{nil, ""},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.want, func(t *testing.T) {
|
||||
assert.Equal(t, []byte(tt.want), getLastLine(tt.output))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_getVolumeSize(t *testing.T) {
|
||||
files := map[string][]byte{
|
||||
"/file1.txt": []byte("file1"),
|
||||
"/file2.txt": []byte("file2"),
|
||||
"/file3.txt": []byte("file3"),
|
||||
"/files/file4.txt": []byte("file4"),
|
||||
"/files/nested/file5.txt": []byte("file5"),
|
||||
}
|
||||
fakefs := test.NewFakeFileSystem()
|
||||
|
||||
var expectedSize int64
|
||||
for path, content := range files {
|
||||
fakefs.WithFile(path, content)
|
||||
expectedSize += int64(len(content))
|
||||
}
|
||||
|
||||
fileSystem = fakefs
|
||||
defer func() { fileSystem = filesystem.NewFileSystem() }()
|
||||
|
||||
actualSize, err := getVolumeSize("/")
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, expectedSize, actualSize)
|
||||
}
|
||||
@@ -74,6 +74,10 @@ type CachePVC struct {
|
||||
ResidentThresholdInMB int64 `json:"residentThresholdInMB,omitempty"`
|
||||
}
|
||||
|
||||
type CSISnapshotMetadataService struct {
|
||||
SAName string `json:"saName,omitempty"`
|
||||
}
|
||||
|
||||
type NodeAgentConfigs struct {
|
||||
// LoadConcurrency is the config for data path load concurrency per node.
|
||||
LoadConcurrency *LoadConcurrency `json:"loadConcurrency,omitempty"`
|
||||
@@ -104,4 +108,7 @@ type NodeAgentConfigs struct {
|
||||
|
||||
// PodLabels are labels to be added to pods created by node-agent, i.e., data mover pods.
|
||||
PodLabels map[string]string `json:"podLabels,omitempty"`
|
||||
|
||||
// CSISnapshotMetadataServiceConfigs is the config for CSI snapshot metadata service
|
||||
CSISnapshotMetadataServiceConfigs *CSISnapshotMetadataService `json:"csiSnapshotMetadataServiceConfigs,omitempty"`
|
||||
}
|
||||
|
||||
+92
-30
@@ -16,40 +16,102 @@ limitations under the License.
|
||||
|
||||
package cbt
|
||||
|
||||
import "github.com/vmware-tanzu/velero/pkg/cbtservice"
|
||||
import (
|
||||
"math/bits"
|
||||
|
||||
// Bitmap defines the methods to store and iterate the CBT bitmap
|
||||
type Bitmap interface {
|
||||
// Set sets bits within the provided range
|
||||
Set(cbtservice.Range)
|
||||
"github.com/RoaringBitmap/roaring"
|
||||
|
||||
// SetFull sets all bits to the bitmap
|
||||
SetFull()
|
||||
"github.com/vmware-tanzu/velero/pkg/uploader/cbt/types"
|
||||
)
|
||||
|
||||
// Snapshot returns snapshot of the bitmap
|
||||
SourceID() string
|
||||
const (
|
||||
InvalidOffset64 = ^uint64(0)
|
||||
)
|
||||
|
||||
// ChangeID returns the changeID of the bitmap
|
||||
ChangeID() string
|
||||
|
||||
// Iterator returns the iterator for the CBT Bitmap
|
||||
Iterator() Iterator
|
||||
type bitmapImpl struct {
|
||||
bitmap *roaring.Bitmap
|
||||
blockSize uint
|
||||
blockSizeLog int
|
||||
length uint64
|
||||
snapshot string
|
||||
changeID string
|
||||
volumeID string
|
||||
}
|
||||
|
||||
// Iterator defines the methods to iterate the CBT bitmap and query the associated information
|
||||
type Iterator interface {
|
||||
// ChangeID returns the changeID of the bitmap
|
||||
ChangeID() string
|
||||
|
||||
// Snapshot returns snapshot of the bitmap
|
||||
Snapshot() string
|
||||
|
||||
// BlockSize returns the granularity of the bitmap
|
||||
BlockSize() int
|
||||
|
||||
// Count returns the toal number of count in the bitmap
|
||||
Count() uint64
|
||||
|
||||
// Next returns the offset of the next set block and whether it comes to the end of the iteration
|
||||
Next() (int64, bool)
|
||||
type bitmapIterator struct {
|
||||
bitmapImpl
|
||||
iterator roaring.IntPeekable
|
||||
}
|
||||
|
||||
func NewBitmap(blockSize uint, length uint64, snapshot string, changeID string, volumeID string) types.Bitmap {
|
||||
return &bitmapImpl{
|
||||
bitmap: roaring.New(),
|
||||
blockSize: blockSize,
|
||||
blockSizeLog: bits.Len(blockSize) - 1,
|
||||
length: length,
|
||||
snapshot: snapshot,
|
||||
changeID: changeID,
|
||||
volumeID: volumeID,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *bitmapImpl) Set(offset, length uint64) {
|
||||
if offset >= c.length {
|
||||
return
|
||||
}
|
||||
|
||||
if offset+length > c.length {
|
||||
length = c.length - offset
|
||||
}
|
||||
|
||||
start := offset >> c.blockSizeLog
|
||||
end := (offset + length + uint64(c.blockSize) - 1) >> c.blockSizeLog
|
||||
|
||||
c.bitmap.AddRange(start, end)
|
||||
}
|
||||
|
||||
func (c *bitmapImpl) SetFull() {
|
||||
start := uint64(0)
|
||||
end := (c.length + uint64(c.blockSize) - 1) >> c.blockSizeLog
|
||||
|
||||
c.bitmap.AddRange(start, end)
|
||||
}
|
||||
|
||||
func (c *bitmapImpl) Snapshot() string {
|
||||
return c.snapshot
|
||||
}
|
||||
|
||||
func (c *bitmapImpl) ChangeID() string {
|
||||
return c.changeID
|
||||
}
|
||||
|
||||
func (c *bitmapImpl) VolumeID() string {
|
||||
return c.volumeID
|
||||
}
|
||||
|
||||
func (c *bitmapImpl) Iterator() types.Iterator {
|
||||
if c.bitmap == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
return &bitmapIterator{
|
||||
bitmapImpl: *c,
|
||||
iterator: c.bitmap.Iterator(),
|
||||
}
|
||||
}
|
||||
|
||||
func (c *bitmapIterator) Next() (uint64, bool) {
|
||||
if !c.iterator.HasNext() {
|
||||
return InvalidOffset64, false
|
||||
}
|
||||
|
||||
return uint64(c.iterator.Next()) << c.blockSizeLog, true
|
||||
}
|
||||
|
||||
func (c *bitmapIterator) Count() uint64 {
|
||||
return c.bitmap.GetCardinality()
|
||||
}
|
||||
|
||||
func (c *bitmapIterator) BlockSize() uint {
|
||||
return c.blockSize
|
||||
}
|
||||
|
||||
@@ -0,0 +1,256 @@
|
||||
/*
|
||||
Copyright The Velero Contributors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package cbt
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestBitmapProperties(t *testing.T) {
|
||||
b := NewBitmap(1024*1024, 10000*1024*1024, "snap-1", "change-1", "vol-1")
|
||||
assert.Equal(t, "snap-1", b.Snapshot())
|
||||
assert.Equal(t, "change-1", b.ChangeID())
|
||||
assert.Equal(t, "vol-1", b.VolumeID())
|
||||
}
|
||||
|
||||
func TestBitmapSet(t *testing.T) {
|
||||
const mb = 1024 * 1024
|
||||
const gb = 1024 * 1024 * 1024
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
blockSize uint
|
||||
totalLength uint64
|
||||
setCalls []struct{ offset, length uint64 }
|
||||
expectedCount uint64
|
||||
expectedNext []uint64
|
||||
}{
|
||||
{
|
||||
name: "set single block within bounds",
|
||||
blockSize: mb,
|
||||
totalLength: 10 * gb,
|
||||
setCalls: []struct{ offset, length uint64 }{
|
||||
{0, 1000},
|
||||
},
|
||||
expectedCount: 1,
|
||||
expectedNext: []uint64{0},
|
||||
},
|
||||
{
|
||||
name: "set exactly one block",
|
||||
blockSize: mb,
|
||||
totalLength: 10 * gb,
|
||||
setCalls: []struct{ offset, length uint64 }{
|
||||
{0, mb},
|
||||
},
|
||||
expectedCount: 1,
|
||||
expectedNext: []uint64{0},
|
||||
},
|
||||
{
|
||||
name: "set overlapping two blocks",
|
||||
blockSize: mb,
|
||||
totalLength: 10 * gb,
|
||||
setCalls: []struct{ offset, length uint64 }{
|
||||
{mb - 1, 2},
|
||||
},
|
||||
expectedCount: 2,
|
||||
expectedNext: []uint64{0, mb},
|
||||
},
|
||||
{
|
||||
name: "set multiple non-contiguous blocks",
|
||||
blockSize: mb,
|
||||
totalLength: 20 * gb,
|
||||
setCalls: []struct{ offset, length uint64 }{
|
||||
{0, 100},
|
||||
{2 * mb, 100},
|
||||
},
|
||||
expectedCount: 2,
|
||||
expectedNext: []uint64{0, 2 * mb},
|
||||
},
|
||||
{
|
||||
name: "set completely out of bounds (offset >= length)",
|
||||
blockSize: mb,
|
||||
totalLength: 10 * gb,
|
||||
setCalls: []struct{ offset, length uint64 }{
|
||||
{10 * gb, 100},
|
||||
{15 * gb, 100},
|
||||
},
|
||||
expectedCount: 0,
|
||||
expectedNext: []uint64{},
|
||||
},
|
||||
{
|
||||
name: "set partially out of bounds (truncated)",
|
||||
blockSize: mb,
|
||||
totalLength: 10 * gb,
|
||||
setCalls: []struct{ offset, length uint64 }{
|
||||
{10*gb - mb/2, mb}, // Starts in the last block, length pushes it out of bounds
|
||||
},
|
||||
expectedCount: 1, // Only the last block should be set
|
||||
expectedNext: []uint64{10*gb - mb},
|
||||
},
|
||||
{
|
||||
name: "set spanning entire length",
|
||||
blockSize: mb,
|
||||
totalLength: 3 * mb, // 3 blocks: 0-1MB, 1MB-2MB, 2MB-3MB
|
||||
setCalls: []struct{ offset, length uint64 }{
|
||||
{0, 3 * mb},
|
||||
},
|
||||
expectedCount: 3,
|
||||
expectedNext: []uint64{0, mb, 2 * mb},
|
||||
},
|
||||
{
|
||||
name: "set large contiguous range",
|
||||
blockSize: mb,
|
||||
totalLength: 100 * gb,
|
||||
setCalls: []struct{ offset, length uint64 }{
|
||||
{10 * mb, 5 * mb}, // Starts at 10MB, spans 5 full blocks
|
||||
},
|
||||
expectedCount: 5,
|
||||
expectedNext: []uint64{10 * mb, 11 * mb, 12 * mb, 13 * mb, 14 * mb},
|
||||
},
|
||||
{
|
||||
name: "set empty length",
|
||||
blockSize: mb,
|
||||
totalLength: 10 * gb,
|
||||
setCalls: []struct{ offset, length uint64 }{
|
||||
{mb, 0},
|
||||
},
|
||||
expectedCount: 0,
|
||||
expectedNext: []uint64{},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
b := NewBitmap(tt.blockSize, tt.totalLength, "snap-1", "change-1", "vol-1")
|
||||
|
||||
for _, call := range tt.setCalls {
|
||||
b.Set(call.offset, call.length)
|
||||
}
|
||||
|
||||
iter := b.Iterator()
|
||||
require.NotNil(t, iter)
|
||||
|
||||
assert.Equal(t, tt.expectedCount, iter.Count())
|
||||
|
||||
var actualNext []uint64
|
||||
for {
|
||||
offset, hasNext := iter.Next()
|
||||
if !hasNext {
|
||||
assert.Equal(t, InvalidOffset64, offset)
|
||||
break
|
||||
}
|
||||
actualNext = append(actualNext, offset)
|
||||
}
|
||||
|
||||
if len(tt.expectedNext) == 0 {
|
||||
assert.Empty(t, actualNext)
|
||||
} else {
|
||||
assert.Equal(t, tt.expectedNext, actualNext)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmapSetFull(t *testing.T) {
|
||||
const mb = 1024 * 1024
|
||||
// Total length 3MB, blockSize 1MB. This means 3 blocks total:
|
||||
// block 0: 0 - 1MB
|
||||
// block 1: 1MB - 2MB
|
||||
// block 2: 2MB - 3MB
|
||||
b := NewBitmap(mb, 3*mb, "snap-1", "change-1", "vol-1")
|
||||
b.SetFull()
|
||||
|
||||
iter := b.Iterator()
|
||||
require.NotNil(t, iter)
|
||||
|
||||
assert.Equal(t, uint64(3), iter.Count())
|
||||
|
||||
expectedOffsets := []uint64{0, mb, 2 * mb}
|
||||
var actualOffsets []uint64
|
||||
for {
|
||||
offset, hasNext := iter.Next()
|
||||
if !hasNext {
|
||||
break
|
||||
}
|
||||
actualOffsets = append(actualOffsets, offset)
|
||||
}
|
||||
|
||||
assert.Equal(t, expectedOffsets, actualOffsets)
|
||||
}
|
||||
|
||||
func TestBitmapIterator(t *testing.T) {
|
||||
const mb = 1024 * 1024
|
||||
const gb = 1024 * 1024 * 1024
|
||||
|
||||
b := NewBitmap(mb, 10*gb, "snap-1", "change-1", "vol-1")
|
||||
|
||||
// Set multiple ranges to test contiguous iteration
|
||||
b.Set(mb, 100) // Block 1
|
||||
b.Set(3*mb, 5*mb) // Blocks 3, 4, 5, 6, 7
|
||||
b.Set(10*gb-mb, mb) // Last block
|
||||
|
||||
iter := b.Iterator()
|
||||
require.NotNil(t, iter)
|
||||
|
||||
// Test iterator properties
|
||||
assert.Equal(t, "snap-1", iter.Snapshot())
|
||||
assert.Equal(t, "change-1", iter.ChangeID())
|
||||
assert.Equal(t, "vol-1", iter.VolumeID())
|
||||
assert.Equal(t, uint(mb), iter.BlockSize())
|
||||
assert.Equal(t, uint64(7), iter.Count()) // 1 + 5 + 1 = 7 blocks
|
||||
|
||||
expectedOffsets := []uint64{
|
||||
mb,
|
||||
3 * mb, 4 * mb, 5 * mb, 6 * mb, 7 * mb,
|
||||
10*gb - mb,
|
||||
}
|
||||
|
||||
// Test iteration
|
||||
var actualOffsets []uint64
|
||||
for {
|
||||
offset, hasNext := iter.Next()
|
||||
if !hasNext {
|
||||
assert.Equal(t, InvalidOffset64, offset)
|
||||
break
|
||||
}
|
||||
actualOffsets = append(actualOffsets, offset)
|
||||
}
|
||||
|
||||
assert.Equal(t, expectedOffsets, actualOffsets)
|
||||
|
||||
// Test end of iteration multiple times to ensure it stays exhausted
|
||||
offset, hasNext := iter.Next()
|
||||
assert.False(t, hasNext)
|
||||
assert.Equal(t, InvalidOffset64, offset)
|
||||
|
||||
offset, hasNext = iter.Next()
|
||||
assert.False(t, hasNext)
|
||||
assert.Equal(t, InvalidOffset64, offset)
|
||||
}
|
||||
|
||||
func TestBitmapIteratorNilBitmap(t *testing.T) {
|
||||
// Directly create bitmapImpl with a nil roaring.Bitmap to test safety
|
||||
b := &bitmapImpl{
|
||||
bitmap: nil,
|
||||
}
|
||||
|
||||
iter := b.Iterator()
|
||||
assert.Nil(t, iter)
|
||||
}
|
||||
+31
-19
@@ -19,31 +19,43 @@ package cbt
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/vmware-tanzu/velero/pkg/cbtservice"
|
||||
"github.com/vmware-tanzu/velero/pkg/uploader/cbt/types"
|
||||
)
|
||||
|
||||
// SetBitmapOrFull translates the allocated/changed blocks from CBT service to the given bitmap or set the bitmap to full when error happens
|
||||
func SetBitmapOrFull(ctx context.Context, service cbtservice.Service, bitmap Bitmap) error {
|
||||
var err error
|
||||
func SetBitmapOrFull(ctx context.Context, service cbtservice.Service, bitmap types.Bitmap) (err error) {
|
||||
defer func() {
|
||||
if err != nil {
|
||||
bitmap.SetFull()
|
||||
}
|
||||
}()
|
||||
|
||||
if service == nil {
|
||||
return errors.New("CBT service is absent")
|
||||
}
|
||||
|
||||
if bitmap.Snapshot() == "" {
|
||||
return errors.New("invalid snapshot")
|
||||
}
|
||||
|
||||
if bitmap.ChangeID() == "" {
|
||||
err = setFromAllocatedBlocks(ctx, service, bitmap)
|
||||
} else {
|
||||
err = setFromChangedBlocks(ctx, service, bitmap)
|
||||
return errors.Wrapf(service.GetAllocatedBlocks(ctx, bitmap.Snapshot(), func(blocks []cbtservice.Range) error {
|
||||
for _, b := range blocks {
|
||||
bitmap.Set(b.Offset, b.Length)
|
||||
}
|
||||
|
||||
return nil
|
||||
}), "error getting allocated blocks from CBT service")
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
bitmap.SetFull()
|
||||
}
|
||||
return errors.Wrapf(service.GetChangedBlocks(ctx, bitmap.Snapshot(), bitmap.ChangeID(), func(blocks []cbtservice.Range) error {
|
||||
for _, b := range blocks {
|
||||
bitmap.Set(b.Offset, b.Length)
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// TODO implement in following PRs
|
||||
func setFromAllocatedBlocks(_ context.Context, _ cbtservice.Service, _ Bitmap) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// TODO implement in following PRs
|
||||
func setFromChangedBlocks(_ context.Context, _ cbtservice.Service, _ Bitmap) error {
|
||||
return nil
|
||||
return nil
|
||||
}), "error getting changed blocks from CBT service")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
/*
|
||||
Copyright The Velero Contributors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package cbt
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/mock"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/vmware-tanzu/velero/pkg/cbtservice"
|
||||
cbtservicemocks "github.com/vmware-tanzu/velero/pkg/cbtservice/mocks"
|
||||
cbtmocks "github.com/vmware-tanzu/velero/pkg/uploader/cbt/types/mocks"
|
||||
)
|
||||
|
||||
func TestSetBitmapOrFull(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
nilService bool
|
||||
setupMocks func(*cbtservicemocks.Service, *cbtmocks.Bitmap)
|
||||
expectedErrStr string
|
||||
}{
|
||||
{
|
||||
name: "nil service",
|
||||
nilService: true,
|
||||
setupMocks: func(svc *cbtservicemocks.Service, bmp *cbtmocks.Bitmap) {
|
||||
bmp.On("SetFull").Return()
|
||||
},
|
||||
expectedErrStr: "CBT service is absent",
|
||||
},
|
||||
{
|
||||
name: "invalid snapshot",
|
||||
setupMocks: func(svc *cbtservicemocks.Service, bmp *cbtmocks.Bitmap) {
|
||||
bmp.On("Snapshot").Return("")
|
||||
bmp.On("SetFull").Return()
|
||||
},
|
||||
expectedErrStr: "invalid snapshot",
|
||||
},
|
||||
{
|
||||
name: "allocated blocks success",
|
||||
setupMocks: func(svc *cbtservicemocks.Service, bmp *cbtmocks.Bitmap) {
|
||||
bmp.On("Snapshot").Return("snap-1")
|
||||
bmp.On("ChangeID").Return("")
|
||||
|
||||
svc.On("GetAllocatedBlocks", mock.Anything, "snap-1", mock.Anything).Run(func(args mock.Arguments) {
|
||||
record := args.Get(2).(func([]cbtservice.Range) error)
|
||||
record([]cbtservice.Range{
|
||||
{Offset: 0, Length: 4096},
|
||||
{Offset: 8192, Length: 4096},
|
||||
})
|
||||
}).Return(nil)
|
||||
|
||||
bmp.On("Set", uint64(0), uint64(4096)).Return()
|
||||
bmp.On("Set", uint64(8192), uint64(4096)).Return()
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "allocated blocks error",
|
||||
setupMocks: func(svc *cbtservicemocks.Service, bmp *cbtmocks.Bitmap) {
|
||||
bmp.On("Snapshot").Return("snap-1")
|
||||
bmp.On("ChangeID").Return("")
|
||||
|
||||
svc.On("GetAllocatedBlocks", mock.Anything, "snap-1", mock.Anything).Return(errors.New("mock alloc error"))
|
||||
bmp.On("SetFull").Return()
|
||||
},
|
||||
expectedErrStr: "error getting allocated blocks from CBT service: mock alloc error",
|
||||
},
|
||||
{
|
||||
name: "changed blocks success",
|
||||
setupMocks: func(svc *cbtservicemocks.Service, bmp *cbtmocks.Bitmap) {
|
||||
bmp.On("Snapshot").Return("snap-1")
|
||||
bmp.On("ChangeID").Return("change-1")
|
||||
|
||||
svc.On("GetChangedBlocks", mock.Anything, "snap-1", "change-1", mock.Anything).Run(func(args mock.Arguments) {
|
||||
record := args.Get(3).(func([]cbtservice.Range) error)
|
||||
record([]cbtservice.Range{
|
||||
{Offset: 4096, Length: 4096},
|
||||
})
|
||||
}).Return(nil)
|
||||
|
||||
bmp.On("Set", uint64(4096), uint64(4096)).Return()
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "changed blocks error",
|
||||
setupMocks: func(svc *cbtservicemocks.Service, bmp *cbtmocks.Bitmap) {
|
||||
bmp.On("Snapshot").Return("snap-1")
|
||||
bmp.On("ChangeID").Return("change-1")
|
||||
|
||||
svc.On("GetChangedBlocks", mock.Anything, "snap-1", "change-1", mock.Anything).Return(errors.New("mock changed error"))
|
||||
bmp.On("SetFull").Return()
|
||||
},
|
||||
expectedErrStr: "error getting changed blocks from CBT service: mock changed error",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
svcMock := new(cbtservicemocks.Service)
|
||||
bmpMock := new(cbtmocks.Bitmap)
|
||||
|
||||
if tt.setupMocks != nil {
|
||||
tt.setupMocks(svcMock, bmpMock)
|
||||
}
|
||||
|
||||
var svc cbtservice.Service
|
||||
if !tt.nilService {
|
||||
svc = svcMock
|
||||
}
|
||||
|
||||
err := SetBitmapOrFull(context.Background(), svc, bmpMock)
|
||||
|
||||
if tt.expectedErrStr != "" {
|
||||
require.Error(t, err)
|
||||
require.EqualError(t, err, tt.expectedErrStr)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
if !tt.nilService {
|
||||
svcMock.AssertExpectations(t)
|
||||
}
|
||||
bmpMock.AssertExpectations(t)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,294 @@
|
||||
// Code generated by mockery; DO NOT EDIT.
|
||||
// github.com/vektra/mockery
|
||||
// template: testify
|
||||
|
||||
package mocks
|
||||
|
||||
import (
|
||||
mock "github.com/stretchr/testify/mock"
|
||||
"github.com/vmware-tanzu/velero/pkg/uploader/cbt/types"
|
||||
)
|
||||
|
||||
// NewBitmap creates a new instance of Bitmap. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
|
||||
// The first argument is typically a *testing.T value.
|
||||
func NewBitmap(t interface {
|
||||
mock.TestingT
|
||||
Cleanup(func())
|
||||
}) *Bitmap {
|
||||
mock := &Bitmap{}
|
||||
mock.Mock.Test(t)
|
||||
|
||||
t.Cleanup(func() { mock.AssertExpectations(t) })
|
||||
|
||||
return mock
|
||||
}
|
||||
|
||||
// Bitmap is an autogenerated mock type for the Bitmap type
|
||||
type Bitmap struct {
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
type Bitmap_Expecter struct {
|
||||
mock *mock.Mock
|
||||
}
|
||||
|
||||
func (_m *Bitmap) EXPECT() *Bitmap_Expecter {
|
||||
return &Bitmap_Expecter{mock: &_m.Mock}
|
||||
}
|
||||
|
||||
// ChangeID provides a mock function for the type Bitmap
|
||||
func (_mock *Bitmap) ChangeID() string {
|
||||
ret := _mock.Called()
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for ChangeID")
|
||||
}
|
||||
|
||||
var r0 string
|
||||
if returnFunc, ok := ret.Get(0).(func() string); ok {
|
||||
r0 = returnFunc()
|
||||
} else {
|
||||
r0 = ret.Get(0).(string)
|
||||
}
|
||||
return r0
|
||||
}
|
||||
|
||||
// Bitmap_ChangeID_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ChangeID'
|
||||
type Bitmap_ChangeID_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// ChangeID is a helper method to define mock.On call
|
||||
func (_e *Bitmap_Expecter) ChangeID() *Bitmap_ChangeID_Call {
|
||||
return &Bitmap_ChangeID_Call{Call: _e.mock.On("ChangeID")}
|
||||
}
|
||||
|
||||
func (_c *Bitmap_ChangeID_Call) Run(run func()) *Bitmap_ChangeID_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run()
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Bitmap_ChangeID_Call) Return(s string) *Bitmap_ChangeID_Call {
|
||||
_c.Call.Return(s)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Bitmap_ChangeID_Call) RunAndReturn(run func() string) *Bitmap_ChangeID_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// Iterator provides a mock function for the type Bitmap
|
||||
func (_mock *Bitmap) Iterator() types.Iterator {
|
||||
ret := _mock.Called()
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for Iterator")
|
||||
}
|
||||
|
||||
var r0 types.Iterator
|
||||
if returnFunc, ok := ret.Get(0).(func() types.Iterator); ok {
|
||||
r0 = returnFunc()
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(types.Iterator)
|
||||
}
|
||||
}
|
||||
return r0
|
||||
}
|
||||
|
||||
// Bitmap_Iterator_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Iterator'
|
||||
type Bitmap_Iterator_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// Iterator is a helper method to define mock.On call
|
||||
func (_e *Bitmap_Expecter) Iterator() *Bitmap_Iterator_Call {
|
||||
return &Bitmap_Iterator_Call{Call: _e.mock.On("Iterator")}
|
||||
}
|
||||
|
||||
func (_c *Bitmap_Iterator_Call) Run(run func()) *Bitmap_Iterator_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run()
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Bitmap_Iterator_Call) Return(iterator types.Iterator) *Bitmap_Iterator_Call {
|
||||
_c.Call.Return(iterator)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Bitmap_Iterator_Call) RunAndReturn(run func() types.Iterator) *Bitmap_Iterator_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// Set provides a mock function for the type Bitmap
|
||||
func (_mock *Bitmap) Set(v uint64, v1 uint64) {
|
||||
_mock.Called(v, v1)
|
||||
return
|
||||
}
|
||||
|
||||
// Bitmap_Set_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Set'
|
||||
type Bitmap_Set_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// Set is a helper method to define mock.On call
|
||||
// - v uint64
|
||||
// - v1 uint64
|
||||
func (_e *Bitmap_Expecter) Set(v interface{}, v1 interface{}) *Bitmap_Set_Call {
|
||||
return &Bitmap_Set_Call{Call: _e.mock.On("Set", v, v1)}
|
||||
}
|
||||
|
||||
func (_c *Bitmap_Set_Call) Run(run func(v uint64, v1 uint64)) *Bitmap_Set_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
var arg0 uint64
|
||||
if args[0] != nil {
|
||||
arg0 = args[0].(uint64)
|
||||
}
|
||||
var arg1 uint64
|
||||
if args[1] != nil {
|
||||
arg1 = args[1].(uint64)
|
||||
}
|
||||
run(
|
||||
arg0,
|
||||
arg1,
|
||||
)
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Bitmap_Set_Call) Return() *Bitmap_Set_Call {
|
||||
_c.Call.Return()
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Bitmap_Set_Call) RunAndReturn(run func(v uint64, v1 uint64)) *Bitmap_Set_Call {
|
||||
_c.Run(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// SetFull provides a mock function for the type Bitmap
|
||||
func (_mock *Bitmap) SetFull() {
|
||||
_mock.Called()
|
||||
return
|
||||
}
|
||||
|
||||
// Bitmap_SetFull_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetFull'
|
||||
type Bitmap_SetFull_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// SetFull is a helper method to define mock.On call
|
||||
func (_e *Bitmap_Expecter) SetFull() *Bitmap_SetFull_Call {
|
||||
return &Bitmap_SetFull_Call{Call: _e.mock.On("SetFull")}
|
||||
}
|
||||
|
||||
func (_c *Bitmap_SetFull_Call) Run(run func()) *Bitmap_SetFull_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run()
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Bitmap_SetFull_Call) Return() *Bitmap_SetFull_Call {
|
||||
_c.Call.Return()
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Bitmap_SetFull_Call) RunAndReturn(run func()) *Bitmap_SetFull_Call {
|
||||
_c.Run(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// Snapshot provides a mock function for the type Bitmap
|
||||
func (_mock *Bitmap) Snapshot() string {
|
||||
ret := _mock.Called()
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for Snapshot")
|
||||
}
|
||||
|
||||
var r0 string
|
||||
if returnFunc, ok := ret.Get(0).(func() string); ok {
|
||||
r0 = returnFunc()
|
||||
} else {
|
||||
r0 = ret.Get(0).(string)
|
||||
}
|
||||
return r0
|
||||
}
|
||||
|
||||
// Bitmap_Snapshot_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Snapshot'
|
||||
type Bitmap_Snapshot_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// Snapshot is a helper method to define mock.On call
|
||||
func (_e *Bitmap_Expecter) Snapshot() *Bitmap_Snapshot_Call {
|
||||
return &Bitmap_Snapshot_Call{Call: _e.mock.On("Snapshot")}
|
||||
}
|
||||
|
||||
func (_c *Bitmap_Snapshot_Call) Run(run func()) *Bitmap_Snapshot_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run()
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Bitmap_Snapshot_Call) Return(s string) *Bitmap_Snapshot_Call {
|
||||
_c.Call.Return(s)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Bitmap_Snapshot_Call) RunAndReturn(run func() string) *Bitmap_Snapshot_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// VolumeID provides a mock function for the type Bitmap
|
||||
func (_mock *Bitmap) VolumeID() string {
|
||||
ret := _mock.Called()
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for VolumeID")
|
||||
}
|
||||
|
||||
var r0 string
|
||||
if returnFunc, ok := ret.Get(0).(func() string); ok {
|
||||
r0 = returnFunc()
|
||||
} else {
|
||||
r0 = ret.Get(0).(string)
|
||||
}
|
||||
return r0
|
||||
}
|
||||
|
||||
// Bitmap_VolumeID_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'VolumeID'
|
||||
type Bitmap_VolumeID_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// VolumeID is a helper method to define mock.On call
|
||||
func (_e *Bitmap_Expecter) VolumeID() *Bitmap_VolumeID_Call {
|
||||
return &Bitmap_VolumeID_Call{Call: _e.mock.On("VolumeID")}
|
||||
}
|
||||
|
||||
func (_c *Bitmap_VolumeID_Call) Run(run func()) *Bitmap_VolumeID_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run()
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Bitmap_VolumeID_Call) Return(s string) *Bitmap_VolumeID_Call {
|
||||
_c.Call.Return(s)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Bitmap_VolumeID_Call) RunAndReturn(run func() string) *Bitmap_VolumeID_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
@@ -0,0 +1,309 @@
|
||||
// Code generated by mockery; DO NOT EDIT.
|
||||
// github.com/vektra/mockery
|
||||
// template: testify
|
||||
|
||||
package mocks
|
||||
|
||||
import (
|
||||
mock "github.com/stretchr/testify/mock"
|
||||
)
|
||||
|
||||
// NewIterator creates a new instance of Iterator. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
|
||||
// The first argument is typically a *testing.T value.
|
||||
func NewIterator(t interface {
|
||||
mock.TestingT
|
||||
Cleanup(func())
|
||||
}) *Iterator {
|
||||
mock := &Iterator{}
|
||||
mock.Mock.Test(t)
|
||||
|
||||
t.Cleanup(func() { mock.AssertExpectations(t) })
|
||||
|
||||
return mock
|
||||
}
|
||||
|
||||
// Iterator is an autogenerated mock type for the Iterator type
|
||||
type Iterator struct {
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
type Iterator_Expecter struct {
|
||||
mock *mock.Mock
|
||||
}
|
||||
|
||||
func (_m *Iterator) EXPECT() *Iterator_Expecter {
|
||||
return &Iterator_Expecter{mock: &_m.Mock}
|
||||
}
|
||||
|
||||
// BlockSize provides a mock function for the type Iterator
|
||||
func (_mock *Iterator) BlockSize() uint {
|
||||
ret := _mock.Called()
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for BlockSize")
|
||||
}
|
||||
|
||||
var r0 uint
|
||||
if returnFunc, ok := ret.Get(0).(func() uint); ok {
|
||||
r0 = returnFunc()
|
||||
} else {
|
||||
r0 = ret.Get(0).(uint)
|
||||
}
|
||||
return r0
|
||||
}
|
||||
|
||||
// Iterator_BlockSize_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'BlockSize'
|
||||
type Iterator_BlockSize_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// BlockSize is a helper method to define mock.On call
|
||||
func (_e *Iterator_Expecter) BlockSize() *Iterator_BlockSize_Call {
|
||||
return &Iterator_BlockSize_Call{Call: _e.mock.On("BlockSize")}
|
||||
}
|
||||
|
||||
func (_c *Iterator_BlockSize_Call) Run(run func()) *Iterator_BlockSize_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run()
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Iterator_BlockSize_Call) Return(v uint) *Iterator_BlockSize_Call {
|
||||
_c.Call.Return(v)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Iterator_BlockSize_Call) RunAndReturn(run func() uint) *Iterator_BlockSize_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// ChangeID provides a mock function for the type Iterator
|
||||
func (_mock *Iterator) ChangeID() string {
|
||||
ret := _mock.Called()
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for ChangeID")
|
||||
}
|
||||
|
||||
var r0 string
|
||||
if returnFunc, ok := ret.Get(0).(func() string); ok {
|
||||
r0 = returnFunc()
|
||||
} else {
|
||||
r0 = ret.Get(0).(string)
|
||||
}
|
||||
return r0
|
||||
}
|
||||
|
||||
// Iterator_ChangeID_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ChangeID'
|
||||
type Iterator_ChangeID_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// ChangeID is a helper method to define mock.On call
|
||||
func (_e *Iterator_Expecter) ChangeID() *Iterator_ChangeID_Call {
|
||||
return &Iterator_ChangeID_Call{Call: _e.mock.On("ChangeID")}
|
||||
}
|
||||
|
||||
func (_c *Iterator_ChangeID_Call) Run(run func()) *Iterator_ChangeID_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run()
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Iterator_ChangeID_Call) Return(s string) *Iterator_ChangeID_Call {
|
||||
_c.Call.Return(s)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Iterator_ChangeID_Call) RunAndReturn(run func() string) *Iterator_ChangeID_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// Count provides a mock function for the type Iterator
|
||||
func (_mock *Iterator) Count() uint64 {
|
||||
ret := _mock.Called()
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for Count")
|
||||
}
|
||||
|
||||
var r0 uint64
|
||||
if returnFunc, ok := ret.Get(0).(func() uint64); ok {
|
||||
r0 = returnFunc()
|
||||
} else {
|
||||
r0 = ret.Get(0).(uint64)
|
||||
}
|
||||
return r0
|
||||
}
|
||||
|
||||
// Iterator_Count_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Count'
|
||||
type Iterator_Count_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// Count is a helper method to define mock.On call
|
||||
func (_e *Iterator_Expecter) Count() *Iterator_Count_Call {
|
||||
return &Iterator_Count_Call{Call: _e.mock.On("Count")}
|
||||
}
|
||||
|
||||
func (_c *Iterator_Count_Call) Run(run func()) *Iterator_Count_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run()
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Iterator_Count_Call) Return(v uint64) *Iterator_Count_Call {
|
||||
_c.Call.Return(v)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Iterator_Count_Call) RunAndReturn(run func() uint64) *Iterator_Count_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// Next provides a mock function for the type Iterator
|
||||
func (_mock *Iterator) Next() (uint64, bool) {
|
||||
ret := _mock.Called()
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for Next")
|
||||
}
|
||||
|
||||
var r0 uint64
|
||||
var r1 bool
|
||||
if returnFunc, ok := ret.Get(0).(func() (uint64, bool)); ok {
|
||||
return returnFunc()
|
||||
}
|
||||
if returnFunc, ok := ret.Get(0).(func() uint64); ok {
|
||||
r0 = returnFunc()
|
||||
} else {
|
||||
r0 = ret.Get(0).(uint64)
|
||||
}
|
||||
if returnFunc, ok := ret.Get(1).(func() bool); ok {
|
||||
r1 = returnFunc()
|
||||
} else {
|
||||
r1 = ret.Get(1).(bool)
|
||||
}
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// Iterator_Next_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Next'
|
||||
type Iterator_Next_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// Next is a helper method to define mock.On call
|
||||
func (_e *Iterator_Expecter) Next() *Iterator_Next_Call {
|
||||
return &Iterator_Next_Call{Call: _e.mock.On("Next")}
|
||||
}
|
||||
|
||||
func (_c *Iterator_Next_Call) Run(run func()) *Iterator_Next_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run()
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Iterator_Next_Call) Return(v uint64, b bool) *Iterator_Next_Call {
|
||||
_c.Call.Return(v, b)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Iterator_Next_Call) RunAndReturn(run func() (uint64, bool)) *Iterator_Next_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// Snapshot provides a mock function for the type Iterator
|
||||
func (_mock *Iterator) Snapshot() string {
|
||||
ret := _mock.Called()
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for Snapshot")
|
||||
}
|
||||
|
||||
var r0 string
|
||||
if returnFunc, ok := ret.Get(0).(func() string); ok {
|
||||
r0 = returnFunc()
|
||||
} else {
|
||||
r0 = ret.Get(0).(string)
|
||||
}
|
||||
return r0
|
||||
}
|
||||
|
||||
// Iterator_Snapshot_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Snapshot'
|
||||
type Iterator_Snapshot_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// Snapshot is a helper method to define mock.On call
|
||||
func (_e *Iterator_Expecter) Snapshot() *Iterator_Snapshot_Call {
|
||||
return &Iterator_Snapshot_Call{Call: _e.mock.On("Snapshot")}
|
||||
}
|
||||
|
||||
func (_c *Iterator_Snapshot_Call) Run(run func()) *Iterator_Snapshot_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run()
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Iterator_Snapshot_Call) Return(s string) *Iterator_Snapshot_Call {
|
||||
_c.Call.Return(s)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Iterator_Snapshot_Call) RunAndReturn(run func() string) *Iterator_Snapshot_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// VolumeID provides a mock function for the type Iterator
|
||||
func (_mock *Iterator) VolumeID() string {
|
||||
ret := _mock.Called()
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for VolumeID")
|
||||
}
|
||||
|
||||
var r0 string
|
||||
if returnFunc, ok := ret.Get(0).(func() string); ok {
|
||||
r0 = returnFunc()
|
||||
} else {
|
||||
r0 = ret.Get(0).(string)
|
||||
}
|
||||
return r0
|
||||
}
|
||||
|
||||
// Iterator_VolumeID_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'VolumeID'
|
||||
type Iterator_VolumeID_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// VolumeID is a helper method to define mock.On call
|
||||
func (_e *Iterator_Expecter) VolumeID() *Iterator_VolumeID_Call {
|
||||
return &Iterator_VolumeID_Call{Call: _e.mock.On("VolumeID")}
|
||||
}
|
||||
|
||||
func (_c *Iterator_VolumeID_Call) Run(run func()) *Iterator_VolumeID_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run()
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Iterator_VolumeID_Call) Return(s string) *Iterator_VolumeID_Call {
|
||||
_c.Call.Return(s)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Iterator_VolumeID_Call) RunAndReturn(run func() string) *Iterator_VolumeID_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
/*
|
||||
Copyright The Velero Contributors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package types
|
||||
|
||||
// Bitmap defines the methods to store and iterate the CBT bitmap
|
||||
type Bitmap interface {
|
||||
// Set sets bits within the provided range
|
||||
Set(uint64, uint64)
|
||||
|
||||
// SetFull sets all bits to the bitmap
|
||||
SetFull()
|
||||
|
||||
// Snapshot returns snapshot of the bitmap
|
||||
Snapshot() string
|
||||
|
||||
// ChangeID returns the changeID of the bitmap
|
||||
ChangeID() string
|
||||
|
||||
// VolumeID return ID of the volume from which the snapshot is taken
|
||||
VolumeID() string
|
||||
|
||||
// Iterator returns the iterator for the CBT Bitmap
|
||||
Iterator() Iterator
|
||||
}
|
||||
|
||||
// Iterator defines the methods to iterate the CBT bitmap and query the associated information
|
||||
type Iterator interface {
|
||||
// ChangeID returns the changeID of the bitmap
|
||||
ChangeID() string
|
||||
|
||||
// Snapshot returns snapshot of the bitmap
|
||||
Snapshot() string
|
||||
|
||||
// VolumeID return ID of the volume from which the snapshot is taken
|
||||
VolumeID() string
|
||||
|
||||
// BlockSize returns the granularity of the bitmap
|
||||
BlockSize() uint
|
||||
|
||||
// Count returns the total number of count in the bitmap
|
||||
Count() uint64
|
||||
|
||||
// Next returns the offset of the next set block and whether it comes to the end of the iteration
|
||||
Next() (uint64, bool)
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
/*
|
||||
Copyright The Velero Contributors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package provider
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/vmware-tanzu/velero/internal/credentials"
|
||||
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
|
||||
repokeys "github.com/vmware-tanzu/velero/pkg/repository/keys"
|
||||
"github.com/vmware-tanzu/velero/pkg/repository/udmrepo"
|
||||
"github.com/vmware-tanzu/velero/pkg/uploader"
|
||||
)
|
||||
|
||||
type blockProvider struct {
|
||||
requestorType string
|
||||
bkRepo udmrepo.BackupRepo
|
||||
credGetter *credentials.CredentialGetter
|
||||
log logrus.FieldLogger
|
||||
}
|
||||
|
||||
// NewBlockUploaderProvider initialized with open or create a repository
|
||||
func NewBlockUploaderProvider(
|
||||
requestorType string,
|
||||
ctx context.Context,
|
||||
credGetter *credentials.CredentialGetter,
|
||||
backupRepo *velerov1api.BackupRepository,
|
||||
log logrus.FieldLogger,
|
||||
) (Provider, error) {
|
||||
bp := &blockProvider{
|
||||
requestorType: requestorType,
|
||||
log: log,
|
||||
credGetter: credGetter,
|
||||
}
|
||||
|
||||
repoUID := string(backupRepo.GetUID())
|
||||
repoOpt, err := udmrepo.NewRepoOptions(
|
||||
udmrepo.WithPassword(bp, ""),
|
||||
udmrepo.WithConfigFile("", repoUID),
|
||||
udmrepo.WithDescription("Initial velero block uploader provider"),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "error to get repo options")
|
||||
}
|
||||
|
||||
repoSvc := BackupRepoServiceCreateFunc(backupRepo.Spec.RepositoryType, log)
|
||||
log.WithField("repoUID", repoUID).Info("Opening backup repo")
|
||||
|
||||
bp.bkRepo, err = repoSvc.Open(ctx, *repoOpt)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "Failed to find backup repository")
|
||||
}
|
||||
|
||||
return bp, nil
|
||||
}
|
||||
|
||||
func (bp *blockProvider) Close(ctx context.Context) error {
|
||||
return bp.bkRepo.Close(ctx)
|
||||
}
|
||||
|
||||
func (bp *blockProvider) GetPassword(param any) (string, error) {
|
||||
if bp.credGetter.FromSecret == nil {
|
||||
return "", errors.New("invalid credentials interface")
|
||||
}
|
||||
rawPass, err := bp.credGetter.FromSecret.Get(repokeys.RepoKeySelector())
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "error to get password")
|
||||
}
|
||||
|
||||
return strings.TrimSpace(rawPass), nil
|
||||
}
|
||||
|
||||
// TODO: implement in the following PRs
|
||||
func (bp *blockProvider) RunBackup(
|
||||
ctx context.Context,
|
||||
path string,
|
||||
realSource string,
|
||||
tags map[string]string,
|
||||
forceFull bool,
|
||||
parentSnapshot string,
|
||||
cbtParam CBTParam,
|
||||
volMode uploader.PersistentVolumeMode,
|
||||
uploaderCfg map[string]string,
|
||||
updater uploader.ProgressUpdater) (string, bool, int64, int64, error) {
|
||||
return "", false, 0, 0, errors.New("block backup not implemented")
|
||||
}
|
||||
|
||||
// TODO: implement in the following PRs
|
||||
func (bp *blockProvider) RunRestore(
|
||||
ctx context.Context,
|
||||
snapshotID string,
|
||||
volumePath string,
|
||||
volMode uploader.PersistentVolumeMode,
|
||||
uploaderCfg map[string]string,
|
||||
updater uploader.ProgressUpdater) (int64, error) {
|
||||
return 0, errors.New("block restore not implemented")
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
/*
|
||||
Copyright The Velero Contributors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package provider
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/mock"
|
||||
"github.com/stretchr/testify/require"
|
||||
corev1api "k8s.io/api/core/v1"
|
||||
|
||||
"github.com/vmware-tanzu/velero/internal/credentials"
|
||||
"github.com/vmware-tanzu/velero/internal/credentials/mocks"
|
||||
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
|
||||
"github.com/vmware-tanzu/velero/pkg/repository"
|
||||
"github.com/vmware-tanzu/velero/pkg/repository/udmrepo"
|
||||
udmrepomocks "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/mocks"
|
||||
)
|
||||
|
||||
func TestNewBlockUploaderProvider(t *testing.T) {
|
||||
requestorType := "testRequestor"
|
||||
ctx := t.Context()
|
||||
backupRepo := repository.NewBackupRepository(velerov1api.DefaultNamespace, repository.BackupRepositoryKey{VolumeNamespace: "fake-volume-ns-02", BackupLocation: "fake-bsl-02", RepositoryType: "fake-repository-type-02"})
|
||||
mockLog := logrus.New()
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
mockCredGetter *mocks.SecretStore
|
||||
mockBackupRepoService udmrepo.BackupRepoService
|
||||
expectedError string
|
||||
}{
|
||||
{
|
||||
name: "Success",
|
||||
mockCredGetter: func() *mocks.SecretStore {
|
||||
mockCredGetter := &mocks.SecretStore{}
|
||||
mockCredGetter.On("Get", mock.Anything).Return("test", nil)
|
||||
return mockCredGetter
|
||||
}(),
|
||||
mockBackupRepoService: func() udmrepo.BackupRepoService {
|
||||
backupRepoService := &udmrepomocks.BackupRepoService{}
|
||||
var backupRepo udmrepo.BackupRepo
|
||||
backupRepoService.On("Open", t.Context(), mock.Anything).Return(backupRepo, nil)
|
||||
return backupRepoService
|
||||
}(),
|
||||
expectedError: "",
|
||||
},
|
||||
{
|
||||
name: "Error to get repo options",
|
||||
mockCredGetter: func() *mocks.SecretStore {
|
||||
mockCredGetter := &mocks.SecretStore{}
|
||||
mockCredGetter.On("Get", mock.Anything).Return("test", errors.New("failed to get password"))
|
||||
return mockCredGetter
|
||||
}(),
|
||||
mockBackupRepoService: func() udmrepo.BackupRepoService {
|
||||
backupRepoService := &udmrepomocks.BackupRepoService{}
|
||||
var backupRepo udmrepo.BackupRepo
|
||||
backupRepoService.On("Open", t.Context(), mock.Anything).Return(backupRepo, nil)
|
||||
return backupRepoService
|
||||
}(),
|
||||
expectedError: "error to get repo options",
|
||||
},
|
||||
{
|
||||
name: "Error open repository service",
|
||||
mockCredGetter: func() *mocks.SecretStore {
|
||||
mockCredGetter := &mocks.SecretStore{}
|
||||
mockCredGetter.On("Get", mock.Anything).Return("test", nil)
|
||||
return mockCredGetter
|
||||
}(),
|
||||
mockBackupRepoService: func() udmrepo.BackupRepoService {
|
||||
backupRepoService := &udmrepomocks.BackupRepoService{}
|
||||
var backupRepo udmrepo.BackupRepo
|
||||
backupRepoService.On("Open", t.Context(), mock.Anything).Return(backupRepo, errors.New("failed to init repository"))
|
||||
return backupRepoService
|
||||
}(),
|
||||
expectedError: "Failed to find backup repository",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
credGetter := &credentials.CredentialGetter{FromSecret: tc.mockCredGetter}
|
||||
BackupRepoServiceCreateFunc = func(string, logrus.FieldLogger) udmrepo.BackupRepoService {
|
||||
return tc.mockBackupRepoService
|
||||
}
|
||||
_, err := NewBlockUploaderProvider(requestorType, ctx, credGetter, backupRepo, mockLog)
|
||||
|
||||
if tc.expectedError != "" {
|
||||
require.ErrorContains(t, err, tc.expectedError)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
tc.mockCredGetter.AssertExpectations(t)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlockProviderClose(t *testing.T) {
|
||||
mockBRepo := udmrepomocks.NewBackupRepo(t)
|
||||
mockBRepo.On("Close", mock.Anything).Return(nil)
|
||||
|
||||
bp := &blockProvider{
|
||||
bkRepo: mockBRepo,
|
||||
}
|
||||
|
||||
err := bp.Close(t.Context())
|
||||
require.NoError(t, err)
|
||||
mockBRepo.AssertExpectations(t)
|
||||
}
|
||||
|
||||
func TestBlockProviderGetPassword(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
emptySecret bool
|
||||
credGetterFunc func(*mocks.SecretStore, *corev1api.SecretKeySelector)
|
||||
expectError bool
|
||||
expectedPass string
|
||||
}{
|
||||
{
|
||||
name: "valid credentials interface",
|
||||
credGetterFunc: func(ss *mocks.SecretStore, selector *corev1api.SecretKeySelector) {
|
||||
ss.On("Get", selector).Return("test", nil)
|
||||
},
|
||||
expectError: false,
|
||||
expectedPass: "test",
|
||||
},
|
||||
{
|
||||
name: "empty from secret",
|
||||
emptySecret: true,
|
||||
expectError: true,
|
||||
expectedPass: "",
|
||||
},
|
||||
{
|
||||
name: "ErrorGettingPassword",
|
||||
credGetterFunc: func(ss *mocks.SecretStore, selector *corev1api.SecretKeySelector) {
|
||||
ss.On("Get", selector).Return("", errors.New("error getting password"))
|
||||
},
|
||||
expectError: true,
|
||||
expectedPass: "",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
credGetter := &credentials.CredentialGetter{}
|
||||
mockCredGetter := &mocks.SecretStore{}
|
||||
if !tc.emptySecret {
|
||||
credGetter.FromSecret = mockCredGetter
|
||||
}
|
||||
repoKeySelector := &corev1api.SecretKeySelector{LocalObjectReference: corev1api.LocalObjectReference{Name: "velero-repo-credentials"}, Key: "repository-password"}
|
||||
|
||||
if tc.credGetterFunc != nil {
|
||||
tc.credGetterFunc(mockCredGetter, repoKeySelector)
|
||||
}
|
||||
|
||||
bp := &blockProvider{
|
||||
credGetter: credGetter,
|
||||
}
|
||||
|
||||
password, err := bp.GetPassword(nil)
|
||||
if tc.expectError {
|
||||
require.Error(t, err, "Expected an error")
|
||||
} else {
|
||||
require.NoError(t, err, "Expected no error")
|
||||
}
|
||||
|
||||
assert.Equal(t, tc.expectedPass, password, "Expected password to match")
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -36,9 +36,8 @@ import (
|
||||
"github.com/vmware-tanzu/velero/pkg/repository/udmrepo/service"
|
||||
)
|
||||
|
||||
// BackupFunc mainly used to make testing more convenient
|
||||
var BackupFunc = kopia.Backup
|
||||
var RestoreFunc = kopia.Restore
|
||||
var kopiaBackupFunc = kopia.Backup
|
||||
var kopiaRestoreFunc = kopia.Restore
|
||||
var BackupRepoServiceCreateFunc = service.Create
|
||||
|
||||
// kopiaProvider recorded info related with kopiaProvider
|
||||
@@ -118,6 +117,7 @@ func (kp *kopiaProvider) RunBackup(
|
||||
tags map[string]string,
|
||||
forceFull bool,
|
||||
parentSnapshot string,
|
||||
_ CBTParam,
|
||||
volMode uploader.PersistentVolumeMode,
|
||||
uploaderCfg map[string]string,
|
||||
updater uploader.ProgressUpdater) (string, bool, int64, int64, error) {
|
||||
@@ -165,7 +165,7 @@ func (kp *kopiaProvider) RunBackup(
|
||||
uploaderCfg[kopia.UploaderConfigMultipartKey] = "true"
|
||||
}
|
||||
|
||||
snapshotInfo, _, err := BackupFunc(ctx, kpUploader, repoWriter, path, realSource, forceFull, parentSnapshot, volMode, uploaderCfg, tags, log)
|
||||
snapshotInfo, _, err := kopiaBackupFunc(ctx, kpUploader, repoWriter, path, realSource, forceFull, parentSnapshot, volMode, uploaderCfg, tags, log)
|
||||
if err != nil {
|
||||
snapshotID := ""
|
||||
if snapshotInfo != nil {
|
||||
@@ -233,7 +233,7 @@ func (kp *kopiaProvider) RunRestore(
|
||||
// We use the cancel channel to control the restore cancel, so don't pass a context with cancel to Kopia restore.
|
||||
// Otherwise, Kopia restore will not response to the cancel control but return an arbitrary error.
|
||||
// Kopia restore cancel is not designed as well as Kopia backup which uses the context to control backup cancel all the way.
|
||||
size, fileCount, err := RestoreFunc(context.Background(), repoWriter, progress, snapshotID, volumePath, volMode, uploaderCfg, log, restoreCancel)
|
||||
size, fileCount, err := kopiaRestoreFunc(context.Background(), repoWriter, progress, snapshotID, volumePath, volMode, uploaderCfg, log, restoreCancel)
|
||||
|
||||
if err != nil {
|
||||
return 0, errors.Wrapf(err, "Failed to run kopia restore")
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user