diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index c4e7d9923..0c4b67d3c 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -60,6 +60,19 @@ branches. space-delimited: `/backport release-1.17 release-1.18`. The label causes the backport to run automatically when the PR merges. - **After merge:** the same comment immediately creates the backport PR. +- **Shorthand:** a bare version like `/backport 1.17` is automatically expanded + to `release-1.17`; this works generically for any `X.Y` version. +- **Changelog filename:** the cherry-picked commit(s) carry over the source + PR's `changelogs/unreleased/-` file. The workflow + automatically renames it to `-` on the backport branch so + `hack/changelog-check.sh` passes and release notes cite the correct PR. +- **Changelog-not-required:** if the source PR is labeled + `kind/changelog-not-required`, that label is copied to the backport PR so + it isn't flagged as missing a changelog. +- **DCO signoff:** every commit on a backport branch is re-signed with the + bot's `Signed-off-by` trailer (`git rebase --signoff`), including + cherry-picked commits from the original author, so the DCO check always + passes on backport PRs. - Only repository **owners, members, and collaborators** may trigger these commands. ## General coding guidelines diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 670e16103..7aaf37058 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -16,6 +16,26 @@ name: Backport merged pull request # In both cases multiple target branches can be space-delimited in a comment: # /backport release-1.17 release-1.18 # +# As a shorthand, a bare release version (e.g. `1.17`) is automatically +# expanded to the corresponding `release-1.17` branch, so `/backport 1.17` +# and `/backport release-1.17` are equivalent. This works generically for +# any `X.Y` version, e.g. `/backport 1.18 1.19`. +# +# The cherry-picked commit(s) carry over the original PR's changelog file +# (changelogs/unreleased/-), which no longer matches the +# backport PR's own number. After the backport PR is created, its changelog +# file is automatically renamed to - so that +# hack/changelog-check.sh passes and release notes cite the correct PR. +# +# If the source PR is labeled `kind/changelog-not-required` (i.e. it has no +# changelog file), that label is copied to the backport PR so it isn't +# flagged as missing a changelog either. +# +# Every commit on a backport branch (the cherry-picked commit(s), even from +# the original author, plus the changelog rename commit) is re-signed with +# the bot's Signed-off-by trailer via `git rebase --signoff`, so the DCO +# check always passes regardless of whether the original commit had one. +# # See: https://github.com/velero-io/velero/issues/9603 on: @@ -89,6 +109,10 @@ jobs: fi for branch in $branches; do + # Shorthand: a bare version like "1.17" expands to "release-1.17". + if [[ "$branch" =~ ^[0-9]+\.[0-9]+$ ]]; then + branch="release-${branch}" + fi label="backport ${branch}" echo "Applying label: '${label}'" # Create the label if it does not exist yet (idempotent). @@ -141,18 +165,31 @@ jobs: # (may be empty, falls back to labels). line=$(printf '%s' "$COMMENT_BODY" | head -n1 | tr -d '\r') branches=$(printf '%s' "$line" | sed -E 's#^/(backport|cherrypick)[[:space:]]*##') - echo "branches=${branches}" >> "$GITHUB_OUTPUT" + + normalized="" + for branch in $branches; do + # Shorthand: a bare version like "1.17" expands to "release-1.17". + if [[ "$branch" =~ ^[0-9]+\.[0-9]+$ ]]; then + branch="release-${branch}" + fi + normalized="${normalized}${normalized:+ }${branch}" + done + echo "branches=${normalized}" >> "$GITHUB_OUTPUT" - uses: actions/checkout@v7 with: fetch-depth: 0 - name: Create backport pull requests + id: backport # Pin to commit SHA: workflow has contents/pull-requests write. uses: korthout/backport-action@2e830a1d0b8269505846ddd407a70876913ad1f8 # v4.6.0 with: # Labels like `backport release-1.17` select the target branch. label_pattern: '^backport ([^ ]+)$' + # Carry over `kind/changelog-not-required` from the source PR so + # backport PRs of changelog-exempt changes aren't flagged as missing one. + copy_labels_pattern: '^kind/changelog-not-required$' # Prefer draft PRs with conflict markers over failing the job silently. experimental: | { @@ -161,3 +198,63 @@ jobs: # Empty when triggered by merge labels; set when `/backport` or `/cherrypick` includes branches. target_branches: ${{ steps.parse.outputs.branches }} github_token: ${{ secrets.GITHUB_TOKEN }} + + - name: Rename changelog file(s) and ensure DCO signoff + # The cherry-picked commit(s) still carry the source PR's changelog + # filename (e.g. changelogs/unreleased/9795-kaovilai), which no + # longer matches the new backport PR's number. Rename it on each + # created backport branch so hack/changelog-check.sh passes and the + # release notes cite the correct PR. + # + # Also ensure every commit on the backport branch passes the DCO + # check by re-signing it with the bot's Signed-off-by trailer via + # `git rebase --signoff`. This covers the cherry-picked commits + # (even when the original author's commit had no trailer) as well + # as the changelog rename commit added above; it preserves any + # existing Signed-off-by trailers rather than replacing them. + if: steps.backport.outputs.created_pull_numbers != '' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + SOURCE_PR_NUMBER: ${{ github.event.pull_request.number || github.event.issue.number }} + CREATED_PR_NUMBERS: ${{ steps.backport.outputs.created_pull_numbers }} + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + shopt -s nullglob + for new_pr in $CREATED_PR_NUMBERS; do + if [ "$new_pr" = "$SOURCE_PR_NUMBER" ]; then + continue + fi + + branch=$(gh pr view "$new_pr" --repo "$REPO" --json headRefName -q .headRefName) + base_branch=$(gh pr view "$new_pr" --repo "$REPO" --json baseRefName -q .baseRefName) + git fetch origin "$branch" "$base_branch" + git checkout -B "$branch" "origin/${branch}" + + files=(changelogs/unreleased/"${SOURCE_PR_NUMBER}"-*) + if [ ${#files[@]} -gt 0 ]; then + for old_file in "${files[@]}"; do + suffix=$(basename "$old_file" | sed -E "s/^${SOURCE_PR_NUMBER}-//") + new_file="changelogs/unreleased/${new_pr}-${suffix}" + if [ "$old_file" != "$new_file" ]; then + git mv "$old_file" "$new_file" + fi + done + if ! git diff --cached --quiet; then + git commit -m "Rename changelog to match backport PR #${new_pr}" + fi + else + echo "No changelog file for PR ${SOURCE_PR_NUMBER} found on ${branch}; skipping rename." + fi + + # Add the bot's Signed-off-by trailer to every commit ahead of + # the target branch (cherry-picked commits + the rename commit). + if ! git rebase --signoff "origin/${base_branch}"; then + echo "::error::git rebase --signoff failed for PR #${new_pr} on branch ${branch}; aborting rebase, branch left unchanged." >&2 + git rebase --abort + exit 1 + fi + git push --force-with-lease origin "HEAD:${branch}" + done diff --git a/.github/workflows/e2e-test-kind.yaml b/.github/workflows/e2e-test-kind.yaml index 6370a2b56..0029e734f 100644 --- a/.github/workflows/e2e-test-kind.yaml +++ b/.github/workflows/e2e-test-kind.yaml @@ -94,12 +94,14 @@ jobs: id: set-matrix # everything excluding older tags. limits needs to be high enough to cover all latest versions # and test labels - # grep -E "v[1-9]\.(2[5-9]|[3-9][0-9])" filters for v1.25 to v9.99 + # grep -E "^v[1-9]\.(2[5-9]|[3-9][0-9])\.[0-9]+$" filters for well-formed v1.25.x to v9.99.x + # GA releases only, so a pre-release tag like v1.37.0-rc.1 can't reach the + # awk step below and be misparsed as a patch release (e.g. "1.37.1") # and removes older patches of the same minor version # awk -F. '{if(!a[$1"."$2]++)print $1"."$2"."$NF}' run: | echo "matrix={\ - \"k8s\":$(wget -q -O - "https://hub.docker.com/v2/namespaces/kindest/repositories/node/tags?page_size=50" | grep -o '"name": *"[^"]*' | grep -o '[^"]*$' | grep -v -E "alpha|beta" | grep -E "v[1-9]\.(2[5-9]|[3-9][0-9])" | awk -F. '{if(!a[$1"."$2]++)print $1"."$2"."$NF}' | sort -r | sed s/v//g | jq -R -c -s 'split("\n")[:-1]'),\ + \"k8s\":$(wget -q -O - "https://hub.docker.com/v2/namespaces/kindest/repositories/node/tags?page_size=50" | grep -o '"name": *"[^"]*' | grep -o '[^"]*$' | grep -E "^v[1-9]\.(2[5-9]|[3-9][0-9])\.[0-9]+$" | awk -F. '{if(!a[$1"."$2]++)print $1"."$2"."$NF}' | sort -r | sed s/v//g | jq -R -c -s 'split("\n")[:-1]'),\ \"labels\":[\ \"Basic && (ClusterResource || NodePort || StorageClass)\", \ \"ResourceFiltering && !FSBackup\", \ @@ -140,7 +142,7 @@ jobs: - name: Install MinIO run: | docker run -d --rm -p 9000:9000 -e "MINIO_ROOT_USER=minio" -e "MINIO_ROOT_PASSWORD=minio123" -e "MINIO_DEFAULT_BUCKETS=bucket,additional-bucket" bitnami/minio:local - - uses: helm/kind-action@v1 + - uses: helm/kind-action@7a97ed793754775518f9db3a8151ee7461dc9c31 # v1 + fix: add curl retry flags (https://github.com/helm/kind-action/pull/165) with: cluster_name: "kind" version: "v0.32.0" diff --git a/.github/workflows/nightly-trivy-scan.yml b/.github/workflows/nightly-trivy-scan.yml index 4c1381a5b..d3f2e6062 100644 --- a/.github/workflows/nightly-trivy-scan.yml +++ b/.github/workflows/nightly-trivy-scan.yml @@ -31,6 +31,6 @@ jobs: output: 'trivy-results.sarif' - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v4.37.6 + uses: github/codeql-action/upload-sarif@v4.37.7 with: sarif_file: 'trivy-results.sarif' \ No newline at end of file diff --git a/.github/workflows/pr-codespell.yml b/.github/workflows/pr-codespell.yml index a2d22dd73..97cdb48d4 100644 --- a/.github/workflows/pr-codespell.yml +++ b/.github/workflows/pr-codespell.yml @@ -14,8 +14,7 @@ jobs: - name: Codespell uses: codespell-project/actions-codespell@master with: - # ignore the config/.../crd.go file as it's generated binary data that is edited elsewhere. - skip: .git,*.png,*.jpg,*.woff,*.ttf,*.gif,*.ico,./config/crd/v1beta1/crds/crds.go,./config/crd/v1/crds/crds.go,./config/crd/v2alpha1/crds/crds.go,./go.sum,./LICENSE + skip: .git,*.png,*.jpg,*.woff,*.ttf,*.gif,*.ico,./go.sum,./LICENSE ignore_words_list: iam,aks,ist,bridget,ue,shouldnot,atleast,notin,sme,optin,sie check_filenames: true check_hidden: true diff --git a/README.md b/README.md index 5457d145b..2357be2bb 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,9 @@ ![100] [![Build Status][1]][2] [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/3811/badge)](https://bestpractices.coreinfrastructure.org/projects/3811) -![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/velero-io/velero) +[![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/velero-io/velero)](https://github.com/velero-io/velero/releases) +[![GitHub stars](https://img.shields.io/github/stars/velero-io/velero)](https://github.com/velero-io/velero/stargazers) +[![Docker Pulls](https://img.shields.io/docker/pulls/velero/velero.svg)](https://hub.docker.com/r/velero/velero) ## Overview diff --git a/changelogs/unreleased/10185-Lyndon-Li b/changelogs/unreleased/10185-Lyndon-Li new file mode 100644 index 000000000..7f8cfa422 --- /dev/null +++ b/changelogs/unreleased/10185-Lyndon-Li @@ -0,0 +1 @@ +Support full backup for file system data mover and pod volume backup \ No newline at end of file diff --git a/changelogs/unreleased/10229-Ralthos b/changelogs/unreleased/10229-Ralthos new file mode 100644 index 000000000..7963342e0 --- /dev/null +++ b/changelogs/unreleased/10229-Ralthos @@ -0,0 +1 @@ +Add printer columns for DownloadRequest and ServerStatusRequest so kubectl shows their target, status and server version diff --git a/changelogs/unreleased/10242-beep-boopp b/changelogs/unreleased/10242-beep-boopp new file mode 100644 index 000000000..42312c448 --- /dev/null +++ b/changelogs/unreleased/10242-beep-boopp @@ -0,0 +1 @@ +Fix schedule reconciler aliasing &c.skipImmediately into Schedule specs, corrupting the server-wide --schedule-skip-immediately default after the first reconcile diff --git a/changelogs/unreleased/10245-Ralthos b/changelogs/unreleased/10245-Ralthos new file mode 100644 index 000000000..50e603dcd --- /dev/null +++ b/changelogs/unreleased/10245-Ralthos @@ -0,0 +1 @@ +Document that the DownloadRequest Processed phase means a URL has been signed, and that it does not imply the target object exists diff --git a/changelogs/unreleased/10247-opbot-xd b/changelogs/unreleased/10247-opbot-xd new file mode 100644 index 000000000..d1caa9255 --- /dev/null +++ b/changelogs/unreleased/10247-opbot-xd @@ -0,0 +1 @@ +Refactor: Replace context.TODO() with properly plumbed contexts in CSI backup actions and utility functions to enable proper cancellation of in-flight API requests. diff --git a/changelogs/unreleased/10252-Ralthos b/changelogs/unreleased/10252-Ralthos new file mode 100644 index 000000000..8766a8d42 --- /dev/null +++ b/changelogs/unreleased/10252-Ralthos @@ -0,0 +1 @@ +Skip signing a download URL when no artifacts can exist yet, and set a Failed phase with the reason so callers stop waiting diff --git a/changelogs/unreleased/10254-Lyndon-Li b/changelogs/unreleased/10254-Lyndon-Li new file mode 100644 index 000000000..e3ad3b7a8 --- /dev/null +++ b/changelogs/unreleased/10254-Lyndon-Li @@ -0,0 +1 @@ +Ignore credentialFile filled into BSL by users to avoid unexpected credentials used by Velero \ No newline at end of file diff --git a/changelogs/unreleased/10255-Lyndon-Li b/changelogs/unreleased/10255-Lyndon-Li new file mode 100644 index 000000000..f6ddc1e76 --- /dev/null +++ b/changelogs/unreleased/10255-Lyndon-Li @@ -0,0 +1 @@ +Use thread safe map for cancel recorder \ No newline at end of file diff --git a/changelogs/unreleased/10256-Lyndon-Li b/changelogs/unreleased/10256-Lyndon-Li new file mode 100644 index 000000000..c9fdb5779 --- /dev/null +++ b/changelogs/unreleased/10256-Lyndon-Li @@ -0,0 +1 @@ +Clarify the security context to Velero server \ No newline at end of file diff --git a/changelogs/unreleased/10258-Lyndon-Li b/changelogs/unreleased/10258-Lyndon-Li new file mode 100644 index 000000000..d73f8f4c6 --- /dev/null +++ b/changelogs/unreleased/10258-Lyndon-Li @@ -0,0 +1 @@ +Cap the unzip of metadata download to avoid OOM kill \ No newline at end of file diff --git a/changelogs/unreleased/10259-Jay2006sawant b/changelogs/unreleased/10259-Jay2006sawant new file mode 100644 index 000000000..ebe0ded02 --- /dev/null +++ b/changelogs/unreleased/10259-Jay2006sawant @@ -0,0 +1 @@ +Trim spaces around resource names in --ordered-resources so comma-separated lists with spaces still match diff --git a/changelogs/unreleased/10260-Lyndon-Li b/changelogs/unreleased/10260-Lyndon-Li new file mode 100644 index 000000000..418371f47 --- /dev/null +++ b/changelogs/unreleased/10260-Lyndon-Li @@ -0,0 +1 @@ +Add cap for backup data extraction \ No newline at end of file diff --git a/changelogs/unreleased/10269-blackpiglet b/changelogs/unreleased/10269-blackpiglet new file mode 100644 index 000000000..b23d58520 --- /dev/null +++ b/changelogs/unreleased/10269-blackpiglet @@ -0,0 +1 @@ +Remove PVC and PV inclusion check during creating PVR. \ No newline at end of file diff --git a/changelogs/unreleased/10270-Lyndon-Li b/changelogs/unreleased/10270-Lyndon-Li new file mode 100644 index 000000000..5bdc7e773 --- /dev/null +++ b/changelogs/unreleased/10270-Lyndon-Li @@ -0,0 +1 @@ +Cap the metadata decompression in object store \ No newline at end of file diff --git a/changelogs/unreleased/10279-harshitsaini17 b/changelogs/unreleased/10279-harshitsaini17 new file mode 100644 index 000000000..d524acdfa --- /dev/null +++ b/changelogs/unreleased/10279-harshitsaini17 @@ -0,0 +1 @@ +Use the well-known label constants exported by k8s.io/api/core/v1 instead of hardcoded label strings for kubernetes.io/hostname, kubernetes.io/os, kubernetes.io/arch, topology.kubernetes.io/zone and failure-domain.beta.kubernetes.io/zone diff --git a/changelogs/unreleased/10280-nitishmalang b/changelogs/unreleased/10280-nitishmalang new file mode 100644 index 000000000..f0a5e49b9 --- /dev/null +++ b/changelogs/unreleased/10280-nitishmalang @@ -0,0 +1 @@ +Bound WaitRestoreExecHook polling with resourceTimeout to avoid an infinite wait when restore exec hooks never complete. diff --git a/changelogs/unreleased/10283-opbot-xd b/changelogs/unreleased/10283-opbot-xd new file mode 100644 index 000000000..5f7c3aa48 --- /dev/null +++ b/changelogs/unreleased/10283-opbot-xd @@ -0,0 +1 @@ +test: add verification for skippedPVTracker in backup tests diff --git a/changelogs/unreleased/10292-samay43 b/changelogs/unreleased/10292-samay43 new file mode 100644 index 000000000..9b688ea86 --- /dev/null +++ b/changelogs/unreleased/10292-samay43 @@ -0,0 +1 @@ +Fix nil pointer dereference in EnsureDeleteVS and EnsureDeleteVSC when the API call times out before the object is retrieved diff --git a/changelogs/unreleased/10293-samay43 b/changelogs/unreleased/10293-samay43 new file mode 100644 index 000000000..eb5166687 --- /dev/null +++ b/changelogs/unreleased/10293-samay43 @@ -0,0 +1 @@ +Fix nil pointer dereference in EnsureDeletePVC, EnsureDeletePV and EnsureDeletePod when the API call times out before the object is retrieved diff --git a/changelogs/unreleased/10308-kaovilai b/changelogs/unreleased/10308-kaovilai new file mode 100644 index 000000000..ef832d521 --- /dev/null +++ b/changelogs/unreleased/10308-kaovilai @@ -0,0 +1 @@ +Fix block data mover cancellation being reported as a backup failure diff --git a/changelogs/unreleased/10312-samay43 b/changelogs/unreleased/10312-samay43 new file mode 100644 index 000000000..6b6cb3704 --- /dev/null +++ b/changelogs/unreleased/10312-samay43 @@ -0,0 +1 @@ +Assert expected errors from the test case rather than the returned error in pkg/util/csi tests diff --git a/changelogs/unreleased/10315-opbot-xd b/changelogs/unreleased/10315-opbot-xd new file mode 100644 index 000000000..9a0abe63d --- /dev/null +++ b/changelogs/unreleased/10315-opbot-xd @@ -0,0 +1 @@ +Testing: Implement missing unit tests for pkg/backup/snapshots.go diff --git a/changelogs/unreleased/10317-opbot-xd b/changelogs/unreleased/10317-opbot-xd new file mode 100644 index 000000000..0a18a2ec3 --- /dev/null +++ b/changelogs/unreleased/10317-opbot-xd @@ -0,0 +1 @@ +Cleanup: Remove deprecated --wait flag from velero uninstall diff --git a/changelogs/unreleased/10322-Lyndon-Li b/changelogs/unreleased/10322-Lyndon-Li new file mode 100644 index 000000000..e7f6baf9a --- /dev/null +++ b/changelogs/unreleased/10322-Lyndon-Li @@ -0,0 +1 @@ +Fix issue #10321, when data mover pod is evicted get the message from the data mover pod instead of the terminal message \ No newline at end of file diff --git a/changelogs/unreleased/10329-lubronzhan b/changelogs/unreleased/10329-lubronzhan new file mode 100644 index 000000000..c24f6e0a2 --- /dev/null +++ b/changelogs/unreleased/10329-lubronzhan @@ -0,0 +1 @@ +Replace generated `config/crd/{v1,v2alpha1}/crds/crds.go` with `go:embed` of the CRD YAML bases, removing the codegen step and its CI drift check diff --git a/changelogs/unreleased/10333-reasonerjt b/changelogs/unreleased/10333-reasonerjt new file mode 100644 index 000000000..7cce9187b --- /dev/null +++ b/changelogs/unreleased/10333-reasonerjt @@ -0,0 +1 @@ +Enforce namespace of the "musthave" resources in restore \ No newline at end of file diff --git a/changelogs/unreleased/10338-blackpiglet b/changelogs/unreleased/10338-blackpiglet new file mode 100644 index 000000000..bf45dad9e --- /dev/null +++ b/changelogs/unreleased/10338-blackpiglet @@ -0,0 +1 @@ +Avoid duplicated InitContainer names generated in velero install CLI. \ No newline at end of file diff --git a/changelogs/unreleased/10339-shubham-pampattiwar b/changelogs/unreleased/10339-shubham-pampattiwar new file mode 100644 index 000000000..9f23a6668 --- /dev/null +++ b/changelogs/unreleased/10339-shubham-pampattiwar @@ -0,0 +1 @@ +Add readWriteOncePod backupPVC config to enable mount-level SELinux labeling diff --git a/changelogs/unreleased/10342-shubham-pampattiwar b/changelogs/unreleased/10342-shubham-pampattiwar new file mode 100644 index 000000000..adc848948 --- /dev/null +++ b/changelogs/unreleased/10342-shubham-pampattiwar @@ -0,0 +1 @@ +Fix issue #10341, avoid mutating the cached node-agent LoadAffinity so the OS node selector term is not appended repeatedly to data mover pods diff --git a/changelogs/unreleased/10343-chlins b/changelogs/unreleased/10343-chlins new file mode 100644 index 000000000..0c87d3e52 --- /dev/null +++ b/changelogs/unreleased/10343-chlins @@ -0,0 +1 @@ +Only sync finished backups from object storage diff --git a/changelogs/unreleased/10344-Lyndon-Li b/changelogs/unreleased/10344-Lyndon-Li new file mode 100644 index 000000000..ddd6aeeef --- /dev/null +++ b/changelogs/unreleased/10344-Lyndon-Li @@ -0,0 +1 @@ +Fix repo connection contest of the two repositories with the same storage type \ No newline at end of file diff --git a/changelogs/unreleased/10346-reasonerjt b/changelogs/unreleased/10346-reasonerjt new file mode 100644 index 000000000..eab333072 --- /dev/null +++ b/changelogs/unreleased/10346-reasonerjt @@ -0,0 +1 @@ +Double check the label for backup when deleting VSC- #10346 diff --git a/changelogs/unreleased/10352-samay43 b/changelogs/unreleased/10352-samay43 new file mode 100644 index 000000000..53ac1a9db --- /dev/null +++ b/changelogs/unreleased/10352-samay43 @@ -0,0 +1 @@ +Fix nil pointer dereference in WaitUntilVSCHandleIsReady when a VolumeSnapshotContent error has no message diff --git a/changelogs/unreleased/10357-samay43 b/changelogs/unreleased/10357-samay43 new file mode 100644 index 000000000..1c25341db --- /dev/null +++ b/changelogs/unreleased/10357-samay43 @@ -0,0 +1 @@ +translate parent snapshot "auto" to an empty parent snapshot in both data mover micro services diff --git a/changelogs/unreleased/10359-lubronzhan b/changelogs/unreleased/10359-lubronzhan new file mode 100644 index 000000000..b0696bb21 --- /dev/null +++ b/changelogs/unreleased/10359-lubronzhan @@ -0,0 +1 @@ +Fix e2e kind test matrix generation misparsing kindest/node pre-release tags (e.g. v1.37.0-rc.1) as bogus patch versions diff --git a/changelogs/unreleased/10370-samay43 b/changelogs/unreleased/10370-samay43 new file mode 100644 index 000000000..3ffb47ca0 --- /dev/null +++ b/changelogs/unreleased/10370-samay43 @@ -0,0 +1 @@ +fix log format string mismatches that produce wrong or mangled output diff --git a/changelogs/unreleased/10371-samay43 b/changelogs/unreleased/10371-samay43 new file mode 100644 index 000000000..d1ab7571e --- /dev/null +++ b/changelogs/unreleased/10371-samay43 @@ -0,0 +1 @@ +prevent panic when the restore hook init container command annotation is empty diff --git a/changelogs/unreleased/9795-kaovilai b/changelogs/unreleased/9795-kaovilai new file mode 100644 index 000000000..6394ff4d3 --- /dev/null +++ b/changelogs/unreleased/9795-kaovilai @@ -0,0 +1 @@ +Skip DeleteSnapshot when ProviderSnapshotID is empty diff --git a/changelogs/unreleased/9920-shubham-pampattiwar b/changelogs/unreleased/9920-shubham-pampattiwar new file mode 100644 index 000000000..b9bda73f9 --- /dev/null +++ b/changelogs/unreleased/9920-shubham-pampattiwar @@ -0,0 +1 @@ +Support copying namespace-scoped secrets and configmaps for backup and restore PVC provisioning to enable datamover backup/restore of encrypted CSI volumes diff --git a/config/crd/v1/bases/velero.io_downloadrequests.yaml b/config/crd/v1/bases/velero.io_downloadrequests.yaml index 9db2e9fb8..771bf8daf 100644 --- a/config/crd/v1/bases/velero.io_downloadrequests.yaml +++ b/config/crd/v1/bases/velero.io_downloadrequests.yaml @@ -16,7 +16,23 @@ spec: singular: downloadrequest scope: Namespaced versions: - - name: v1 + - additionalPrinterColumns: + - description: The type of file to download + jsonPath: .spec.target.kind + name: Target Kind + type: string + - description: The name of the resource the file is associated with + jsonPath: .spec.target.name + name: Target Name + type: string + - description: The status of the download request + jsonPath: .status.phase + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 schema: openAPIV3Schema: description: |- @@ -79,8 +95,9 @@ spec: description: DownloadRequestStatus is the current status of a DownloadRequest. properties: downloadURL: - description: DownloadURL contains the pre-signed URL for the target - file. + description: |- + DownloadURL contains the pre-signed URL for the target file. It is signed for a fixed + lifetime and expires at Expiration, so it should be used promptly and not cached. type: string expiration: description: Expiration is when this DownloadRequest expires and can @@ -88,13 +105,24 @@ spec: format: date-time nullable: true type: string + message: + description: Message explains a Failed phase. It is empty in every + other phase. + type: string phase: - description: Phase is the current state of the DownloadRequest. + description: |- + Phase is the current state of the DownloadRequest. Processed means a URL has been + signed into DownloadURL. It does not mean the target object exists in object storage, + so a request whose target never produced a file still reaches Processed and the URL + returns 404. Callers should check that the backup or restore is in a phase that + produces the target before relying on the download. enum: - New - Processed + - Failed type: string type: object type: object served: true storage: true + subresources: {} diff --git a/config/crd/v1/bases/velero.io_podvolumebackups.yaml b/config/crd/v1/bases/velero.io_podvolumebackups.yaml index 3f4c83deb..90e9f4e4a 100644 --- a/config/crd/v1/bases/velero.io_podvolumebackups.yaml +++ b/config/crd/v1/bases/velero.io_podvolumebackups.yaml @@ -96,6 +96,13 @@ spec: description: Node is the name of the node that the Pod is running on. type: string + parentSnapshot: + description: |- + ParentSnapshot specifies the parent snapshot that current backup is based on. + If its value is "" or "auto", the data mover finds the recent backup of the same volume as parent. + If its value is "none", the data mover will do a full backup + If its value is a specific snapshotID, the data mover finds the specific snapshot as parent. + type: string pod: description: Pod is a reference to the pod containing the volume to be backed up. diff --git a/config/crd/v1/bases/velero.io_serverstatusrequests.yaml b/config/crd/v1/bases/velero.io_serverstatusrequests.yaml index af35815fe..f7f6de9a4 100644 --- a/config/crd/v1/bases/velero.io_serverstatusrequests.yaml +++ b/config/crd/v1/bases/velero.io_serverstatusrequests.yaml @@ -16,7 +16,23 @@ spec: singular: serverstatusrequest scope: Namespaced versions: - - name: v1 + - additionalPrinterColumns: + - description: The status of the server status request + jsonPath: .status.phase + name: Status + type: string + - description: The Velero server version + jsonPath: .status.serverVersion + name: Server Version + type: string + - description: The time the request was processed by the controller + jsonPath: .status.processedTimestamp + name: Processed + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 schema: openAPIV3Schema: description: |- @@ -82,3 +98,4 @@ spec: type: object served: true storage: true + subresources: {} diff --git a/config/crd/v1/crds.go b/config/crd/v1/crds.go new file mode 100644 index 000000000..111b68cdc --- /dev/null +++ b/config/crd/v1/crds.go @@ -0,0 +1,58 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package crds embeds the controller-tools generated CRD manifests from +// ./bases into the binary via go:embed. +package crds + +import ( + "embed" + + apiextinstall "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install" + apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/client-go/kubernetes/scheme" +) + +//go:embed bases/*.yaml +var basesFS embed.FS + +var CRDs = crds() + +func crds() []*apiextv1.CustomResourceDefinition { + apiextinstall.Install(scheme.Scheme) + decode := scheme.Codecs.UniversalDeserializer().Decode + + entries, err := basesFS.ReadDir("bases") + if err != nil { + panic(err) + } + + objs := make([]*apiextv1.CustomResourceDefinition, 0, len(entries)) + for _, entry := range entries { + data, err := basesFS.ReadFile("bases/" + entry.Name()) + if err != nil { + panic(err) + } + + obj, _, err := decode(data, nil, nil) + if err != nil { + panic(err) + } + objs = append(objs, obj.(*apiextv1.CustomResourceDefinition)) + } + + return objs +} diff --git a/config/crd/v1/crds/crds.go b/config/crd/v1/crds/crds.go deleted file mode 100644 index 7887493a6..000000000 --- a/config/crd/v1/crds/crds.go +++ /dev/null @@ -1,69 +0,0 @@ -/* -Copyright the Velero contributors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by crds_generate.go; DO NOT EDIT. - -package crds - -import ( - "bytes" - "compress/gzip" - "io" - - apiextinstall "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install" - apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "k8s.io/client-go/kubernetes/scheme" -) - -var rawCRDs = [][]byte{ - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xccW͎\xdb6\x10\xbe\xfb)\x06鵲\x1b\x14-\n\xdf\x12\xa7\x01\x82&\xc1\xc2\x0e\xf6NSc\x99Y\x8aTɡS\xf7\xe7\u074b!%[\x96hk\xbd\x87\xa2\xbc\x89\x9c\xffo\xe6#U\x14\xc5L4\xea\x11\x9dW\xd6,A4\n\xff 4\xfc\xe5\xe7O\xbf\xf8\xb9\xb2\x8b\xc3\xebٓ2\xe5\x12V\xc1\x93\xad\xd7\xe8mp\x12\xdf\xe1N\x19EʚY\x8d$JAb9\x03\x10\xc6X\x12\xbc\xed\xf9\x13@ZC\xcej\x8d\xae\xa8\xd0̟\xc2\x16\xb7A\xe9\x12]4\u07b9>\xfc0\x7f\xfd\xf3\xfc\xa7\x19\x80\x115.a+\xe4Sh\x1c6\xd6+\xb2N\xa1\x9f\x1fP\xa3\xb3seg\xbeA\xc9\xd6+gC\xb3\x84\xf3A\xd2n=\xa7\xa8\xdfFC\xeb\xce\xd01\x1ei\xe5\xe9\xb7\xec\xf1G\xe5)\x8a4:8\xa1s\x81\xc4c\xbf\xb7\x8e>\x9f\x9d\x15\xc0\"\xe9H\x99*h\xe1F\xba\xec\xdbK\xdb\xe0\x12\xa2f#$\x963\x80\xb6\b\xd1R\x01\xa2,cY\x85~p\xca\x10\xba\x95ա6'?_\xbd5\x0f\x82\xf6K\x98w\x85\x9fK\x87\xb1\xe6_T\x8d\x9eD\xddDٮ\x96o*l\xbf\xe9\xc8\xceKA86\xc6E\x9d\x9fc\xfdrl\xf0\xc2ʹF\xd0;K\x16=9e\xaa\xd9Y\xf8\xf0:\x95B\xee\xb1\x16\xcbV\xd66h\xde<|x\xfcqs\xb1\r\xd08۠#\xd5\x153\xad^g\xf6v\x01J\xf4ҩ\x86b\xdf\xfc]\\\x9c\x01\xb0\x83\xa4\x05%\xb7(z\xa0=v5Ʋ\x8d\t\xec\x0eh\xaf<\xc3\xe6УIM\xcb\xdb\u0080\xdd~EI\xf3\x81\xe9\r:6\xc3\xd8\a]rg\x1f\xd0\x118\x94\xb62\xeaϓm\x0fd\xa3S-\b=AD\xd1\b\r\a\xa1\x03~\x0f\u0094\x03˵8\x82C\xf6\t\xc1\xf4\xecE\x05?\x8c\xe3\x93u\b\xca\xec\xec\x12\xf6D\x8d_.\x16\x95\xa2n^\xa5\xad\xeb`\x14\x1d\x17q\xf4\xd46\x90u~Q\xe2\x01\xf5«\xaa\x10N\xee\x15\xa1\xa4\xe0p!\x1aU\xc4DL\x9c\xd9y]~\xe7\xda\t\xf7\x17nG@\xa7\x15\x87\xec\x0exx\xea@y\x10\xad\xa9\x94\xe2\x19\x05\xde\xe2ҭ\x7f\xdd|\x81.\x92\x84T\x02\xe5,:\xaaK\x87\x0fWS\x99\x1d\xba\xa4\xb7s\xb6\x8e6є\x8dU\x86\xe2\x87\xd4\n\r\x81\x0f\xdbZ\x11\xb7\xc1\xef\x01=1tC\xb3\xab\xc8i\xb0E\b\r\x8fN9\x14\xf8``%j\xd4+\xe1\xf1?ƊQ\xf1\x05\x83\xf0,\xb4\xfaL=\x14N\xe5\xed\x1dt,{\x05\xda!sn\x1a\x94\x8c,\x17\x97U\xd5N\xc94S;\xeb@\x8c\xe4/+\x95\xa7\x00^\x89D7d\x9d\xa8\xf0\xa3M6\x87BSm\xc7\xebm\xceP\x171\xd3V\xe2\x04\xcc\vf\f\xd2^P\x8f\fH(s\xe2\x94l\x927\x90\x89\xe8\bf\n#\x8c\xc4\xf7\xb1\x1f\x8d\xd8\x1b\xb4\x8dd\x05\xa4*\xed$\xcd\xfb\x15\x96\x82\xdc\xd0\x02\xf8\r\xd5\xf0ʼ\xb2\\\xd1s˄$n\xb5-~\xbf\xb2#o\xebC0\xdc#\xacu\x8a宄\xac3\xd1l+\xb6f\x99\x9bNk\xa9\x1a\xbd\xe3\x14q\x97B\xf1\xa9o\x8b\xab}o\xc7\xd6\xfb\x12\x1d\x88\xad\x18:\aM\xb6\xf2)h\x1b\x8b\xb0\x159\v\x10rR\x953\xf2\xc4\xccv\x00\x94\x90Rj\xcdV\x1c\xfc\xbc#Ld\xbcʭH~\xa88Ge\xb6\x14\x99\x82ª\v\xdeg5! \xaab8\xd89\xb6\x8e\xfc܂5\xf8:\xc2@[2\xcd\xee\x04-\xf5V\xa2\xa9\x92\x95\x99 \xd0`\x12\xdars\xb7\xecAiQ\xcf\x04\xfbYiȭ6{\xa2\xcc 3o\xee\x96\xe4\x01\xe9\x1aZ\a\xcf\xc7TJ\xd8\xe9\x13\xe9\xeb3\xd0|\x7f/\x7f\xd6\x10\x1c\x81`\x1agd\x05k;E\x14\xd8\xf6\xf6\x13\xfa\"օ2nXC2\x13\xb4\xef9\xaciōW L\x93\xb7\x7f&\x05\x13\x95\x19\xcc\xc1\x83Դ\xd2Q\xc8\x1d\xa8S\x88\xf8\x8e\x1a\xfa\x93mܣ\x1d\x8a\x1cB\xb5\xc4[y:\xae\xf6-\x7f$\x86\xd6r݂\xc84\xb9\xb8 R\x91\v\xe72_\xcc\x1ch\x8f\xb6u\xc6͜\x89v_O\x8c\xf3\xd0\xdbqDp@\x1dc\xf5\xbd\xfc\xa0ݤ:\x89&#\xb0Z$zڂق\"\xa5\xac]\x825\xe3@\xf4^\x1b(\x82\xc3\xe6ͬ\xc7'\xd2\x13*\x17\xce=\bm\xe9\xeb\x11\x19\"/*\xce\xe9\x8a\xc3\x151\xaa\x82\x11ڬ\xa4\xe4@\xc5\x04q>\x836,;\ai\x1c\xa4\ba\x94\xffС\x00z\x15\xf4\x11\b\x8d\x80\xf64\xb3\xee\v\xe7-\xc2v\xa9\x12\x1dS\xa9 \xb3f\xedʛK\x06\x1cM\xb4\x90\x84K\xb1\x01\xe5z\xb7\xda/\b\x98\x02+p9\xb1\x96H\x01\xb7斬+k\xa4\x16\xc4\xce\xf2Q\x19`B\x1b\xa0\x11\xe1|\x06\x7f\xe0\x8b\xd5Ґ\xdf8\xcf\xf4\xce.\xef\xf2\xb0\xdc\x1d\x98\x95\x14>\xbd?\bѻ/\x9ce\xe8%{\x87x\x8e\xcbʘ\x986^\x8c5Q\xb8浬\xf4\xc3nܓ\x83zA\x83\xb1\x8d.\xfet1C\x0ew{\xed\xf6\xa1\tUP\x93%Y\x7fBQ\x9a\xfd\xb063PD\xa8xP\x9f$\xf2\x93*E\xf7#ܬ\x97\xe7g\xe4\xe7\x18\xcc\x1eGE\xa8\xf6\xca<\xed\xf7\xfb\xef\xcc\xd5\xf3\xf0Qc\x98\x8a2a\xf9Ǚ6\x1d\xf6i\xb7\xc0\xb5d\x13\xd2D\xe09\xff\x0er\\\xbb\x1e\xe0\xd6\xefD\xac\xb3\xc8\xfc\x98\x90ײ\xe5\x85\xf7_\x92R[)\x1f\xa7\xa8\xf3\x83\xadӬ\x1aI\x86\xe1P\xb2\x82-\xdd1\xa9<ꍩ\x85/\x90U&:\xeb\xa9!9[\xafAY8\x18\xba\xd3.\x8e0N\x90\xf1\xf5\ri\xa9\x91\xe8\xc7\x1e\x1e\r#-\x9b\x10\xf3\xb1\xa1[?\xa2o%C\xb1\x03\xb5n6\x1a\xe3\x9c\xedX^Q\x8ev\x99\x8a\xcc\xe1C\xebqŴ\xcc\x01&\x0f\xc6\x1c\x95LW\x9cC\x10\x90\xb2L\xea,%\xa5\x00\xeb\xfb\x16vm0\xac:\x8e\xf9\x8aZ_E\x8eaO\x90Y\xaa\xe2\xa0}W9\xba\x91\x8dΘ5L\xc1H\r\xe1t\x05\x9ch\xe0\x90\x19\xa9\xe2\x14\x99\xe2\xb3+)Jp\x84\x90\x11\xcd\xd7]q4\b\x1c\x00Ip)\xb7e\xd9ֹzV\x88\x10\x0e\xc9%X\x87\xcf\x10Z\x960\xe7\x80ڕ\xddE3\xbf\x9d\xb6\xbfp\xfb%\xd8\t\xcd\xd0c\xc1\xb6\xa5\x92\x19\xe8\xe8^vS\x12\xecE$[\xa4\x8d{\x1ds\xa4n\x95\xe4rV\x0e\x87@CIwy-!\x8e\\/\xbc\xff\xd2\n\x88ڹo\xff\x9e\x92\xb1c\xc7E0\u05f7(h?\x8f+i\x887\xaee\x98\r\x1e\x90[|\xa8M\x85\x9a Ֆ\xd7\x02\xf858\n\x05\x13K쀼}\x01\xc7\xc2\xeb\xd0X\xd2I\xac\x9c\xe6\xcaބN\x1a\xee\xd4?\xb8\xa9\\J\xdc*P\xd0a\xde0\xaa\x8e~\xa8\x90\xa6\x15\x908\xc2\xdd,e\xfe\x9d&k\xa6\xb4i\x0fA\x8f\xa4\xa9D\xc1\x1c\xb9\xf0\x12\x98\xbd|\x02q?\xb9\x96\xbdD2\x9f\xbf\xe6\b\x93\x889\xee/\x01ak\xc2\f\x01\x91\xc9J`\x00\xc7\xcec\xec\xc2\x11\xd7iX\x96:I\xd2f?\x19\xcdE\x8b\x959J\n\x13\a#=\xed\xea\x1f(\x1b&\xac\xc5ʑl3c\xd9l\xb1rڜ\b\xa9n\xed\x8cł~aEU\x10ZX\x1e\xa11g\x05t\x99\xde$\xc0\xd9\x16h&\x8c\xb43\xa6\xe4`\xc0'\xb1%\x8e!\x93B\xb3\x1cj\xe3\xea\x05A\nBɚ2^\xa9D\rx\x14y\x8fY\x8axMp\xbe5FZ\xe7s$EB47\xd1W<\xac\x8dK\x95\xee\xf1M\xb9Y\n\x8e\xf7\xb2J\xc5$\xa6\a\x9e\xd9\xd1\xf2\t\x95T\xec\xbfyZ\xa9C\xfd\xe6i\x1d*\xdf<\xad\x89\xf2\xcd\xd3\xfa\xe6i\xa5\xd4\xfc\xe6i}\xf3\xb4\xda\xe5\xff\x84\xa755\xa29\xc6\xd0F>N\x8e\"a\xab\xfa\xd0\x10\x0f\xc0\xf7\xc9\x15>\a\xfcY\xb9\x98\xcb8\xa8H\xe2\xffHZwLi5ƣNδ\xb3&ȼ;\x7f5\xe1J>#\xeb>tz\xbe\xac\xfb\xe5A\x88gʺ\xf7Þ\xf6\xb1Oʹ\x0fD9.;{\xe6\x135\n\xa0!\xac\xee\xb6\xe1cx\x8dI\xc8D\xff\xaf\x9c\x98;\xc8\x1a;\xa3|\xbcx\x16\x7f\xb2\x8cDYz\U000672ef\x8f\xfc\xe7!\xf8(\x89\x87\xb4\xf3\a\xc0#P\xed\n\xb4\x9d\x16\xd6\xcd\xc2\xfb:\xc5\xf8,r\x9b\x9a\x89_\x131\x02\xab+\x92=*~\xad\xba\xc0@\xf1\xa9\xf4\x16\xe9\x19'V\x97\x118IgV\xa9ދl\xab\xa4\x90\x95\xf6Q\t\v\xeb:s'\xfe\x03Ș\xb0Fg\xf8\x7f\x90\xad\xac\"\x99\xe0\a\xc87\x91\x118\x8d|'9\xd0oB\x83\xa1\xbb\xb7\x8b\xee\x17#}\xaa\xe0\xd8\x19\xe7\xa7-\b\xdca\x17\x9b\xf6\x01\x80pa\x83\xbf\xb9\xa0/`\x11@R\x11\xc1\xb8\x93\xbc\xfa\xba\x87\xb6ܑO\xa5\x8b=\x1d\xedw\x1c\x8e\xa9\xa4%\x13\x9e\x9cB\xd8M\x11\x1c\xf1K\x8f\xdd\xed>ˑ\x89\xdf%5\xf0\xf8\x84\xc0\x94\x88\xd8D\xf2\xdf\t)\x7f\x89\xb9\xc5\xcfޞOI\xea;f\xc5\xfcb\t|\xe7O\xdbK\xa2\xcft\x8a\xde1\xd4y\xf1t\xbcWL\xc2{\x9dԻĄ\xbb\xf3eΧ\xc5cO\xca\x1c\x9b\x0e\x1d\x8c'\xcdM\xa6\xcaM\x86\x16\xa6\x10;\x1a\xa5\xc9\x14\xb8c\x12\xdf&\xb9\x936\xcd^-\xb5\xed\xd5\x12\xda^7\x8d\xed\xa0\x14\x1d\xfcxL\xa2Z\xfc\xde\x1e2il\a\xb7\xfa\r*\x9cS\xe2\x92cq\xa3\x93\x8e\xbf\xd6\xe48\x95mRu\xdc\xed\x93փ\x9fz0\xac\xa0\x06W\xf4\x95|\xfa\xa2↕\x1c7~w,\x8f\x06G\xcc\x16\xf6\xf5\x85\x1f\xbfJ<*\xebo\xb0\xf9\xf4\xb9\x9ee\x8b\xdeʄj\xf2\x04\x9c\x13\x1a\xd3\x03\x03\xcc3w\xb5V&\xe7`\xed\xa7\xd5&\xfe\"\x13\x7f\x1f\xd7\xccMO<\r\x8cV\xb8\x88\x85Ĩ\x18\xbf\xf5f\xd4Х\xe8ǁ\xc7\xed\xd6\r\xf8\xdbo\x15\xa8=\xc1{wj\xbf\xac9\xb4\xe6\x15\x89\xb6\vǠڼ\x9a\x1d\x8b\xf7\x0f\x16)\x8d\xea!\xd7\xc2y\t\xfd\xf1`\x1b\xabӚE\x98U\xd4\"v\xe1\x14\t\x13l\xd8\\Ⱥu\xa4ٔC\x9fz\xba\xebe\x97d\xc7/\xca&\xbd\xa0tO\xf5w:\xb5u\xcai\xad\xb4\x84\x85\xc9\xd3Y/\xb5D\x9bZ\xa4%\xfb\xa5i\xa7\xaf\x8e\xdb\xdc|\xc1\xd3V/q\xca*\x91R)\xa7\xaa\x8e\xa3\xd3+\x9c\xa2z\xd5\xd3S\xafuj*\xf9\xb4TRJN\xf2\xaeujJ͉\xc7\x7f\xa6\xf7\xa4\x0f\x9f~J8\xf5\x94\xb0[=\x8d\xe4\t\xe8%\x9cj:\xee4S\x02\xcfR\xa7\xe2+\x9eZz\xc5\xd3J\xaf}JiB\xb2&>\x1fw\x1a\xe9\xe4-\x16\xa9rP\a\xb7\xa9R\xa5\xf0\xa0\xfc\xa5\xacm\xba\x03\xe9\xedτ[\nm\xad\x8e\xbf\x8c\xe6\xc1\xdf\x1c\x8bw\x04\x8fm\xb7ZIky\x1b\x9d\xbd\xb3\xc6\xfd\xe9:\x93\xfe\xe2`\xb7\xbd\xa6\xa1\xa4\n/\xa3^\xed]\xfaM\xd44\xbf\xa7ٶ\a}K5YKUPC.\xea\r\xcbK\a\xdc\xfe}\xb1 䃬s8\xda\xf7\biV\x94|oW(\xe4\xa2\xdd\xe04\t\x88J[\xe8\xedVr\x96E|\xb7\xe8]R\xae\xf2\xe0r\x0f\xbc\xe1*k\xa78\x94\xb6b\xdcuC7\xaf{e\xe7Zr.\x9f\x8e\x8dU\x94\xec/\xf8D\xc03\xa2Y\u05f7K\x84\x11\xc4\x03\xdf\x1c\xa8\x93\xc9jlV`\xcdr\x83\xe7\xd8\xdc_\xae;\x10\xbby\x99\xedێ!w\x17[\a\xb7\xc0\xab\xceLZ\xedr\xbbt\xe3\x18\xeb\xc5\xca\f\x15{\"1\x03\xc8l\x99\xca\xe7%Uf\xef\x12Kf\x9d1\x04[z(\x1a5j=\x86\x97uG\xc9\x1b\xee\xe8\xc6\x1d\xd5}\xd9ݤ\xee\xd3\xee\x94q\x8c\x9f\xb6\x9c\xc4[\xb5\x9c㖾pޱ\\G\x10\x18\x83\xd3z\xd8\xe5\x89\x19\x7f\xd1\xd8yo\x86\x1d[\xf2\x8c=Y\x81O\x11L?Z\xe1^,\xf0O\xdd\xf8\xe9X)\xbc\xd6տf\x80נ>\xe3݊N\xb2\x9a\xbe6\x06\x8a\xd2\xc4|\x8diu\xf8\xfd!\x80\xb5\x9f\xd6{\x80\x89\x86\n1O[\xefEv(\x11\xcek\xa3\x03\xdc<4\x1fc\x04\xb8\xf1\xe77\xceF\x80\x1a\xe0\x18\x01t\x95e\xa0\xf5\xba\xe2|_\x1f\x1f\xf9J\xa8\xf1\x812~>R8h\xa3\x82`\xd1;\bi\x12a\x9f\x9e\x0e\"\x0f3=\x1c\xad:\x8e\x14\x9e\v\xed\x17\xb1N\xa1\xc1\xcd\x10\f\xbe\xc1\xa4\xf2V\x12(m?\xfbU\xb3?f\\\x1ap\xae%.\xb2,4\xc8\t\xec@\x10k\x9d\x1d\x89\xc3\xe3vGB\xf1'r\x9d\x85\xeb>\x836\xf2\xd2\x14\xf1\xd1\x0e\x8d/\x1a}\xa7k\x98\x98ۊ\xef\xb0\f\x890t~]\xb4\xc2=-6\xb7 N\xf3Z\xc7^\xa1\xe9څ\xe7)\xb9\x9b\xbb\xe5\x18\xb8ST\xdc\xf0\x99\x9agN\xe3!\xba\xcfRiCt\x8fRh\x11\x88\xb5\x8c\x9f\x1fw\xf7:\xe0I\x97л\xf7\b\xd1\xe1\xc8\u0099?ʹ?\x98Y\x80\xd6t\x13n\x9f\x7f\xb2K\x8f\r\bp\xe19\xb7y\x12\x01ڜ\xe2\xeb\u07bd\xee\xa6\f\xcdLEyx\t\xd1%$\xb7j}\x87O\x12F\xa0\xe2\x034,<\xfd\x16\xd6dG\x12\xeaK\xc9T\xca\x1a\xee}]\xd1\xd2\x06=a\xe4N\xf3X\x1fp\xb6\xc1\xa7\xa8,\xe76T\xad\xe8\x06\xe6\x99\xe4\x1cP[\x0f\xc7\xf5\x92sݟ\x95\xfc\fTO\xa2\xf6\xa1]\xd7\xef\x00:n\xbb\x8do\xea\xd2\xf3\xf196\xc3T\xef=\xc8\u0380$v|\x94\xa3\xec\xa8\x10}6p8\xd2v\xdd0\xeb\xbcZ\xf6q^\xffj\xe0\xacy\t,2\u0382\xfe*Ռ\x14L\xd8\x7f\xa8\xc8\xdd\x06^h|\xd4\xf8\xb7R>\xdeE\x9c\xd8\xc1\xe0\x7f\xa8+6[\x1dL\xb8a\xe3\x01ו\xac\xfc\xee{\xed\xd0ƷU\xf0%\x813/7\x11\xe6\x01{0@g4\xa2\xfbC\aҤ)p=\x8f\xc0\xba\vO\xd3q\xbe\x9f\xf5!\xf7\x9e\xc1l`\xb7^Z\xf0n@s\x7f\xc2HGaG*\n\xa4\xbe\xa8\xa3\xad\xd0OY\xf5z2\x8f9\x93\x03\x1a\xff\xd0\xd4\x1e\xa3\xa3\x1bf\xcb\xdd\x1bA\xb0\xe3\x04\x9ew\xc1\x8e\xcfjL\b\xff\xad\xadSߵ\xd0Z\xb8\x85,\xb1\xd1(\xdd\xd8\v}\x1fa\xb8]1'\x7f\xab\xa0\x8a\xd0`\x1e\x1e\xb4\xc37a#\x9f\x1d\x911\xa3\x03gc\xa4\xca\xe0m\xe0\xf6\xc7_(3Ll>Hu˫\r\x13\x9fƏ(\x1d\xaa|K\x95aV\xd8\xddxb\x03e\x82r\xf6\xf7\x98^k\x7f\x9c\x06t3\xba\xc0\x9a\x93\x84a\x8c}x\a\xd6\xc7\x1d\x8d\vDUh\xe9\xe9z\x8a\xbf\x12x2\xa5Sk_\xa2\xf1EB\xb7\v\xf2QF\x15\x83O\x87b]\x98\xd6%\x03m\xe6\xb0^Ke\xdcn\xf5|N\xd8:\x04\x1f\xac\xce\xc1\xb8\x99{|\x94\xb0\xd86s\x9dhҘ/\fz+\xb4\xc2x\xf5~A\xf7ng\x8afYe=\xacKm(\x8f88\xcfR\xfc\x18\xe5\xf9\x1e\x1f\xda\xfc\xf9Y;y\xcb6\xa0a\xd0\x11\xfbq$\xc5\xcb?\x9c\xd7\xc7-\x8a ȓb\xc6X\x9fJ\x1eH%\xf0\xa42ַ\xe2\x9chKꓢ\x8fĩ\xd1\xe5xJN\x1a\xca\xf75\x941\xf5\xec\xb1\xc6\x17%\xeb\xd7L}\xf6\x91\xafeٜm\xa9،ި\xb0U\xb2\xdal\x83$\x8f8\xd3$\xaf\x00\x83\xb5\xa8Rtx)\xdaTJ\xb4R\t\x0e\x1cS'A\x18p\xb84{\xc4wW\xddK\xcc\xfe\x01\xf8K\xfff\xcb|\xadd1\xf7\xfdb,u\xe6w\xf2\x15\x93\xd6s1\xdb(Չ\xf3\xda\xfd\xb3\b(\te\t\x82P\xed{N\xb8\xd9\xead3\xf5\x9b5\r\xb7R\xb3\x04o?\xca\xf1\xbf\xb5\x01\x04\x86\x97\xe1\xef.3\xfc\n\x06\xfb\x8c\xe1\xf1\xc9_\x19\x00;*\x8c[N\xd4&\xf2\xc2\x19\xb1\x8b\xa3\x162\xddw\xd0Oڻ\xeb@\x98\x88\xcf\xf8g\xd9c\xa8\xdd\xf9t\rwq\xd9M\xffE\xf5\x19\xd1L\x84\x97\xcc]ꇓ\xfe\xe8N\xa0\xc0\x875\xa5\x8agc\x1e\x0e\xb8t\x11z\xddXˮ\xf6$ޟ\xbc\x14\x7f\xe8\xc1\xe8\x1dB\xc7wT\xeb*a\xf9\xfc\a\x16\xdb\x0f\xc04\xde̢\xf2\xc7\xdf\xfdp\xf9.i\xa9\x17\xa7ȡ\x95\x1f.\xeaƗp\xddwSo9\xd8٦\x01\xba\x8bʣ\xe6\xdc\xee\x8cѴs\x86\xd2\u009b\xfd\xe7\x89%\xed\xce\x18D{\xb1\b\xdayQ~\xa2\xf8\xb0\xf5I\xb3\xf6\x17\xdf6\x12B\xf3`\xcf\x1dDk\xc5\xd0\xc2\xc0_5\x8a\x16\xb5\xb9\x83\x1fQO\xe7-m\xe1{j\xffR\xad\x9a\xe7\x15\xc9?\xfe\xf9\xe6\x7f\x03\x00\x00\xff\xff\xc3!Ko6\x87\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xccZK\x93۸\x11\xbe\xebWt\xed\x1e\xf6\xb2\x94줒J\xe96\x96\x93*W\xc6\xf1\xd4hvr]\bhJ\xb0@\x80\x01@\xc9\xca㿧\x1a\x0f\x89\xe2C\x0f;q\u008b-\x12h\xf4\xf3\xeb\x0f\xc0\x14E1a\xb5|E\xeb\xa4\xd1s`\xb5\xc4/\x1e5\xfdr\xd3\xed\x1f\xdcT\x9a\xd9\xee\xedd+\xb5\x98âq\xdeT\xcf\xe8Lc9\xbe\xc7Rj\xe9\xa5ѓ\n=\x13̳\xf9\x04\x80im<\xa3\u05ce~\x02p\xa3\xbd5J\xa1-֨\xa7\xdbf\x85\xabF*\x816\b\xcfK\xef\xdeL\xdf\xfe~\xfa\xbb\t\x80f\x15\xcea\xc5\xf8\xb6\xa9\x9d7\x96\xadQ\x19\x1eENw\xa8К\xa94\x13W#\xa7\x15\xd6\xd64\xf5\x1cN\x1f\xa2\x84\xb4z\xd4\xfc]\x10\xb6\x8c\xc2\x1e\x93\xb0\xf0]I\xe7\xff<>\xe6Q:\x1f\xc6ժ\xb1L\x8d\xa9\x15\x86\xb8\x8d\xb1\xfe/\xa7\xa5\vX9\x15\xbfH\xbdn\x14\xb3#\xd3'\x00\x8e\x9b\x1a\xe7\x10f\u05cc\xa3\x98\x00$\xd7\x04i\x050!\x82\xb3\x99z\xb2R{\xb4\v\xa3\x9aJ\x1f\xd7\x12踕\xb5\x0fΌ\xb6@2\x06\xb25\xe0<\xf3\x8d\x03\xd7\xf0\r0\a\x0f;&\x15[)\x9c\xfd\xa2Y\xfe\x7f\x90\a\xf0\xd9\x19\xfd\xc4\xfcf\x0e\xd38kZo\x98\xcb_c\x8c\x9eZo\xfc\x81\fp\xdeJ\xbd\x1eR\xe9\x919\xffʔ\x14A\x93\x17Y!H\a~\x83\xa0\x98\xf3\xe0\xe9\x05\xfd\x8a\x1e\x02r\x11B\xf6\x10\xec\x99K\xeb\x00좔\xe0\xa3aMUo\xad3\xb5I\x15x\xedH\x89\xfaӛ\xa4}Kl\xce\xef)\xb7x\x14\xe9<\xab\xea3\xb9\x0fk\x1c\x13v\xe6\x8a\xf7X\xb2F\xf9\xb6\xa9\x14%\xd5\xce\xcbs\xb3j\xe4S\x11g\x9d\xad\xf8\xfe\xec]\\ue\x8cB\x16\xa5\xc4Q\xbb\xb71\v\xf9\x06+6O\x83M\x8d\xfa\xe1\xe9\xc3\xebo\x97g\xafa(\x91:EA\x81c\xad\xd8l\xd0\"\xbc\x86\xfa\x8bqsɴ\xa3L\x00\xb3\xfa\x8cܟ\x82X[S\xa3\xf52\x17K|ZX\xd4z\xdb\xd1\xe9\x9f\xc5\xd97\x002#\xce\x02A\xa0\x841\xafR\xfd\xa0H\x96\x83)\xc1o\xa4\x03\x8b\xb5E\x87:\xc2\x14\xbdf:)8\xed\x88^\xa2%1Tۍ\x12\x84e;\xb4\x1e,r\xb3\xd6\xf2\xefG\xd9\x0e\xbcI\xc9\xec\xd1y\b\x15\xaa\x99\xa2dm\xf0g`Zt$W\xec\x00\x16iMhtK^\x98\xe0\xbaz|\xa4j\x90\xba4s\xd8x_\xbb\xf9l\xb6\x96>#47U\xd5h\xe9\x0f\xb3\x00\xb6r\xd5xc\xddL\xe0\x0e\xd5\xcc\xc9u\xc1,\xdfH\x8f\xdc7\x16g\xac\x96E0DGH\xadď6a\xba;[\xb6W\xd2\xf1\t\x90zGx\b^c\xcaDQ\xd1\xc4S\x14\xe8\x15\xb9\xee\xf9\x8f\xcb\x17Ț\xc4HŠ\x9c\x86\xf6\xfc\x92\xe3Cޔ\xbaD\x1b\xe7\x95\xd6TA&jQ\x1b\xa9}\xf8\xc1\x95D\xed\xc15\xabJzJ\x83\xbf5\xe8<\x85\xae+v\x11\xba\x18\xac\x10\x9a:\x80Dw\xc0\a\r\vV\xa1Z0\x87\xdf9V\x14\x15WP\x10n\x8aV\xbb7w\aG\xf7\xb6>\xe4\x9e:\x12\xdaA4X\xd6\xc8\xcf\xeaN\xa0\x93\x96*\xc33\x8f\xa1\xba:\x0eJP1ޔ\xf33\f\x12\xf40\xceѹ\x8fF`\xf7KG\xe5\x87\xe3\xc03\x1dk\xb4\x95t\xa1\xbdBil\xb7\xf3\xb0#\x92\xb7\x9f\x8cx݀\x03\xa0n\xaa\xbe\"\x05<#\x13\x9f\xb4:\x8c|\xfa\xab\x95\xbe\xbf\xd0H \xe9\x89*.\x0f\x9a?\xa1\x95F\\1\xfe]g\xf8\xd1\x05\x1b\xb3\x872\xe4\xbf\xf6\xea@\xd8\xe5\x0e\x9a\xf7Q;?\x0fO\x1f2\x82\xc7\xdaJ\x85\x99|5\x85\x87TԦ\x847 \xa4#\"\xe1\x82о\xb3t\xa3\x02ј\x83\xb7\xcd]\xe6s\xa3K\xb9\xee\x1b\xdd\xe6Fc\x19sEt\xc7s\x8b\xb0\x12\xa1\x16eGm\xcdN\n\xb4\x05Շ,%O\x9a46v\x90R\xa2\x12=l\x1a\xad\xb2`\x8aEAE\xcdԕ\x18.\x8e\x03\x03\x93fR\xc7\f>\t\bXc\xabԚ\xb5G-\xb0\xdbm\x826&\x00\x9aC\x01{\xe97\x11)\xd5P\xdd\xc1\xc5ڣg\x8b\x87\xa1\xd7\x1d\xdd_6H#c\xe3Ep\xc8-\xfa\x90m\xa8(}(\x95\xa6\x00\x1f\x1b\x17\xb0\xb6\x8b\x13\xf9\t\x84/\xcf\xde\xe2\xa1\xefh\xb8\x16\xdcD\x85FT\x0e$j\x0e?\xfcpݤ^w\xcb\x0fQ\xf7l\xa8\xc5\x12-\xea\x1e\x9b\xc8\xcfK\xe8Q\x944\x94aX\x96Ƚܡ:\x84\x9eD\xe0\xf93\xac\x1a\x0f\xa2\xc1\x105Ʒ{f\x85\x03n\xaa\x9ay\xb9\x92J\xfa\x03H7\"\x9f)e\xf6(Rı\xaa\xfda\n\x1f\xb4\xf3LstG\x1eD\x1e\x8b\xa9\xc0t\x1c\x95\xaa8\x10:f\x8700\x8a\xaf\x8c\xf3\xc0\xd1R:\xaa\x03\xec\xad\xd1\xeb1c\a\xda!\xed\x01\xadF\x8f\xa1#\n\xc3\x1d5C\x8e\xb5w3\xb3C\xbb\x93\xb8\x9f\xed\x8d\xddJ\xbd.H\xc1\"\x81\xcf,\xec\xecf?\x86\x7f\xbe&\vL\x1dq\xe2\x86\xe4]\x86Z?\x10\xbd\xf5\x1b\x8c-b\x19s\xd0X \x02A\xa9]\xa5܍\xc8:TvC\xbc\xbc\xfd\xe4\x90\x0f\xf5\x8f-\xf6[\xc7\x05P\x01\xf8R\x9c|[T\xac.\xe2h\xe6M%\xf9\xa4km\xcc\xfb\xcb\xf8\x937+R\vɉܞ\xe3F\xdeĉ\xb3=̀\x1b\xba\xbb\x9c1\xb4\x1cvS47q\x85+\x1a\x7fj\x8f=m}#t\xa7\xfe\xef\xd0\x13\xeft\xa0\x91\xf8\x01\xb3}?\a\xc0\xe4FkB*o\x80\x1d\xdb\xc0O\xae\xdb\xff\xeeD\xcfU÷8\xe0\xf8\x9e)\xef\xc2\xc0\xec\xe38\x8dti\x1c\x86\xc6tM\r\xb8^\x11\x9c-\xd0ޢ\xcb(\xf2-\x1eH\u0091[0X<\xc0\xaa\xd1BaVu\xbfAM\xdb1Y\x1e\x88\xec\xbf<.\xb3c\x03\x01K[\xa7\xec\xde1 yO\xbb\x00JA1\x87_\x1c\xa6u\x9f\xb1\x04\xa9\x9dG\xd6#\xe9\xf1\x89\xbdq\x0e\xab\xc3\x00\u05fa\xd9A\xcfX~\xbb\x8f\x82\xae\xe4\xa1\xd4 8\xc6\xc4J\xb0\x92\xfa{\xde\x0f-\x1e\x02\xc4\x12\xdf \"}\xe6ґe\xeetth\xd0i\xf1\fdR\xc7\x02a\xd5\xd8\":\x1fR\x01\x8by9H䇃q\xb9.\xe0\x12\xb3\xe89\xfb>v1*\x13\x80\xdd\xc80\xe0z\xb2\xc0E\xa6\x017\xb0\x8d\x9e\x99\xa39\x05w\xb2\x0e\xf8\x0e\xcc\x03\xfe\xfb\xec\x03\xeef \xf0\xddY\bܖ)\x97\xd9\b|\x13#\xb9\xe0\x8bK\\\x05\xae\xf2\x15\xb8\xc8Y`\x94\xb7\xc05\xee\x02w\xf2\x17\bx\x82\xa5\xfcr\x032?\x85\x81\xb9\x93\xd6\xcco\xa8kH\x81\xc0\x06\xfaj<\xa1\x18q\xd0q\xd3\xfb)\x85\xef+\xfa\xee%\xd2\x17չ\x87\xf7e@\xbfB\x8c\x9eҰ\xa3\x17\xf2\xef\x04 \xe7\a c\x04mТ\xdd\xf1\xb4\xfdO\xf1X\x81\x0f\xa0\xf8\x992\xaf\xfd\x19\x17\x8e'\xf2\x99\xff\x10K\xa3Ͱ\xb1\x16]m\xb4\xa0\xb6w\xdb\xe1\xc4I\xe5\xff\xdc\x11\xc5pX\x8bs\xfa\xda\xf9\x96\xa3p\xd3\xf9\\\xb8߸\xfb\x84.\xde\xfa\xb4Ͽ\xccʡݵ\x0e\xe9:6~\x97\xb3\xb9\xc1\xce\xd6:\xb0#\xaa\xa4\xa1\xd1\xe1\xc8\"4\xad\xe9d`F\x9b\x17\xfa\xd0<\xa4\x03m\xf64\xb9%-v=\x13\xe9M8\xb4dZ\xa4\xe3b\xfa4 y/\x95\xa2\x1ef\xb12\xe4,\xd4^Zj\x96,\xb4\xb1\xddo\xa6o\xfewg\x81\x8a9\xbf\xc6Qq\xff\x99\xa6\x00[\x99\xc6\x0f\xf4\xfeV\xc2\x0f\xd6t\xb8e\xbfG\xc7\xf0\xb7\x03\xd7\xe8\t\x8d\xc9\x11፵\xe1\xb2._\"ݱ\xd1\x1cC\xe0\x87Ο8\xb4\xbf\xf5\xff\x00\xe2\x06\xbb\x06\xbbt\xefe촭\xb8&'\xb7\xdf4\xab\xe3\x15\xec\x1c\xfe\xf1\xafɿ\x03\x00\x00\xff\xff%\xff\\)\x99#\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcVK\x8f\xdb6\x10\xbe\xfbW\f\xd0k%wQ\xb4(tk69,\xda\x06\xc6n\x90;M\x8e-f)\x92\x9d\x19z\xeb>\xfe{AR\xf2C\xb67\xc9%\xba\x99\xf3\xe4\xf7\xcd|t\xd34\v\x15\xedG$\xb6\xc1w\xa0\xa2ſ\x04}\xfe\xc5\xed\xf3/\xdcڰ\xdc\xdd-\x9e\xad7\x1d\xdc'\x960<\"\x87D\x1a\xdf\xe2\xc6z+6\xf8ŀ\xa2\x8c\x12\xd5-\x00\x94\xf7AT>\xe6\xfc\x13@\a/\x14\x9cCj\xb6\xe8\xdb\xe7\xb4\xc6u\xb2\xce \x95\xe4S\xe9\xdd\x0f\xed\xdd\xcf\xedO\v\x00\xaf\x06\xec\xc0\xa0C\xc1\xb5\xd2\xcf)\x12\xfe\x99\x90\x85\xdb\x1d:\xa4\xd0ڰ\xe0\x88:\xe7\xdfRH\xb1\x83\xa3\xa1Ə\xb5k\xdfoK\xaa7%\xd5cMU\xacβ\xfcv\xcb\xe3w;zE\x97H\xb9\xeb\r\x15\a\xee\x03\xc9\xfbc\xd1\x06̚\xaa\xc5\xfamr\x8a\xae\x06/\x00X\x87\x88\x1d\x94ب4\x9a\x05\xc0\bH\xc9Հ2\xa6@\xac܊\xac\x17\xa4\xfb\xe0\xd2\xe0\x8f\x95\x905\xd9(\x05\xc2\x0f=\x96\xcbC\u0600\xf4\b\xb5\x1cH\x805\x8e\x1d\x98\x12\a\xf0\x89\x83_)\xe9;h3\x92mu͍\x8c\x0e\x95\x847\xf3c\xd9\xe7\x86Y\xc8\xfa\xed\xad\x16X\x94$\x9e\x9a(um\xf0@'ȟ7P\xfc\xdb\xd8+>\xaf\xfeT\f\xb7*W\x9f\xdd]EZ\xf78\xa8n\xf4\r\x11\xfd\xaf\xab\x87\x8f?>\x9d\x1d\xc3y\xafWH\aˠ\xa6N3p\x155\b\x1e!\x10\f\x81&T\xb9=$\x8d\x14\"\x92؉\xff\xfa\x9d\xac\xd5\xc9鬅\x7f\x9b3\x1b@\xee\xbaF\x81\xc9\xfb\x85\\@\x1c\x87\x02\xcdx\xd1\n\xaee \x8c\x84\x8c\xben\\>V\x1e\xc2\xfa\x13jig\xa9\x9f\x90r\x9a<\xaeə\xbc\x96;$\x01B\x1d\xb6\xde\xfe}\xc8\xcd\xf9\u07b9\xa8SR \xc9c畃\x9dr\t\xbf\a\xe5\xcd,\xf3\xa0\xf6@\x98kB\xf2'\xf9J\x00\xcf\xfb\xf8#\x83h\xfd&tЋD\xee\x96˭\x95Ilt\x18\x86\xe4\xad\xec\x97E7\xec:I ^\x1aܡ[\xb2\xdd6\x8ato\x05\xb5$¥\x8a\xb6)\x17\xf1Ep\xda\xc1|G\xa3<\xf1Yً\xe9\xa9_ч\xaf\xa0'\xabE\x9d\x91\x9a\xaa^\xf1\xc8B>\xca\xd0=\xbe{\xfa\x00S'\x95\xa9J\xca\xd1\xf5\x02\x97\x89\x9f\x8c\xa6\xf5\x1b\xa4\x1a\xb7\xa10\x94\x9c\xe8M\f\xd6K\xf9\xa1\x9dE/\xc0i=X\xe1ib3u\xf3\xb4\xf7E\x90\xb3\x02\xa4h\x94\xa0\x99;b\xa3\x13Q\x19\xbe\x83ΫkA_\x8a\x05\x12\x05\xba8\x9d5\xf5\xae8\x95\xff\x12\xcaz\x06\xe5\xf7c H\xaf\x04^\x90\xf2\x86萲Z\xa1\x01\x93.\xf0\x1ba9}\x93\"\x05\x8d|\xb1\x8a\x00Vp\xb8\xd2\xd3+\xec\xe4\xcf'\xe7\xd4\xdaa\aB\to0\xab\x88\xd4~f+o\xdfg Xe\x9fk\x1c\x1c\xde\xfbϒP\xe0\xf6i\xb8\xac\xd4\xc0{|\xb9r\xfa\xe0W\x14\xb6\x84<\x1f\xf9l\\U\xf4p\xfe\x1e\xbc\x82\xd2ա\xbc8\xe4,\x85\xe6\x04E\x96@j{\x8a+\xa7\xf5A\xe9;\xf8\xe7\xbf\xc5\xff\x01\x00\x00\xff\xffߙ6&\xcb\n\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcW\xcdn\xe36\x10\xbe\xfb)\x06\xe8\xa5\x05Vr\x83\xa2E\xe1[\xeb\xec!\xd8t\x1b\xc4\xdb\xdc)j$\xb1\xa1H\x96\x1c:\x9b\xa2\x0f_\f)\xf9G\x96\x1d粺y8\x9c\x9fof\xbe\xa1\x8b\xa2X\b\xa7\x9e\xd0\ae\xcd\n\x84S\xf8\x95\xd0\xf0\xafP>\xff\x1aJe\x97ۛų2\xf5\n\xd61\x90\xed\x1f1\xd8\xe8%\xdeb\xa3\x8c\"e͢G\x12\xb5 \xb1Z\x00\bc,\t\x16\a\xfe\t \xad!o\xb5F_\xb4h\xca\xe7Xa\x15\x95\xae\xd1'\xe3\xa3\xeb\xed\x8f\xe5\xcd/\xe5\xcf\v\x00#z\\Am_\x8c\xb6\xa2\xf6\xf8O\xc4@\xa1ܢFoKe\x17\xc1\xa1dۭ\xb7ѭ`\x7f\x90\xef\x0e~s̷\x83\x99\xc7l&\x9dh\x15\xe8\xd3\xdc\xe9\xbd\x1a4\x9c\x8e^\xe8\xd3 \xd2a謧\xcf{G\x05\xf0y>R\xa6\x8dZ\xf8\x93\x9b\v\x80 \xad\xc3\x15\xa4\x8bNH\xac\x17\x00C\xf6\xc9P1$\xbe\xbdɦd\x87\xbd\xc8\x1e\x00\xacC\xf3\xdb\xc3\xdd\xd3O\x9b#1@\x8dAz\xe5(a\xf8_\xb1\x93\xc347P\x01\x04\f\xe1\x00\xd9]\x84 \f\bO\xaa\x11\x92\xa0\xf1\xb6\x87J\xc8\xe7\xe8\xc0V\x7f\xa3$\bd\xbdh\xf1\x03\x84(;\x10l%+\x1c\xf8Ҷ\x85Fi,w2\xe7\xadCOj\x04)\x7f\a\xbdv \xbd\x94\x05\x7f\x9cx\xbe\x0557\x1d\x06\xa0\x0eG\xf0\xb0\x1e\xb0\x02\xdb\x00u*\x80G\xe71\xa0\xc9m\xc8ba\x86lʉ\xe9\rz6\xc35\x8d\xba\xe6^ݢ'\xf0(mkԿ;ہ\x11c\xa7ZP\x02\xd3\x10z#4l\x85\x8e\xf8\x01\x84\xa9'\x96{\xf1\n\x1e\x13\x82\xd1\x1c\xd8K\x17\xc24\x8e?\xacGP\xa6\xb1+\xe8\x88\\X-\x97\xad\xa2q\x02\xa5\xed\xfbh\x14\xbd.\xd30\xa9*\x92\xf5aY\xe3\x16\xf52\xa8\xb6\x10^v\x8aPR\xf4\xb8\x14N\x15)\x11\x93\xa6\xb0\xec\xeb\xef\xfc0\xb3\xe1\xc8-\xbdrC\x06\xf2ʴ\a\aip\xdeQ\x1e\x1e\xa5\xdc]\xd9TNq_\x05\x161t\x8f\x1f7_`\x8c$Wjh\xb1\x9d\xea\t.c}\x18Me\x1a\xf4\xf9^jS\xb6\x89\xa6vV\x19J?\xa4Vh\bB\xaczEa\xecu.\xdd\xd4\xec:\xb1\x14T\b\xd1Ղ\xb0\x9e*\xdc\x19X\x8b\x1e\xf5Z\x04\xfcƵ⪄\x82\x8bpU\xb5\x0e\xb9w\xaa\x9c\xe1=8\x18\x99\xf3Li'\x94\xb1q(\xb9\xb0\x8c-\xdfT\x8d\x92y\xa4\x1a\xebA\xec\x19d@\xfa\x18\xa8y\x06H\xc1\t\xdf\"M\xa5\x93X\xbe$%v\xff҉c\xc2\xfa\x1e˶d\xce\tC \x99\x8f~\x98\x16\xeaR\f0\xdb賑\x8c\xfd\xcd00\xaeL(Lv\x871\x9d\xba\xe6\x0fM\xec\xe7\x1d\x14\xf0{\x8a\xf9\u07b6\x17\xcf\xd7\xd6\x10\xcf\xc5E\xa5'\xabc\x8f\x1b#\\\xe8\xec\x1b\xbaw\x84\xfd\x9f\x0e}\xde\xd0\x17U\xc7E\xbfۊ\x17\x14\xa3>\xeb\xf7\x11y\x83\xe0\xf9L\a\x85\xab\xac\\\x11ӠyU\xa2\xeb\xcd\xdd{ <\xa3\xfe\x8e\"ݙƾ\x91\xe2^qV\xef\f\r\x8c_zC\xbc\xdd\xd3\xfc\n\x19{\x9a\xaf\xe4݉\xf0)V\xe8\r\x12\x86=S\xbf(\xeaf-\x02\xbctJv\xe9b\x1a\b^\x02!X\xa9\xe6(\xf5\x8a\xf0\x99G\x94Ǚ\xa1,ҰΈ9\xf8\x13\xf1\x19\xf6;\xe7\xa0\x18\x18\xe9*\x06%A1\xbc\x83C\x93\xfe\b\xb5\x8cާ\x15\x95\xa5\xfc2\x99^\xb8\x96DG\xe6\xf9\xeb\xf1\xfe\r&\xbd\xddk\xa6ǸP&G\xe3<\x16A\xb5\xfc\x82\xe23\xe6\xd2\xc4q\xa7`\xe4\xef\xf8\x85w\f\xd4lE\xf1\xabSy\x00\xdf\b\xf1\xe3N1\x13>\x9a\xbc\xe7\xa7o\xd8d\x10\x03?\xb7@\n3\x13c\x85P\xa3F\xc2\x1a\xaa\u05fc\xb9^\x03a\x7f\x1awc}/h\x05\xbc\xff\vR3md\xa2֢Ҹ\x02\xf2\xf1\\\x97\xcd&\xee:\x11f\xc6\xf0(\xe7\a֙k\x8c\xdd0^\xec\f8\xbb_\n\xf8\x8c/3\xd2\ao%\x86\x80\xa7ct6\x93\xd9!8\x11\x06~\xa4\xd5\a(\r\x7f\x19\x06\xc9\xff\x01\x00\x00\xff\xff\xe5\x0fY\x99e\x0e\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4:K\x93\x1b\xb7\xd1\xf7\xfd\x15]\xeb\x83\xed*\ri\xf9\xfb\xe2J\xf1&\xad\xe2\x14\x13[\xd9\x12W\xba\xb8|\xc0\f\x9a\x1cxg\x00\x04\xc0\x90b\x1c\xff\xf7T\x03\x987H\xeeR\xb6\x85\xcb.\xf1h\xf4\xfb\x85ɲ\xec\x86i\xf1\x01\x8d\x15J\xae\x80i\x81\x1f\x1dJ\xfae\x17\x8f\x7f\xb5\v\xa1\x96\xfb\x977\x8fB\xf2\x15\xdc5֩\xfa\x1dZ\u0558\x02\xdf\xe0VHᄒ75:ƙc\xab\x1b\x00&\xa5r\x8c\xa6-\xfd\x04(\x94tFU\x15\x9al\x87r\xf1\xd8\xe4\x987\xa2\xe2h<\xf0\xf6\xea\xfd7\x8b\x97\xdf-\xfer\x03 Y\x8d+Њ\xefU\xd5Ԙ\xb3\xe2\xb1\xd1v\xb1\xc7\n\x8dZ\buc5\x16\x04{gT\xa3W\xd0/\x84\xb3\xf1ހ\xf3\xbd\xe2\x1f<\x98\xd7\x1e\x8c_\xa9\x84u\xffL\xad\xfe \xac\xf3;t\xd5\x18V͑\xf0\x8b\xb6Tƽ\xed/\xca@\xef\xf3\xb0\"䮩\x98\x99\x1d\xbc\x01\xb0\x85Ҹ\x02\x7fN\xb3\x02\xf9\r@$\xde\xc3ɀq\xee\xd9ɪ{#\xa4CsG\x10dw\vG[\x18\xa1\x9dg\xd7\x04s\xb0\x8e\xb9Ƃm\x8a\x12\x98\x85\xb7xX\xae\xe5\xbdQ;\x836\xa0\r\xf0\x8bU\xf2\x9e\xb9r\x05\x8b\xb0}\xa1Kf1\xae\x06\xb6o\xfcB\x9crGB\xd9:#\xe4.\x85ă\xa8\x11xc\xbc\xb8\x89\xfa\x02\xc1\x95\xc2ΰ;0K\x18\x1a\xe7\xc9N\xe3\xe2\xd7\t\xa2u\xac\xd6S\xa4\x06G\x03V\x9c9L\xe1t\xa7j]\xa1C\x0e\xf9\xd1aK\xc9V\x99\x9a\xb9\x15\b\xe9\xbe\xfb\xff\xd3\xec\x88\xfcZ\xf8\xa3o\x94\x1c\xf3\xe65\xcd\xc2`:`B\xb2ڡI2H9V}\n\"\x8e\x00\xbc\x1e\x9c\x0f\x98\x04\xb8\xc3\xf9\x8b\xa8\xacea\xb0Fy\x1dB\xa2?=\xc7f\bz\xb8\xaa\x8dPF\xb8\xe3\n^~\xf3T4\xc9>@m\xc1\x95\bQy6N\x19\xb6C\xf8A\x15A\xd1\x0e%\x9a\xa8hy\xd4\xfeR5\x15\x87\xbc\x15\f\x80u\xca$\x95Mc\xb1\b\xa7\"\xdc\x16\xecD\xe3\xc6w\xfe\x11\x06Q\x18dI\x83h\xdd\xe9\xc2\xef\x10J\xa6\xad\xe2\xd5\x0e\x9fd\x11C\x96Jű\xe3\x1f\xce\xd0\x12\x16\xb4Q\x05Z{\xc6P\t\xc6\b\x91\xb7\xfd\xc4E\x06\x95\xe8\xf7\xb4\xf84\xbaR\x8c\xa3\x01\xa7\xa0d\x92WHd0p\x86I\xbb\x8d*2\x17`{\xec\xe1\xa8Ǩ\xbc\x8f\v\xa7\xd0\t\xbb\xf6/\x83\xbb.J\xac\xd9*\xeeU\x1a\xe5\xab\xfb\xf5\x87\xffی\xa6I\x8d\x95F\xe3D\xeb\xef\xc3\x18\x84\xcd\xc1,\x8c\xc9\xfdo6Z\x03\xa0\v\xc2)\xe0\x14?\xd1z6\xc4@\x80<\xe2\x14\xd8#,\x18\xd4\x06-\x99\x96\xd7(\xb5\x05&A\xe5\xbf`\xe1\x16\x13\xd0\x1b4\x04\xa6\xb5\x85B\xc9=\x1a\a\x06\v\xb5\x93\xe2?\x1dlK\xbc\xa6K+\xe6\xd0:o\x8cF\xb2\n\xf6\xacj\xf0\x050\xc9'\x90kv\x04\x83t'4r\x00\xcf\x1f\xb0S<~T\x06AȭZA霶\xab\xe5r'\\\x9bL\x14\xaa\xae\x1b)\xdcq\xe9\xf3\x02\x917N\x19\xbb\xe4\xb8\xc7ji\xc5.c\xa6(\x85\xc3\xc25\x06\x97L\x8b\xcc\x13\"}B\xb1\xa8\xf9\x17&\xa6\x1fvt\xedL\xd0a\xf8\x1c\xe0\x19⡬\x80\x8c\x80EP\x81\xc4^\n4E\xac{\xf7\xb7\xcd\x03\xb4\x98\x04I\x05\xa1\xf4[g|i\xe5C\xdc\x14rK:O\xe7\xb6F\xd5\x1e&J\xae\x95\x90\xce\xff(*\x81ҁm\xf2Z8R\x83\x7f7h\x1d\x89n\n\xf6\xce'\\\x90\x93-\x91\a\xe0\xd3\rk\tw\xac\xc6\xea\x8eY\xfc\x93eER\xb1\x19\t\xe1I\xd2\x1a\xa6\x91\xd3́\xbd\x83\x856\t\x13\x86a\xc6;%:W\xaaB6\xe5%\x85\xbb\v4S\x00L\t\xcbG[W2\xd7\xe2F\x9bL#圷4\x94|\x968\xb4\xe2\x17\xf0\x8a720\xb8E\x83>\x1b\t\xbe_+\x1f!\x1c\x13\xb2\xf5i\xa1^\x01\xa7\x12\x98\xe5A\x89\x90\xc3\xd46\xe0\xac}\xc0\x99@\x99\xc4\xf8\xd5\xfd\xba\r\x86-\x13#\xee\xb3xw\x91?4\xb6\x02+\xee3\x87\xcbw'5\x97\xc6z\x1b\x90\xf0\x11\xc1)`\xa0\x05\x168\x8a\xc6 \xa4u\xc8x\x9c$'h0\xae\xbd\b\x9e\xfe$\x92\x10\x8a\xca\x18\xb5I&\xc0(\xf2\b\x0e\xff\xd8\xfc\xeb\xed\xf2\xef*\xd0\x01\xac\xa0\xd4\xcc\xd7z>\xdf~\xd1\xd5{\x1c\xad0ȩz\xc3Eͤآu\x8b\b\r\x8d\xfd\xe9۟\xd3\xfc\x03\xf8^\x19\xc0\x8f\x8c\xaa\xa6\x17 \x02ϻ`֪\x8d\xb0\x81\xf0\x0e\"\x1c\x84+=\xa2Z\xf1H\xe0\xc1\x93\xe0\xd8#Yr \xa1A\xa8\xc4c\xc2~¸\xf5\xd9\\\x8f\xe6\xafd=\xbf\xdd\xc2W\xc1y\xdd\xd2\xcfۀF\x97\xb6\f\r\xacG'X\x99\x11\xbb\x1d\xf6y\xffLY(\xccR\x80\xfa\x1a\x94!Z\xa5\x1a\x80\xf0\x80IN!> \x9f\xa1\xf7ӷ?\xdf\xc2Wc\x1e\x9c\xb8JH\x8e\x1f\xe1[\xf2>\x9e7Z\xf1\xaf\x17\xf0\xe0\xf5\xe0(\x1d\xfbH7\x15\xa5\xb2(A\xc9\xea\x18\x12\xe0=\x82U5\xc2\x01\xab*\v\t\"\x87\x03;\x82ڞ\xb8\xa7\x15\x11\xa9&\x03͌;\x9b$F>\x9c7\x9ay\xd6Ԏ\xa7ًϢ\x9ed\xbd\x9f-\x03y\"'|\xb9\xf0\t\x9c\x18\x96^Wp\xe2\xb1\xc9\xd1Ht\xe8\x99\xc1Ua\x89\x0f\x05jg\x97j\x8ff/\xf0\xb0<(\xf3(\xe4.#ê\xd4\xed\xd2\xf7\xc1\x96_\xf8?\xd7\x12\xee\xdbT\x9fJ\xbd\a\xf2\xf9X@\xb7\xdb\xe55\x1ch\xb3\xfb\xa7Ǯ\x93|\xd8Ąs\n\x93l\xfeP\x8a\xa2lk\xbd\x81\xb7\xad\x19\x0f\xee\x98\xc9\xe3g\xb2\x1d\xe2sc\b\xa3c\x16\x9b\xb8\x19\x93\x9c\xfe\xb7\xc2:\x9a\xbf\x86\xb1\x8d\xf8$\xe7\xf2~\xfd\xe6sZT#\xae\xf1$'j\x980>f=VY\xcdt\x16v3\xa7jQLvS\x0e\xbf\xe6$\xa4\xad@s!\xfd{7\xda\xdc&\xa8\x89j\xa0\xdb\xf3\xac\xfcӱ]\"\xe1\x1bv\xb1ϥ\x85g\xf9uY\x15\x1e\xd8\xce\x023\b\fj\xa6I#\x1e\U0005814cC3A\xe9\x02e\x04]c\x10\x98\xd6\x15\xc5\xf4\x90E$ \xc6\xfc7\xb2\x87YO\xdf)\x86$E\xd9v\xa56蜐\x9f\x919\xef'\x88\xfc\xbe\x8c\xeazv\x85\x92[\xb1\x8b\xdd\xce9\xa7dSU,\xafp\x05\xce4\xa7j\xae\xb3\x8c|\xa0-\xe7\xe9\x7f?\xd8\xdaj\xf8\x85\x06c\x9a\xaaQ\xdbqN\fʦ\x9e\xa3\x92\xc1\xa3҂%\xe6\rZ7\xb3^Z\xb8\xbd}\x8e\x8d\x05\xa5\xbc\xa6\xe4\x0eep\xaa*\x8d\x8a\x1e\x13\xf8\xb62u\xaa\xaf\xf2\x92B\x7f\x86o\xa0\xea\x9eʑ1\xdeY\xba]2\xd93\xe8.\xb7SZ\xf1\xc9\xcc\xd8\rN\x16\x03}O\xea!\xf9\x86\xf63\xbaH\xe1\x91-\xf24\x04G\xd7>\xbdQ\xda}m\x1f\x89\n;\xed\x90w\x8d\xfek$\xfej\n\xc4\xf7~\r\x8fF!j\xecJ\xff\xb1\xaf\v\xc5]\x8e\xa0\rj\x96\xec\n\x81\xef\xdc[\xdf\xc2\xfc\xd2\x06`\xc2Bc\x91\xfb\x0e\xda\xec\xee\x19\x84\xf6\x9d\x893\x87\x19\x9d\xbf\xce_\xa4\x1bS\xe1\xcdo\xf8RrU\x97j\x0ef\xceB\xd6r\xcd?ᴏ\x8d)\x8e\xf5\xe0:~\x05h\xc8}\x15JE\xf2\x96\x89\n9\xb4o\xdcτ\x92\xe3\x96R\x9c\xe0\xe3\xda>ND\xeft\xfdw^\x92\t&\xcc\x13\x9e?R\x98ӧ\xc6\v\x92\\O\xb6C\xa9\xaa(/\xd9\xd49\x1a2L\xff\xe0\t\x12\x0fT\xf7\x17%\x93\xbb\xa4\x93k\x1f\xec\x10*f\x1d\xe4\xfd'\x03)\xe2\x87/\xa6Sʆ/\x9c\xfd\xa8\xd1Z\xb6\xbb\xe4\xce\x7f\f\xbbB\xe7.\x1e\x01\x96\xabƥ\xed\xf7K\x1b]\xd0\xf3\xba\x87ɦ\xd8\xd8\xfb1W\xb6\xcen\xdbT\x95?3\x8c\x1b\xfd\xa7\x1d\x1e\xab\x1c\xd3\x19\xff\x99\xd6\xe19\x04Kf/\xb1\xea\x9e\xf6\xa4\xfcq\x17\xec\xce:d8\x13\xd8\xdf\xe2!1\xdb\xfa\xb9\xc4\xd2}t\x9e\x89\xa5ٗ\x18\xc3\xc5\xd0\x1bOq\xae]K\xc2\xec\xbesH\xac}\xef\xbdʳ\x98\x1d\xf1\xbb\xc6mv\xbd\xf5\xde\xf2\xfcg\v3\xfb\x1b\xe7\x1fL\xf2\xa1\xd8RM\x88\xfe|\xabA\x01Rl\xa4\xc5'\x01ﺜ\x02.\xac\xaeر\xa3ŗ~d\xaa\xe9\xf7\x91ޢZ\x8f\xa9\xf1T*{\xbe\xc3\xdd}-\x92\xaek\xcf\xfb\v\xb8\xe03\xfc\xba:\xed\f\x7f\x8f\x1bΤ\xe2V2mK\xe5\xd6o.\xa8Ʀ\xdb\xd8\xdac_V\xfa\xc0\xe2\x9f\xde⦨\n\tT{\xef\xf6,g1\xfex\xe8\x1a-ތ \\\x88\xfb\xf1[\xa6Ttݐ\x17 \a\xe4\x1fv\xef\xa6_p\xbc\xe8\x82\fs\xb1A\x1e\xe2Q\xaa\xab\xa0\xa4\xaf#\x94\x99\xbf\xb2\xc3\xc5@>&\xe8ό\xe1Iu\x9aMz\xcc\xf9\x00v|\xd3\x1c\xce4y\xf7ܿ\x82_\x7f\xbb\xf9_\x00\x00\x00\xff\xff\xfc¬w\xb0(\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4Z͒\x1b\xb7\x11\xbe\xefSt\xad\x0f\xb6\xab4d\xa4$\xae\x14o\xd2*Nmbo\xb6DI\x17\x97\x0f\xe0\xa09\x03s\x06\x80\x01\f\xb9\xb4\xe3wO5\x80\x19\xce\x0fH.\xa9\x925\x17i\xf1\xd3\xf8\xf0u\xa3\xbb\xd1`\x96e7L\x8b\x8fh\xacPr\x01L\v|r(\xe9/;\xdb\xfc\xc3΄\x9ao_\xdel\x84\xe4\v\xb8k\xacS\xf5;\xb4\xaa19\xbeŵ\x90\xc2\t%ojt\x8c3\xc7\x167\x00LJ\xe5\x185[\xfa\x13 W\xd2\x19UUh\xb2\x02\xe5lӬpՈ\x8a\xa3\xf1\xc2ۥ\xb7\x7f\x99\xbd\xfcn\xf6\xf7\x1b\x00\xc9j\\\x80V|\xab\xaa\xa6F\x83\xd6)\x83v\xb6\xc5\n\x8d\x9a\tuc5\xe6$\xbc0\xaa\xd1\v8t\x84\xc9q\xe1\x00\xfaQ\xf1\x8f^λ \xc7wUº\xff$\xbb\x7f\x10\xd6\xf9!\xbaj\f\xab\x128|\xaf-\x95q\x0f\x87\xb52\xd0[\x13z\x84,\x9a\x8a\x99\xe9\xcc\x1b\x00\x9b+\x8d\v\xf0\x135ˑ\xdf\x00D\x06\xbc\xa0\f\x18\xe7\x9eSV=\x1a!\x1d\x9a;\x12!\xbbe8\xda\xdc\b\xed\xd3%\xb3\x18{\x03\xf9K\xdf\x11\x9bܞ0[g\x84,R(ދ\x1a\x817\xc6+\x9d\xf6\x9f#\xb8R\xd8)\xbc\x1d\xb3\x04\xd18\xbf\xf14\x18\xdfO\"\xadc\xb5\x1e\xa3\xeaM\r\xb08s\x98\x02u\xa7j]\xa1C\x0e\xab\xbd\xc3v+kej\xe6\x16 \xa4\xfb\xeeo\xc7\xf9\x88\x84\xcd\xfcԷJ\x0e\xc9yC\xad\xd0k\x0eHH[\x05\x9a$Cʱ\xeaS\x808\x12\xf0\xa67? \tr\xfb\xedg\xa1\x90\xe9\x81Z\x83+\x11ް|\xd3hX:eX\x81\xf0\x83ʃ\nw%\x1a\xf4#Va\x04\x9dm\x10\xa4;e\x92\xaaӘ\xcf\xc2\xd8(\xac\x955\xd2\xdfp\xa1\xcfb_\xb9A\x96\xb4\xaf\xd6I\xcd\xfc\b\xa1d\xda\xc8^\x17\xf8,\x03\xeb\x13)\x15\xc7\x1ek\x13\\\u00826*GkO\x18>\t\x19 y84\x9c\xa5\xa8D?\xa6\x05\xd4\xe8J1\x8e\x06\x9c\x82\x92I^aС3L\xdau\xb4\x8c\xa9\n\xdbi\xef\xf7z\b\xe5C+\xaf\xd73\xc1\x14\x86n_\x067\x98\x97X\xb3E\x1c\xab4\xca\u05cf\xf7\x1f\xff\xba\x1c4\x03Ѣ\xd18\xd1\xfa\xd1\xf0\xf5BR\xaf\x15\x86{\xfe_6\xe8\x03\xa0\x05\xc2,\xe0\x14\x9b\xd0z.\xa2\x7fE\x1e1\x05\x8e\x84\x05\x83ڠE\x19\xa2\x1553\tj\xf5\v\xe6n6\x12\xbdDCb\xc8\xed7\x15\xa7\x90\xb6E\xe3\xc0`\xae\n)~\xebd[\"\x9c\x16\xad\x98C\xeb\xfcA4\x92U\xb0eU\x83/\x80I>\x92\\\xb3=\x18\xa45\xa1\x91=y~\x82\x1d\xe3\xf8\xd1[\x93\\\xab\x05\x94\xcei\xbb\x98\xcf\v\xe1\xda@\x9d\xab\xban\xa4p\xfb\xb9\x8f\xb9b\xd58e\xec\x9c\xe3\x16\xab\xb9\x15E\xc6L^\n\x87\xb9k\fΙ\x16\x99߈\xf4\xc1zV\xf3\xafL\f\xedv\xb0\xecD\xd1\xe1\xf3\xe1\xf5\x02\xf5P\xbc\xa5\x93\xc0\xa2\xa8\xb0Ń\x16\xa8\x89\xa8{\xf7\xcf\xe5{h\x91\x04M\x05\xa5\x1c\x86Nxi\xf5Cl\n\xb9&çyk\xa3j/\x13%\xd7JH\xe7\xff\xc8+\x81ҁmV\xb5pd\x06\xbf6h\x1d\xa9n,\xf6\xce'3\xb0\xa2\x03E~\x80\x8f\a\xdcK\xb8c5Vw\xcc⟬+Ҋ\xcdH\t\xcf\xd2V?E\x1b\x0f\x0e\xf4\xf6:\xda\xfc\xea\x88j\xc7\xfem\xa91'\xcd\x12\xb94U\xacE\x8c$ke\x80M\xc6\x0f\x99J\xbb\x00\xfa\x92\x11e<\xe8\x9c\xd9\xd1\xf7&%\xa8E,{\x8e<\xc6;\x1b\x03U5\fT\xfdo\x12#\rje\x85Sf\x7f\x88\x94c\x938\xaa\x1d\xfar&s\xac\xae\xd9ޝ\x9f\tBr\xe2\x1d;\x93&g\x14\xa4z\xa0J\x16\x8a\x0e\xd9D\x1dp\xefh\x1cٹE\x97ެ<\x1aل\x84C\x8e\t\xfd\\r\xbc\xed\x95R\x15\xb21\x9bZ\xf13\x9b~T\xd1q\x18\\\xa3A\x1f\xff\x83\x9b\xd5\xca;cDŽl\xddGH\xb9\xc1\xa9\xc4>V\xe4n\x8e\xa9\xe6\xb8\x1d\u0089\x90\x94\x04\xfc\xfa\xf1\xbe\r;\xadeE\xe8\x93\xc8\xd2\xe7'i\x16\xf4\xad\x05V\xdc\a\xea\xf3k'-\x84\xbe\xfbu\x00\xe1}\xafS\xc0@\v\xccq\x10\xf7@H\xeb\x90\xf1\xd8H\xee\xc6`\xec{\x11|\xeaQ\x90\x10\xaeE1>\x92J\x80\x91\x8f\x17\x1c\xfe\xbd\xfc\xef\xc3\xfc_*\xec\x03XN\x99\x90\xbf\xab`\x8dҽ\xe8\xee+\x1c\xad0\xc8\xe9\xf6\x81\xb3\x9aI\xb1F\xebfQ\x1a\x1a\xfbӫ\x9f\xd3\xfc\x01|\xaf\f\xe0\x13\xa3\xa4\xff\x05\x88\xc0y\x176Z\xab\x116l\xbc\x93\b;\xe1J\x0fT+\x1e7\xb8\xf3[plC'&l\xa1A\xa8\xc4\x06\xd3\xec\x03\xdc\xfa\xe4\xe9\x00\xf3wr)\x7f\xdc\xc27\xc1I\xdcҟ\xb7\x01F\x97 \xf4\xbd\xce\x01\x8e+\x99\x03gDQ\xe0!ў\x18\v\x054\n\x05߂2\xb4W\xa9z\"\xbc`\xd2Sp\xc4\xc8'\xf0~z\xf5\xf3-|3\xe4\xe0\xc8RBr|\x82Wt\xc6=7Z\xf1og\xf0\xde\xdb\xc1^:\xf6D+奲(A\xc9j\x1f\xf2\xcd-\x82U5\xc2\x0e\xab*\v\xa9\x18\x87\x1dۃZ\x1fY\xa7U\x11\x99&\x03͌;\x99\x8eE\x1eN\x1f\x9ai~\xd2~\xcf;/>_y\xd6\xe9\xfdb\xb1\xfe\x99L\xf8\xc4\xfc\x13\x98\xe8_u\xae`bӬ\xd0Ht\xe8\xc9\xe0*\xb7\xc4C\x8e\xdaٹڢ\xd9\n\xdc\xcdw\xcal\x84,22\xc6,h\xdd\xce}1g\xfe\x95\xff\xe7ڍ\xfb:˧\xee\xde\v\xf9r\x14\xd0\xeav~\r\x03m\x1e\xfd\xfc\xd8u\x94\x87e\xcc\xec\xc62\xe9\xcc\xefJ\x91\x97\xed\xad\xaa\xe7mkƃ;fr\xff\x85\xce\x0e\xf1\xdc\x18B\xb4\xcfb)2c\x92\xd3\xff\xad\xb0\x8eگ!\xb6\x11\x9f\xe4\\>ܿ\xfd\x92'\xaa\x11\xd7x\x92#\xb7\x85\xf0=e\aTY\xcdt\x16F3\xa7j\x91\x8fFS\xae|\xcfIIk\x81\xe6L\xf6\xf7n0\xb8\xcd\xda\x13Yw7梴\xdbJ\xa6m\xa9\xdc\xfd\xdb38\x96\xdd\xc0\x16\xc3A\x871\xe9leё8\x99k>\x03\xcfR\xfc\x96p[ID4\xb4\xc5T\xa9B\xe4\xac\x02\xeb\xdbd,VF\x98\xad\xec)\xa0T=r\f\xb7_U\xec\xe1\xf5\xbe\xe0ḧ\x1dB\x1e\x8enQ+#\n!Yu\xf0\xd8\xfe\xea(Y\xcd\xfc_\t[\xad\x99\xd6B\x16\x17q\xdbַ\x96蜐E\"\xd1\xef\x97\xdfO]\aN\x9e\x93\xf3.\xe0\xc3\b\b0\x83\xc0hO\xa4\xaa\r\uecd0uj&(e\xa4\xac0\xa6\xd6+\x04\xa6uEy]\xc8$S\xbe\xa9\xad\xd6\xe5J\xaeE\x11+\xa7S\xa6dSUlU\xe1\x02\x9ci\x8e]ڒǽ_(<\xa3\xf1\x0f\xbd\xa1\xad\xbaϔ*ӻ\x1a\x140\xa7\x9bA\xd9\xd4S(\x19l\x94\x16,\xd1N\x87s☨\xe3\xf6\xf6\x12\x93\n'\xff\f\a\xe1Μ*8D\xc7\x11\xaf!\xf1\x8a\x1d\xdcG:\x9a_\xeaP\f\xfe\xdaНj\x880K\xd7VFc\xb4\xe27c\xd2\xfa\xbex\xd4y\xf0\xa4\xe3\x8e\xe1\xa1\x1f\xf5\x06\n\x9eU\x96\xf2\x85\xf2K\nS\xe19,\xf2\x1e\xd2\x00\xd7>\x92\xd1\x05\xe3\xea\xd2\x14\xdda\xb5C\u07bd!\\S\xb7y=\x16\xe2\vʆ\xc7C\"j\xec\x8a\x1c\xd1N̡\xec\x12B\x8c6\xa8Y\xd2\"\xc0?\nX_\x18\xfd\xda\x06i\xc2Bc\x91{\xdf:Y\xfchL\xe0\xccaF\xf3\xafs \xe9bWx\x9e\xeb\xbf\xc2\\U\xf9\x9a\x8a\x99r\xc8:\xda\xfc\xfbP\xfb0\x98\xa2\xec \xaf#,\x88C\xee\xafܠ$\xac\x99\xa8\x90C\xf7,}1\xf3\t\xd0\xd3d\xecs\x92_\xa3\xb5\xac8\xe7\xb4~\f\xa3B\xe5-N\x01\xb6R\x8d;b\x95_\xdbx\xb4.\x8a\xc9R\xf1sH\x1e\x14\xf70\xe4\xf1'\xb7)\x9a\x84Z\xfa\xcfp\x17a\xf4E\xcdsEJ\x1a\x93r5\x1d\xe4Ӿ\x06Nİ\a\xdc%Z\xdb\x13\x9c\xe8z\x8cn!\xd15\xf9=@\xbf3T\x92S9Mۗ\x94\xd9=\xb6'\xfa\xbe\xf7\xc7\xe5\"\xb6#\xbek\x1cBW\x87.U\xd5\xfa\x00\xffH.\x9bz\x85\x86T\xb1Je\xc4\xc0$\xefk.UL\xe8$\xb4a8\x88\x8a\xf5\xb0X@\xf7\xa7\xdc)\xe0\xc2\xea\x8a\xed\xbb\xcd\xf8\x1b\x1c\x1d\xe9\xf4s\xc2\xe1\\\xb5\xbe\x8a\"ϑ\xbc\xedt\xa5\xba\xfb\xd1B\xfa~z:Ӈ3پ\xef\xef~\x8c\xf0yV8\x91w\x0e\x7f\x1cr\x8d\x81,\a\x12\xce\x05\x8b\xf8c\x95\xcb}\xfcp\x99?ӽ'ٛ4z\xe4\xbc';>y\xf5[\x9aU\xf7\x1e\xbc\x80\xdf\xff\xb8\xf9\x7f\x00\x00\x00\xff\xffϡa\xa5-&\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec=]\x93\xdc(\x92\xef\xfe\x15\x84\xefa\xee\"\xba\xca7q\x1fq\xd1o\u07b6}\xee\u06ddv\x87\xdbg?SRV\x89i\x04\x1a@]\xae\xdd\xdb\xff~A\x02\x12R!\x89\xaa\xfe\x98\x99\x8d\xd1KG\xab \x81\xfc\xce$A\xab\xd5\xea\x15m\xd8WP\x9aIqIh\xc3\xe0\xbb\x01a\xff\xd3\xeb\xfb\xff\xd2k&\xdf<\xfc\xf8꞉\xf2\x92\\\xb5\xda\xc8\xfa3h٪\x02\xde\xc1\x96\tf\x98\x14\xafj0\xb4\xa4\x86^\xbe\"\x84\n!\r\xb5\xaf\xb5\xfd\x97\x90B\n\xa3$\xe7\xa0V;\x10\xeb\xfbv\x03\x9b\x96\xf1\x12\x14\x02\x0fC?\xfc\xeb\xfa\xc7\xff\\\xff\xc7+B\x04\xad\xe1\x92(\xd0F*\xd0\xeb\a\xe0\xa0\xe4\x9a\xc9W\xba\x81\xc2\xc2\xdc)\xd96\x97\xa4\xff\xc1\xf5\xf1㹹~v\xdd\xf1\rg\xda\xfc9~\xfb\x17\xa6\r\xfe\xd2\xf0VQ\xde\x0f\x86/u%\x95\xb9\xe9\x01\xae\x88\xf2\xcd5\x13\xbb\x96S\xd5uxE\x88.d\x03\x97\x04\xdb7\xb4\x80\xf2\x15!~Q\xd8\x7fEhY\"\x9a(\xbfUL\x18PW\x92\xb7\xb5蠗\xa0\v\xc5\x1a\x83h\xf8R\x01.\x86\xc8-1\x15\x90\r-\xeeۆ\x98\x8a\xe90(a\x9al\x95\xac\xb1;!?k)n\xa9\xa9.\xc9\xda\"h\xedz\xd8\xf9\xf8\x06\x0e\x9f\x7f\xc2\xd7\xfe\x959\xd89k\xa3\x98إf\xe1\xf1D\xb4\xa1\xa6\xd5D\xb7EE\xa8&7\xb0\x7fs-n\x95\xdc)\xd0:1>6_7\x15\xd5\xc3\xc1\xef\xf0\x87\xcc\xc1\xbfHC9\x11m\xbd\x01e\xd1\x00JI\xa5\t\x97\xbb\x1d\x94\xa4lm?č\x8ah\x9c\x9a\x87\xeb8\x98\xc8\xfb\xf8\x95\x9b\x88%\xc9\x0eT\xceL\xf6T\t&v\xe7\xcc%t\x1d\xcc\xe6\xdb\xf0ej>\x11\xa4 e\xebB\x01\n\xd8\x17V\x836\xb4n\x06@\xdf\xee`\x00\xaf\xa4ƽp??\xfc\xe8X\xb9\xa8\xa0\xa6\x97\xbe\xa5l@\xbc\xbd\xbd\xfe\xfaow\x83\xd7d\x88\x8e\xff[u\xefI\xc7\"L\x13J\xbe\xa2(Z$\xa0j \xa6\xa2\x86(h\x14h\x10F#\x86h\xd3pV\xe0ĉ\xdcF\x90B/\xc7\xd5=\xb4\xc0\xfa\x92Pb\xa8ځ!\x7fn7\xa0\x04\x18Ф\xe0\xad6\xa0\xd6\x1d\xa0F\xc9\x06\x94aAl\xdd\x13i\xb7\xe8\xed\xdc\xc2\xeccq\xe1z\x91Ҫ9pK\xf0r\r\xa5G\x9f\x13R\x94L\xbf\u0530\xd9)H\xab\x97\xdd\xe0\xa6n\xa0\xa2\x0fL\xaa\x94\x18H\x85M#{ޫii\xb5\xa4\a2\xb6q\x99\vN\"\xab\x92\xf2~\x89!>\xda6\xbdu \x05\x86<\xddR<\xb5\xbd\xed\xde\x00\x81\xefP\xb4&1M\x12\x9cC\xa9H#\xb5\x99\xa6\xfb\xb4\xea\"\xb1s\x94\xfaq\x86i\x8eV\x96du\xf7x%\x1c\x88jq0P\xc8R\x80]Fm\x89ڷU\xb2um'\x91B6TCI\xa4\x98\x1c\x19٥\xe5\xa0\xfdX%rF\xaf\x87.\xfa\xf5\xa3\xc7C8\xdd\x00'\x1a8\x14F\xaacd\xe6\xa0\xd4=9\x8au\x02\x95\tm:\x94\x80~\x013 \x89\xe5\xf4}Ŋ\xcay\x18\x96=\x11\x0e)%h\xabM\xd0e>L-\x92,\x91\xdf\x0f2\xa7=\xfagA\xac\xc6\xf0R\x1a\xa5\x7f2\xd4p\xff$Q\xdb\xeb\xde#\xdd\xe2\xdf\x1b9\xbb\xec\x7fL\xc4\x06cr\x06\xd3\xce\xc8?A\xf73\x9b\xa7'\xf9\x16#<\xd0kr\xbd%P7\xe6pA\x98\to\x97$\x81r\x1e\x8d\xf1;\xa6\xcd\xe9L\x9fI\x9a\x1c\x99x&\xc2tC\xfc\x0e\xe9\x82&\xe3\xce[\x8cl\x9a\xfc%\xeeuAضCzyA\xb6\x8c\x1bP#쟥\xea\x03e\x9e\x02\x199V\x8f`\x9e\xc0\x14\xd5\xfb\xef\xd6\xc5\xd1}\x9a6\x13/\xe3\xce\xce7\x0e\x11\xc4\xd0y\xd6;~\xbe\xaf\xee\xbb|\xc1ʚ\x9c\x95\x87`d\x9d\x81\x03\xaf\xbb\xcb\xe5\xf5\xac\xac\xccf\xb4\n\x9c\xb0\xd8t\"9:\xdd4\a)\x8f@\aZqtq\x16\xa9\x1b\xef^\xe6[\x94\x13x\xe1T\xd5\x10\xcdݙ\xe0\x9a6V-\xfc\xcdZZ\x94\xa6\xbf\x93\x862\xa5\xd7\xe4-\xee\xd8r\x18\xfc\xe6\xf3p\x11\x98\x8c!\x1b;\x94\xe5\x9f\aʭ\xed\xb7\n\\\x10\xe0\xce\x13\x90\xdb#\xbf\xe8\x82\xec+\xa9\x9d\xd9\xde2\xe0\xb8_\xf1\xfa\x1e\x0e\xaf/\xec\xf0\x8bC\xc6J\xe6\xf5\xb5x\xed|\x88#\x85\xd19\x1cR\xf0\x03y\x8d\xbf\xbd~\x8c+\x95ɩ\x99\xcd\x06,Z\xd3&\x8fCE2Y\xdf?\x03\x8e\x89s\xf3}R\xde;\xd9s\xab\xcdb\xd1Fj\xf31\x9d7\x9c\x98\xcfm\xe81\xf4\x8c\x139\xb6ň\xc1\xe7\xd1:}o\x9dȭ\x01\xe5s\x89\xce\x06\x84\xf8㑑YjW&\x9el\x97\f\xa4]~\xd7\"x\x81\x9b\xdc\xc6M\xce\x14OqX-^N\xf4\xf6\xdf\x7f\x8f\xf2\x99Vr\xed\xff\xf1B\x9eڡ.d]\xd3\xf1\xaef\xd6T\xaf\\\xcf\xc0\xd3\x1e\x90\xa3\xbeڵ(Ϲ\x16\xb9\xe7!ܿ\xdc3S1AhP\x1b\xa0\x83\xebѡ\xeb9\x9dݫ\x8e&\x1d\xe5\xbb\x17\xced5\xb2$\xfb\n\x14\f\x18\xe38\uf39e\xaa\x90&JY\x9c\xe0\x906\xb2\xfcA\x93-S\xda\xc4SФչ\xb4>\x91|v\xde_X\r\xb25ω\xe0\xf7\xfd0\x83\xbd\xe6\x9a~gu[\x13Z\xcb\xd6\x19s\xc3\xeanWףwO\x99鶭0\x7fc\xa4%A\xc3\xc1\x00\xd9\xc06\xbdߛz\n)4+\xa1+\x1drdc\xd2\n\xe6\x962ަv\x89Rϩ\x11\xb0\xc0\xea\xa73P\xfc\xc9\xf5\x8c\xf2\x8e\x95\xdc\x0f\x11\x94\xb9v\xdcH\x03¶\x84\x19\x02\xa2\xb0\x18\a\xe5T2\x0eᑁ\xa8a\xb9z.O\x81\xdb\aD[\xe7!`\x85\x02\xc9\xc4l\xca-n\xfe\x812\xfe\x1cd\xb3\x9c\xf7A\xaa\xcf@\xcbsr4ߢ\xee\x04\x84n\x15n\xfe;ݱg}\xeedq=\nb\xa8&{\xe0\x9c\xd0\x14w\x1c-\xbfp'\x93\v\xb9\xc2#\x81\x96\xbc\x81I\xfcy\xe6\v'\xc5x\f\f\xa9W'\xe0\x16T\xe0\xe9f\x9dXȤ9\xccѢG~\xb9\x8b.\xf0\xdd/-\xa8\x03\x91\x0fX\xc2ཷ\xfe\xac\x82W7\xdaƘA\x01ze<\xb5\xa9p\x14\xca\xf4\n\x8a\xbc\x15Η\x18\xcf\a\xfbX\xcdׇjV\x9d\xdb(,9\xc6Dw!\xbbމnKn\x7fnQ\xff\xf3\x06n\xa7\x87n\x8b\xbeR\xbe?\xfb+\x15\xeb\x9fS\xa4\x9f\xb7\x1d\xb4X\x94\xff\\\x81\xdcR(\x97\xed\xbd\xe6\x15ݟ\xb6\x89\xfa\x8cE\xf6\xcfQ\\\x9f\x89\xa9\x9cb\xfa\xd3\xf0\xf4\x02\xc5\xf3/Z4\xffR\xc5\xf2\xd9E\xf2Y\xfb\x98ٛV\xb9یgV}/\xef\xba\xcf\x17\xbdg\x14\xbbg\xec\xa4-/\xf2\x8c\xe5e\x14\xb3\x9fVĞA\xb3\\Q|\xc1b\xf5\x17,R\x7f\xe9\xe2\xf4\x05\xceZ\xf8\xf9\xb4\"\xf4\xb3w`\xc2V\xff\x8d,\xe1V*\xb3\x14\x9c\u070e\xdb'vR\xa3\x80M\xf2\x92\x88\xd04\xb1J\f1|xqޢқ\x9e\xc1\x9d\xfeI\x96vnK{,\x9fG͏\xce*oA\x81p\xd7|\xfc\xcfݧ\x9b\x0e~\xca\xe7\xf5\x9e\xf1\xe8z\t\xe7\xc1\x94\x1e9~k\xce\x1739l\xa1\x0f\xf0\xc4\xfb\"\xb4a\xff\x8d\xf7\x0e>\"\x1d\xf4\xf6\xf6\x1aa\x04?\r/2\xec\xaa(\xba\x1d\xcb\rX\x8bաjR,\xae\xb7\x03\x88Ê\xdf\xf8\x1a%(ݕY\xc1b\xb2P\xe3e\x05\xef\xf6\xda\xcdcj\x94\x0f\xd6i\x14\a\"\x1dGVL\x95\xab\x86*s@\xb6\xd1\x17\x839\x0433\x97ΙT\xac\xc7׀%\xd1\x1bn\xff½\xc8C3\xdc\xed\x1d\xe3\xee\x9cyL\x9f?YfX6Jk|sq\xfc`\xbc\xe4j\x11\x19\xbf\x88\xb2\xb7/S\xa6\x93y\xc5\xd6ٗk9\xf4L\xa8\x1fܑ\xb0\xaa\xed\x18Sg\x14\xe8,\x86\xdb\x19\a?\xe6\x13\v\x99W3\xe5\x19\x8c3\xaecB|\xe5\xe2\x8a$oiʼ\x89\xe9WE\xf4\x8cV\xd3E\x05e\xcb\xe1\xdc{X\xef\xa2\xfe\xcb7\xb1\x86\xd12\xeeb\xb5Ȏ\f\xb4\xf5\xb0\x86w\xbezJx\xc81%\xa7\x82pLظ+\x1f\vw;pQ\x80\xd6ۖ\x87\xcaQ\xbc\xc0\x1b\xcaМ\xe9n\xc6'\xd5>\xea{ּs5\x94\xe3\xb0\xfb,\x1cO\x83\v\x97\xf8G\xd6\x01\xf7\x14\xd4\x03\xa8U\x81\x1ea\xab\xa0\f\x15\x9d3^$\xa9\x03H\xa6\xe3H~\xe0\xaa'\xfa\x7f\xab@ W:'*\x94\x8e\xc6\xd0,:\x1a(\t<\x80 lK\xa2yI\x11M8\x05\xfe#\xc5\r8\xd8n\xa10nC\x97\xa2\xc3\x1c\xa4\xf6\b#\xac\x97\xfb\x84g\xf5\b\x83\xd76\\\xd2\x12\x94s\xb4\x17\b\xf9\xbf\x83\xc6#M\x14\x10\xd0_\xa2<{\x01\xed\xa3\xecQC\x15\xe5\x1c\xf8\a\xc6A\xbf\x93{a畡foS\xfd\xa2\x13\xd0E\xab\xac\xb3v\b\xb7\xf0k0f:-\xbb\x95j\xfe,\xd2\xf1\x15\xfb\xc3g\xaf\x98\x81\xbb\x86*\r8\xa3\x8c\x15|\x1buqy\xde-\xa7;Wt^\xb2\x82\x1a\xe8\x04\aG\x98\x9a>\xf6\xd7\b\x8b\x1f\xb0\x06XNl/e\xab\xea\xa9Ï\x93\xcaz\xea\"\xef\x84\x03\x96\xbc\xca\xdb\xf9Y\x05m\f\x1e5E:\"\x11M\xf8\x9c\x84\xdc\x1e\xdd\xe6=\x00;\xcdi\xfe\xc0P\xfc\xed\x83s4\xdd\xd51\x18\xbc\x80_\x95Q\x85{|\x95qW\xcaN\xf6Twǖ\x92\x11U\x0fہA\xb5fA\a\xcddE\x912\x0e\xe5\x1c\xa7~\xe9\xb4\xd5\x0f\xba\x83\x835\xf7\x96\xc5\xef\fU\xa6\x9b\xfa\xb1w\xea\"s\xf7釕\xed}\x9e~J_H\x8e_\xd08\xeb^m\xf7\x1d\x0f\x14\x8f\"\x1cp\xb5>\x8d;\xf9]\x83\xd6t\x17ҽ{P@v ,\u07bb]\xbc\xa4\x1f\x1c\x0e\xcf{\x17`\x90\ue845i)\x0f_\x10\xa1\xf8E\x13_\xa7\x14\xbe\x04\x80\xf9\xe2ݤ\xe1M\xab\n\x7fL\xff3P=\xfe\xb0\xc4\x11.>\xc4m\xfdv\xac[\xb1\xabB\xa0\xee(\x05~Z\xc005\xfe\x94\xc8`J\x12G>\xc9I\xa8\xa4\xbc\xcf\n\x9e>v\r\xfb\x8d\x1b&\x1c+\xe1\xe5\x04\x1bٚ\xc8{\xf5\bOL\x13/\xda~b\xfb\x820ߺ\xa3\xcaS\xbb\x98y\xfe\xfb\xc7\x01\xa4.i1\xfa\xd4\v\xed\x1aT3W\xf5܅\xcf\x14p~\xb8\x18C\x1e}\xff\xa4\x87]\xf5\x97f{M\xd0_\xd421P\xd8_K\x02\xe9\xee\xdb\xee=ͩۍ\x97\xec\x1fB\xfd\x80\x93\xca\xc0\xf1Ǿ\xf5\x14\x1e\xdd4]\x18\x04\"\x9d? \x18R\x9a\xaa\x93\x8c3\xa6>\x13{\xe0爖6\xe3l\x9b\xce\xed\x88\xccU\x17Z|\x9e\x90\xca\xf4\x8d\x12+r\x03\xfb\xc4[\x87,\xac3A\xa9J49\xfa\xc0R\xfc\xe37ʬ\xfb\xf3A\xaa[\xde\xee\x98\xf84}\xc6j\xae\xf1-U\x86Y\xa6u\xf3I\xf4\xbd\n6.\xf1\xdbr\xef\xe9\x1f\x98\xa0\x9c\xfd5\xa5\xcb\xe3\x1f\x97F\x98\xd1w\x8dG\xde9\x16* ~I\x01z\r\xfd\x83\x8e\xccO\x18wMndR\x8c})\x16\x1b\x02e\x9al@\x9b\x15l\xb7R\x19\xb7S\xbeZ\xd9\xf0\xc5;HVC`\xf4\xef\xbe\x1bCX*\xba\xea\x8a\\\x82ò\xf5\tb\x85V\a\x13\t5=\xb8<3-\n\x1b\x13\xc0\x1bmh*\xe2|\x94\x9e\xc6\x04\x84\x97\x95\x1c\x15r\x1d\xb7\xef2\xb7\x9d\xfa@p\x0euxq\x8c3\xe8|\xaazdp/\x15\xd1\x16{\xe7(\x13\xe2\xd4\xd8\xf5t2%\xcf\xd4|\xe9\xa0L\xa9G\xbf\xbe\xc1'/|)\x93od\xc9VTT\xec&\x8f\x8eWJ\xb6\xbb*\xf0\xe6\x94CD\xca\x16\xf3!\r\xaa\x02\x1d>\xd1eZ%\xa2\xf2\x18_\xcd8\xa5\xa5\xbb\xe9N\xfb(\x8fPԪ?Bګ\xaa\x19\x9b\x9f\x9d\xfb\x9d\x80\xb8h\xfb\x13\x10\xa9>\x88b\xf6\xb0\xeb\xf1\xce\xe3I\xaee\x12\t\x9d6~2$t\x10\xa7\x90\x10\xfb\x12}\xc4\xf3\x9b\xc1Ȕ\x8fr&:\xe6\x9d\x18\\\xe2<\xa8\xe5E\xc7N\xd0\xd0\xdd9\r\x1dz\x10\xfc\x9d\x95\xe8\x1b@8%\xf2ű\xd3q\xefo7b}輭\xf7gǮ_G0F\x97\r\xd8(\xb6\x1f&ě\xff̶)yq\xdfA\xdcp\xf8\x97\xa3__\xf8Ҁ\xf0Q\xcas0\x12\xbe]\x99\x88\xe7=\xd8\xe7\x8c\xe8\xbb/q>UL\x9f4KG/\x91\xc1\xcb\b\xcf~\xa4\xf8M\xbb\xe9\xbf\xd9D\xfe\xf6\xf7W\xff\x1f\x00\x00\xff\xff\x95Pn\x17dw\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec=\xdbr\x1c+\x92\xef\xfa\nB\xfb0\x97P\xb7DZ\x97\xd8ЛG\xb6g;\xc6c+,\x1d=\x0f]\x95\xad\xe6\xa8\n\xea\x00%\xb9wv\xfe}#\x13\xa8[\x17]T\xab%ϙ1/\xb6\xaa!I2\x93\xbc@\x02\x8b\xc5\xe2\x8cW\xe2\x0e\xb4\x11J^2^\t\xf8fA\xe2_f\xf9\xf0\xdff)ԛǷg\x0fB\xe6\x97\xec\xaa6V\x95_\xc1\xa8Zg\xf0\x1e6B\n+\x94<+\xc1\xf2\x9c[~y\xc6\x18\x97RY\x8e\x9f\r\xfe\xc9X\xa6\xa4ժ(@/\xeeA.\x1f\xea5\xackQ\xe4\xa0\tx\xe8\xfa\xf1\x0f˷\xff\xb5\xfc\xcf3\xc6$/ᒙl\vy]\x80Y>B\x01Z-\x85:3\x15d\b\xf4^\xab\xba\xbad\xed\x0f\xae\x91\xef\xd0!{\xe3\xdbӧB\x18\xfb\xe7\xde\xe7O\xc2X\xfa\xa9*j͋N\x7f\xf4\xd5l\x95\xb6\x9f[\x98\v\xf7\xbb\xfbM\xc8\xfb\xba\xe0\xbamsƘ\xc9T\x05\x97\x8c\x9aT<\xa3\xaa~l\x04b\xc1x\x9e\x13\xb5xq\xad\x85\xb4\xa0\xafTQ\x97\xb2\xe9 \a\x93iQY\xa2ƍ\xe5\xb66Lm\x98\xddB\xb7\x1f,?\x1b%\xaf\xb9\xdd^\xb2\xa5\xa1z\xcbj\xcbM\xf8Ց\xcf\x01\xf0\x9f\xec\x0eq3V\vy?\xd6\xdb;v\xa5\x95d\xf0\xad\xd2`\x10e\x96\x13s\xe5={ڂdV1]KB\xe5\x8f<{\xa8\xab\x11D*Ȗ\x03<=&\xfd\x8fS\xb8\xdcn\x81\x15\xdcXfE\t\x8c\xfb\x0e\xd9\x137\x84\xc3Fif\xb7\xc2L\xd3\x04\x81\xf4\xb0u\xe8|\x1a~v\b\xe5܂G\xa7\x03*\b\xf62\xd3@2}+J0\x96\x97}\x98\xef\xee!\x01\x18\x91\xa8\xe2\xb5\xf1r\x14Z_w?9\x00k\xa5\n\xe0\xf2\xac\xad\xf4\xf8\xd6\xc9^\xb6\x85\x92_\xfaʪ\x02\xf9\xeezu\xf7\xef7\xbdϬO\xd1\xff[4\xdfY\xc3\r&\f\xe3\xec\x8ef\x10\xd3~J3\xbb\xe5\x96i@1\x00i\xb1F\xa5a\x11H\x9d3\xa5;\xa0*\xd0B\xe5\"\v,\xa2\xc6f\xab\xea\"gk@n-\x9bڕV\x15h+\xc2|r\xa5\xa3z:_\x0f\xa1\x8f\x05G\xecZ91\x05C\x92\xe9g\x1b\xe4\x9eHn\xf2\bӎ\x878\x88\x9f\xb9dj\xfd3dv9\x00}\x03\x1a\xc1\x84QdJ>\x82F\x8ad\xea^\x8a\xffm`\x1b\x9c\x12\x96$Ղ\xb1\x8c\xe6\xb3\xe4\x05{\xe4E\r\x17\x8c\xcb|\x00\xb9\xe4;\xa6\x01\xfbd\xb5\xec\xc0\xa3\x06f\x88\xc7_\x94\x06&\xe4F]\xb2\xad\xb5\x95\xb9|\xf3\xe6^ؠ\x903U\x96\xb5\x14v\xf7\x86t\xabX\xd7Vi\xf3&\x87G(\xde\x18q\xbf\xe0:\xdb\n\v\x99\xad5\xbc\xe1\x95X\xd0@$)\xe5e\x99\xff[\xe0\xb7\xe9u\xbb73]!u:\x83=\xa8g\x9dt9Pn\x88-\x17\xf0\x13\x92\xee뇛ۮ\xe4\t\xe3\x99\xd2\x11\xc0\x18\x7f\x90\x9aBn\xc0낍V%\xc1\x04\x99WJHK\x7fd\x85\x00i\x99\xa9ץ\xb0(\x06\xbf\xd4`,\xb2n\b\xf6\x8a\x8c\x16\nm]\xe1\xdc͇\x15V\x92]\xf1\x12\x8a+n\xe0\x95y\x85\\1\vdB\x12\xb7\xba\xa6xXّ\xb7\xf3C\xb0\xa7\x11\xd6\x06]qSA֛j\xd8NlD\xe6&\x14\xaa\xe4F\x95\fԲ+㳟~!\xdd7\xfc:\xc0\xc3)\xc8\xd0+\x184JvK\xccom#\x8a\x9c\x83ƔfR\xd9=\x98\xfb\xaa\xb5C\t\x0fe\x02\x93=ag{*5Œ\x8e\x00im\xebP\xbe\xa2\xac&\xbc\x1fD\xb5*K\xc8\x05\xb7P\xec\x8eB\xbf\x0fb\x8c̊\xfaak\xa7\xe7ŦG\xf4\xbc\x06&:\xedi2\xfe5\xd4ط\xc6\x7f%\xcbNF\xd4Ѥ\v\xac\x96-\x0f\a\xfdHx\xda'\rc\xab\r\xb3\x1au\xae\xc7\xeeI\x14\x05\xcedĸ\x82\xbc\x87Z\xbc;\xb1a\u0086Ѭ9a \xd9\xd2yQ\xcb\xd6gh\xec?\"8\xc0\x8eԾ\xeb\x1f=\x15n\x99\x84o\xb6\xad\x85Î\x8c`\xc3\v3\x18\x82WH\xb3\x86q\xc1ֵ=\x0e\x03(+\xbb\xbbpm7\xaa(\xd4\x133\xa4l\xd1\bn\xc4}\xad\xddd\xffm\x0e\x1b^\x17\xf6\xd2\xe1\xfc\xbb\x98\xb4\x8eO3\ve\x85&\xf3\x189\xbd\xf5mq\xc08[\xf2&\xfe\bnr\xf0C\x94w?F\x80(\xe7\xc5VZ=\x8aܛ\xf3=u\xc5\x0e\xaa,,\x8eQ\xb78ґ_\a\xa3\xf9cS\xb93\xb7\xb6\xea\x89=b\x00\x00\xe8-r\x1c\x14\x02\x85\x9c\xd5\xd5\xc5(Lƞ\x84ݲJ\x19#\xd6\x05xρ\t\x99\x15u\x8e\n\xe6c]\x14$\x82+\x99i(\xd1\xe1)Ƹ\xcd\x18Ⱥ\x1cG|AP\"?u\xe0\x8e\xd68\xa0\xa7\xb0dF\xdcH^\x99\xad\xb28\x8dTm\x13\x887*\nX\xaenV\x03h\x1d\xea\"\x8f)~\xa0\xc9m\x15{\xe2\u0092\xa1\xba\xbaY\xb1;\xa2{h͜\x86b\xb6\xd6\x12\x9d\x83H\x7f_\x81\xe7\xbb[\xf5\x93\x01\x96\xd7\xe4ׄ\x98\xe0\x82\xada\x83N\x80\x06\x84\x81?\x81\xd6h\x14\r!\xa1\xea=?3\x94['\xc78\xa1\xbc\x9b$\f{\xfb\aV\nY\xdbѩ:Ie\x94\xa6R=\x82~\x0eq\xdfs\xcb\xff\x82@\x064%Q%\xe8~\x96\x11}\xd7;\xfaq\x1d1_\xae\xac6\x1d\xa8°\xf3sT\xa1\xe7.\x86?\xbf\bS\x9aH\xb1\xaeEa\x17Bv\xfb\vz\x1d{<\x8e0\x0e\xb8c\xbe\xb9U\x1f\x8d\x9b\x98ϢS\x04\xe6\x88\x11\xadT\x1e&\xfcF\x14\xc0\xcc\xceX(\x83\xcaoæN,8,\xe4h\x15\x85\ac\x90\xee~P\xe3\x04\x91uQ\xf0u\x01\x97d!\x0f\xd0l\\Y\x8f\x11\xed+\x18+\xb2S\x92\xccA\x1c!\x98\xf6?\xf4(Cq\x17\x7f\x00\xc6#\xe0==1\xc8+\x8a\x0e\xd1\xfbԊ\xe2Vi\xc80\x00\xb8\U00101140\x82\x82\x19\xa9X\xa1\xe4=h\x87Ec\xe8\xd1\xd0\x00\nh\xce\xd0g\xd7h\x9e\x85d\x9b\x1a\xdd\xf9%Cm\x11\x95\x11!\x8d\x05\x1e\x11\xe6\x13\xf0\x0e\xbe\xa1e\x80\xfc\xaa\xa8\x8d\x05}\x93\xa9\n\xf2\xb0z7j\xd6Ry\xf8\xe1 d\x1f\xfc\x15\"\x03\xe4C\xe6*-h\x85,&\xdam\x1c\x88f\x92\x16\xf3\x90\xd5~\bm\x807\xa9c\fXlx\xfe\xfb\xf3\v\x92\x80~\xef\xfd~\f\xe3\x1a\x1a2\xcd\xd2\xd1\xe4.\x8d\xb7\x10\x16\xca\bu'u\xd4\f\xbes\xad\xf9\xee\x00כ\x95\xc8\x17\xe0{\f\xf6\x80\xf32T\xfbN\xbc\x1f\xf6\xff\xaf\xc8\xfd\xd3\xf2\xdb\xd0j>\x17\x12\xf9\\\bc{l6n\t\x10\xc9:\x16\x7f{\x02I\a\x13\xd5\xe4\x14W\xffA\x88yҹ\x13\x9b,\x8dl\xfa\t\xf0OEɭR\x0f)\xd4\xfb\x1f\xac\u05ee\xff\xb1\x8cv\x9c\xd8\x1a\xb6\xfcQ(m\x86k\xcc\xf0\r\xb2\xdaF5\v\xb7,\x17\x9b\rh\x84E{$͖\xca!b\x1d\x8e\xfdXGeE+\f\xc6\xd52\x1dYJԈ\r\x85\xa2\xfb(T\xe7\xe0`\x88A\x0eD.\x1eE^\xf3\x82|\t.37>\xde\xe0\x17\xd3j\x13\x02\xb1\x87\x7fT\xaa]q\x0eM\x18$2\xb1\xb7d\xa8$\xa0\xaf_b\x8c\xb4_5N\x89\xb0\x0es\xb0od\xa6\xae\v0\xbe\xbb\x9c\xdc\xe4V']\xb4\xccr\v4\x05_C\xc1\f\x14\x90Y\xa5\xe3\x14J\x91\x03WR\x95n\x84\xb8#Z\xb6\x1fu\xb5\x83\x99\x00\xcb(\xd4݊l\xeb\xdcW\x144\x82\xc5r\x05\x86\x96\x94xU\x15\x11\xd3ՖI\xe1\xf0\x9dM鍶$h\x90!ܘ.iK\xa2~n\xcb(\xd9۹٧\xfa\xf8&\xc9(\xbe\xffJD\x0fV\xe7Ha\x9f\xd0$\x8c6[\x92\xe7C\x94\xf4Hq\x01f\xd9Y\xda\x146|Mah\xcf\x7f\xdcۇ\xda#ʯ\x8bw\xc7M\x98\x19\xac\x9b\x9cS/˸\xa6\x9b\x7f\x12\xbe\x91ɺ\xf1\x16k\x16\xcf>u[^Ж\x8agH~\xc16\xa2\xb0@N\xd5\x14\xa2l\x06\xe7NI\xa0T\v\xcch\x87\xddf\xdb\x0f\xcd\xc6[B\x8b\x01\xad\x86\x00\x9c\x83\x1e\xa2\x1c\xe2A\x02Hָ\x16\xb4\xe3,܊\xb9YR$\xd9\xfdB\xae\xe0\xbb\xcf\xef\xe3\xb1g\xb7$J\xeaޠ\x12&\xad+\xef\x06\x8eQ\x17W\x1f\xaa\x84_\xc8_k\x02A\xb7\x0fq\xc18{\x80\x9ds\xb1\xb8d\xc87\x1e*'\xa2\xa0\x81\xd2)HS<\xc0\x8e@\x8d\xe7G\x8c\x979\xd2\xe2\xca\x03\x8cl\x99\xc6J\x8f\xae\x88\x9f߈rt\xc3\x0fD\x98\x94\xd9Ԗ\x86\xa8~\xfa\x8cd'\xc4\xcb\f\xbd\x14J\xe0ˑ\xc3N\x16\xa7n_\xfd\x8c\xa2\a\xd8\xfd\xc68^\xe3,\xdb\nڱ\xe3\xb4z\xa36\xb3\x18\xee\xca\x1d/D\xdet\xe6\xe6\xd5J^\xb0\xcf\xca\xe2?\x1f\xbe\t\x83\x1d˜\xbdW`>+K_^\x94\xcan\x10\xafA\xe3\xb0\v\x88\x03t\x96\x04\x89\xd8ͼq\xb6\x14\x05\xb5\xe1\x870l%1$s$\x9a\xd1\x1d%Z\xb9.]gemh\x9fZ*\xb9p\xcbbc\xbdy\x1e(\xddc\xc1I:\xf6\x9dޢ1r\xbf\xb8\x94\xaf\x82g\x90\x87\xad:\xcaE\xe2\x16\xeeE6\xa3\xcf\x12\xf4=\xb0\n\xcdB\xba\xb4\xccP\xd4~d\xf3\xc5+\xdds\xe8\x96o\x8b\x87z\rZ\x82\x05\xb3@\xb3\xb6\xf0P\xac*\x13\xe9\xe2m\xc2H\xc2\xceXY\xe0\\O\xac\x19\xa4%\xa9z$\x9d\xe9p\xf5Tb=\x93L\xe4E\x90ە$\x05ݬ\xe0y\xd6k\xa6\xdc\x1c\xa3b:cq.@\xc9ik\xedoh\xe9i6\xfe\x9dU\\h\xb3d\xef(-\xba\x80\xdeo~a\xb2\x03&\xb1ۊV\xd9\x7f\xa9\xc5#/\xd0\xff@\x03!\x19\x14\xce\x1bQ\x9b=_\xed\x82=m\x95qnC\xb3iw\xfe\x00;\xb7\xb3\x9c\xd4mWa\x9d\xaf\xe4\xb9\xf3e\xf6\x14O\xe3\xf8(Y\xec\xd89\xfdv\xfe\\\xf7n\x86DϨ\xda\x13\xe5\x92W\xe9\x92LI\xc7s\x02\r\fփC\x84\x8d\x9b\xec[\f\x10\xa6(\x90,ʕ2\x91\xa4\x91\bZ\t\x82~\xad\x8cu\xeb\x90=\x7f\x7ft\xa1R\x85\xc5I\xc67\x1643V\xe9\x90ϊ\x8a?e)\xbe[n\xb7`\xc0\xefC\xf9EO\a\x18\xa3\xd8\xf3V78\xabr\xee\xf6¨#\x9e\x91\xf7Dm+\xad20Ѽ\x88\xb6$ڦ\x91\xec\xa5.\x1d\x9au]\ue8bfM\x92\xd6NY\x94\x0ee\x9e#\x8f\xa4;\"2\xfa\xf0\xad\xb3D\x8d\xda\x05\xffN\x91\xd6cpdt\x10\xa6,\xf90\x97:\x19\xdd+\xd7:\xcc1\x0f̅[\xfa\xbe&\x9d3\xc7\xebhD\xf9\x1f͵)\x85\\QG\xec\xed\v\xbaC^\x8b\xc7Ҥ\xc6\xca\xf1N\xfaU\xe8\xac\xe5^\xf3\xc1'$*\xda\xf8\xd1\xd0c\xee\xfe\x9e\by\xd7R\xd9\xce2\xceL'\xbaR\xf9o\f\xdb\bml\x17\rs \xb1j\x14\xd4\x11\xa1\xa7\xfc\xa0\xf5ё\xe7\x17\xd7z\x90B\xe9\xb2\xce\xe7\xc4ہ\xa4[\xfe\b>\xed\x17d\xa6jIKa\xa8\a\xb0\x9b\x19\x10\x1dk\x9c\x15H\xb4w\x9d\xc6\xd1Ḻ\xb2 I\x12rrݬ\xdb\xe4#\x17i\xebV\xec8\xb6\xdaC\xb9\x9cc\xe5\xf8y\x14\x12=\xbbg\x11J\xfeM\x94u\xc9x\x89<$\xb7C\x94\xd0\x1cGp\xecn\xd2?\xb1\x05\x19-\xabp\x96U\x05X\xf0\xe9\x9b3\xf0Ȕ4\"\x87\xc6\xf4{\x11P\x92q\xb6ᢨ\xf5\f\xad:\x9b\xe4s\x830\xafMN\x1fY\xa5#\xb2 \x12%\xae\xb3\xcf\xf0\x82\xa75~\xa5\xe7\xf9\xb1)\x0e\xa3\x86\xf9\xfeb\xa5\x85r')N\xef2\xfa\xf4c.w?|\xc6\x1f>\xe3\x0f\x9fqNG?|Ɖ\xf2\xc3g\xfc\xe13\x1e.?|Ɣ\xf2\xc3g\x9c\x89\xc8\xf7\xf2\x19S0\\\xd0\x1a\xe7\x81\nIX%\xa6BL\xa1=їO\xfa\xf1g5N\x92˼\x1a\a9r\x88'r\xfc\"\xe6u\xb4ƫIn\xc6\x19\x18\xe6\x8e;\x82\x9a\xe00\x9f\xe0\xf4L@\xe0\xf4\xa7gV\a!\x9f\xf0\xf4\x8c\x1fBZ\x84q\xd4ٙ@\xa4\xf9\xa7'.|\x12Q\t\xaaA\xfe\x83mU\x1d9\xb51Aڄ,\xda4\x82\xf4\x92j}b\x04X\xfe\xf8v\xd9\xff\xc5*\x9fbK77D\x80\xd1e\x1e<\xa7\x1b\x1c:\az\xbc\x1e\b\xf7L\r\x852\x02Li&E\xe1$6@\xe8\xc9+\xfbR\xb9\xd5\xc1\xa3\xfd\xa6\xe95\xac\xf4Dܹ\xe9\xb7M\xb6\xe4\xb4\xfb\xfe\x8c\xa4ۓ\x1e\x8d\xfani\xb5\xc7%Ӧ\xaeP&$Φ\xa7˦\xb0Օ\xf4$\xd9\xe4\b95!v\xee\nċ&\xbf\xbeL\xcak2\xcd\xd2\xd2[\xe7R\xecURY_9\x81\xf5\xf5\xd2Vg$\xab\x9e\xfe\xd4K\xfaZ\xfa\xd1ٕi\xcb2\x87\x13N\x93\xd2L\x93\x96nR\x06|\xd4P\x93\xd2G\xe7&\x8d&q2}\xba\xbejZ\xe8\xab&\x83\xbe~\n褴MV\x98\x9b\xe49~Cd(\xd3\x0e\xc0\xde\x05ϣ\x95^BBg\xad\x97\x1e\x9c\xc0\xc5\xf7\x98`\xcfe\xb5ҽ\xf0\xe2Y\xb1\xf3\x97\x01,\x14\xf8\xe0j\xbfb,Sօ\x15U\xd1^\xc8\x17\v\x9a\xb7\xb0k.\\\xfaY\xd11\x7f\x7f\xebؗ\xafͬ]\x0e\"3n\xd8\x13\x14\x05\xe31\xfd\xb2G\x85\xcc]\x04\x9b\xa9\x05\xa0}GM\xe5/\x94\xf2\xb7\xc7^\xb8)O7\x1a\x90\x97PƖ+\xb9<|[\xd9A#\x9c\xaa\x8b\xf7\xa2\v\x173ѷ_j\xd0;Fw\xa75\xfee{0\xd6++\x83\xc1uP\xa1^\xa5\x1f\xda\xf7\xd9\v\xd2Z\x15\xc7\xdeI\xe7\xd5\fq\xa26\xa8;۠\x14\r\x83\x8c]4\xc8\xc2\x04\xdd\a!U\x03!\xd24%\x80\x99sR\xf4%B\xd4S\x04\xa9I^\xdc<\x0f\xfc;\x9e\x00=\xf6\xe4gzBM\xd2Iϗ\bY\xe7\x04\xad\xb3|\xee\xf4\x93\x9c\xf37\xcf_\xf8\xe4\xe6K\x9d\u061cA\xbd\xd4\x13\x9a\xf3i\xf7J'2_\xfd$\xe6k\x9e\xc0\x9cu\xf229\xc5lV\xd6Ĝ\xf4\xb0g\x1c\x19Lˇ\x98>I\x99x\x8221[\"m\xf0G\x0e;\xf1\x84\xe4\xfc\x93\x91\x89\xfc\x9d3\xa5_\xf9\x04\xe4+\x9f|\xfc\x1e'\x1e\x13$0\xa1\xca\xfc\x93\x8d\xcf\xdeVS:\a=\xb9u9Gj'\xe555\x96\xeb#6؛\v7\xe2b\xad^\f@fɿ\xe4@\xafv\x1c\xda\xcaG\xc9\xecxD\xbd\xbd\xd5\xd6]\xeb;\xc4\xfe9\x0f\xb7\xfdj\xa0\xe2h\x00(p\xa3\xf4\xb2\xa8\xab\xf0\x81g\xdbA\x0f[n\xd8F\xe9\x92[v\xdelx\xbfq\x1d\xe0\xdf\xe7K\xc6>\xaa&ߨ{\xe7\x9b\x11eU\xec0\x12c\xe7\xdd\x06ϓ\x92\xa8t\x86\x9e\xafU!\xb2\x88\xcf9z7\xa0k\xb0wa\x12\xdd^\x98u2^b\x81\x0f6\x17\xe1\xe6\xc8\xfe\xb5\xd2\xeeB\xffc\xd7{*\xf1'zo\xeb\x04+\x87\xef\xaeW\x04+\x88\x11=\xe4\xd5$Y6,_\x03\xba\f\xed\xd8\x0f\xe9\x93զ\a\xb5\x9f\xe7\xdc}\xad\x04r\xf74Mp[\xbcj\xce\x14j\xad\xeb\x95\xc3\xe5PO(_\\\xee\x98\xf2o\x8f\b\x9d/*\xae\xed\xce%D]\xf4\xf0\bv}j\xe5\uf835\xda\x7fz\xa7[zd\x0f\xaf\xee\xd0n\xfc\xae\xea'@\f\xe9\xf9\x1c\x9c\x0e\x9f\f\x9f<\x13\xfe\x028\x1dv\xa1\x16D\xc5\xc8O\xd1,Γ\xaf\xba\x1a\xff\xda\xc0_\xd4#\xbc\x8f\xae\xbe\xf6\x9f\xdf\x194\x19I\xaf\fP\xe9\xa2\xfc\b\x05ۜJ\xba\xa7\xfcyj/\x9e/\x19P\xf1\xf7\x9c?gq\xf2\xa6\x0fj\xfcE\x1a\xba\x05>t\x1a\xf3\xaa譯\x1d\xbb\xbe\xa3\xb8\xb5Q\xa5~\xea\xfb\xb85,O\x86$\x89\b,!\x0f>\xd2s*2Z\xa5\xf9=|R\xeeq\xa5\x141\xe9\xb7\xe8=\xbd\xe5=\xb7\x90s\xee'aL\xd1\xfb\xb1\r\x01\xb6gL\xf6\x1e+@l\x8f|\x8e\xc1\xda\xe292r{\xfbɍ\x94\u07b4y\uf7e7A}l\x00Y\x10(࠭\xf1\xbf[\xf5D\x97\xf8\xc7ט\xc3\v2\x9dG\xec\x80\x0e\xbbP\x1a\xf2Qì\xabB\xf1\x1c\xf4\x15\xbd\xa2\x930\xe2\x9fz\r\x06\xee@\xff-\x1eo7#\xe3\t=\xbf`\xa6\x0fztE\x01\xc5GQ\x80q\x88'\x9a\x86\xeb\xfd\x96\x8d\xa5\xa8˵\xf3T7\xf8c\xd3\xc9\x01\xcb\xec\x86J\x1b\f\x15h\xf4\x13\xddVDm\x82\xe4\x1f&\x06k\xf8(\xa4\x85{\x18\x8f\xa1'l\x82{g\x82\x1c\x80\xa0\xc0(\xe2\xfbsl\xe5\xb1G\x90\xbbx\xeb\x81\f4\x8b\x9119Vޭ\xba\xbe\xbb2\xac\x969m\x00\xdc\xfd\xe9\xe6(\xf9}콕\x13tB\x8az\xbf\x1bo\xd9\t\x11:ډ|\xfa\xb8\x12\x8f\xc1\xe2ƨLPTA\x8f#\xd1i\xaa\x97\xbb\a\xfdP\x80x@:j\x03_\x9e$\xe8\xaf\xc1\x02\x99\x95\x8c\xbd=3\xad\xfd~ڃ\x16}s\xc6*\xec{\x04\xc6\x00\x00Sa\x9f˸W\x8d\xc2\xf6\x9a0\xcd\xebv\xfb\xf4\x9cP!qK7\xee\xb0-\xc6\x1f\xe3Z4\x8f\x86\x9d%\x90۽\xe5\xd4\a<\xfe\xa6\xa1{\xf4)㕭uЮ\xb5\xa6\x9b\xe2\x11\b\xb8\x8bԏ{հ}\xec\xee\x18\x06\xb7\xaf͵\xfb\x0f\x93\xef\xe1\x8e\xc0i\xde%\x8c>r\xe6\"j\xf7^\xed\x02\xe1\x1f\xc7\xe3\xd1\x19\x838߸\xc7\xeb&\x88\xf0\xa9\xad96\xe0f\x188d\xff\x1cޫ\x8e\x84\x1e\x0e\x98\x18\xc35\xd6iN\xeaz9\xa2\x86\xe1\xc1\x81\x9b\x18\x13Əs.\xd8g؏\xd8\x17\xec\x83\xc4A\xec\x13\xc0\x9dل\x9c\xb6VH;\xce\x19\xe2cӊ\x0e̎h\xc8i\xb1\xbd\x1b\xc0\x18d\xe3\xd3\xc3UM\x15wbְߊ1o\x94v\xcc2\x1c\xe8\xef\xf6~\x8dj\xf0\x83\xda;\xa6\xb9G\xd5\xc8\xdeGz\t1\xefH\x8e\xf7һ_\xeau\xfb(\x04\xfb\xdb\xdf\xcf\xfe?\x00\x00\xff\xff\xeaC\x1a-[}\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcV\xcfo+5\x10\xbe\xe7\xaf\x18\x89+\xbb\xa1B \x94\x1b*\x1c*\xe0\xa9j\x9ezw\xbc\x93d\xa8\xd7^f\xc6)A\xfc\xf1\xc8\xf6n\x9b\xee:\xb4\x8f\x03\xbe\xad\xed\xf9\xe6\x9bo~x\x9b\xa6Y\x99\x81\x1e\x91\x85\x82߀\x19\b\xffT\xf4\xe9Kڧ\x1f\xa4\xa5\xb0>ݬ\x9e\xc8w\x1b\xb8\x8d\xa2\xa1\x7f@\t\x91-\xfe\x84{\xf2\xa4\x14\xfc\xaaG5\x9dQ\xb3Y\x01\x18\uf0da\xb4-\xe9\x13\xc0\x06\xaf\x1c\x9cCn\x0e\xe8ۧ\xb8\xc3]$\xd7!g\xf0\xc9\xf5\xe9\x9b\xf6\xe6\xfb\xf6\xbb\x15\x807=n@\x90ә\x1a\x8d\xc2\xf8GDQiO\xe8\x90CKa%\x03ڄ\x7f\xe0\x10\x87\r\xbc\x1e\x14\xfb\xd1w\xe1\xbd\xcdP\xdb\f\xf5P\xa0\xf2\xa9#\xd1_\xae\xdd\xf8\x95\xc6[\x83\x8bl\\\x9dP\xbe \xc7\xc0\xfa\xe9\xd5i\x03\"\\N\xc8\x1f\xa23\\5^\x01\x88\r\x03n \xdb\x0e\xc6b\xb7\x02\x18\x05\xc9Xͨ\xc5\xe9\xa6\xc0\xd9#\xf6\xa68\x01\b\x03\xfa\x1f\xef\xef\x1e\xbfݾ\xd9\x06\xe8P,ӠYֿ\x9b\x97}\xa8\x85\t$``\xa4\x04\x1a\xc0X\x8b\"`#3z\x85B\x19\xc8\xef\x03\xf79\xad`v!\xea\x05\xaa\x1e\x11\x1e\xb3\xfec\x98\xed\xcb\xe1\xc0a@V\x9a\xa4)\xeb\xa2\xe2.v\xff\x8dxZ)\xd6b\x05]*=\x94\xecy\xd4\v\xbbQ\x1e\b{\xd0#\t0\x0e\x8c\x82\xbe\x14c\xda6\x1e\xc2\xeew\xb4\xdaΠ\x8b.\x922\x19]\x97*\xf6\x84\xac\xc0h\xc3\xc1\xd3_/ؒ\x04JN\x9dѬ\x9dWdo\x1c\x9c\x8c\x8b\xf85\x18\xdf͐{s\x06\xc6\xe4\x13\xa2\xbf\xc0\xcb\x062\xe7\xf1[`\xccRo\xe0\xa8:\xc8f\xbd>\x90N}hC\xdfGOz^疢]\xd4\xc0\xb2\xee\xf0\x84n-th\f\xdb#)Z\x8d\x8ck3P\x93\x03\xf1\xb9\x17۾\xfb\x8a\xc7Ε7n\xf5\x9cjP\x94\xc9\x1f.\x0er\xeb|AzR#\x95b*P%\xc4\xd7,\xa4\xad$\xdd\xc3\xcf\xdb\xcf01)\x99*Iy\xbd\xba\xd0e\xcaOR\x93\xfc\x1e\xb9\xd8\xed9\xf4\x19\x13}7\x04\xf2\x9a?\xac\xa3\\\xb8qד\xcaT\xda)us\xd8\xdb<\xab`\x87\x10\x87\xce(v\xf3\vw\x1enM\x8f\xee\xd6\b\xfeϹJY\x91&%\xe1Cٺ\x9c\xc0\xf3\xcbEދ\x83iv^ImeJl\a\xb4)\xb9I\xdfdM{\xb2\xa5\xad\xf6\x81\xc1\xd4L\xda\x0f1\xc9\x16_\xc8e\x9cH\x85\xcdlN\xa5.\x7f\x9fM},哣\x11\x9co\xce8ݧ;s\xff\x8e\xf6h\xcf\xd6a\x81(S\bߧ\x92\x16\xfa\xd8/}6\xf0\t\x9f+\xbb\xf7\x1c҄\xc6\xf9\xa8\xb9Z\x1bP\x1e\xb1\x03\xf9E\xb8\xf3\xc8ʭ\xfc0.G~\x0eh\x04\x02\x8eާ\x96\x0e~\x01Yy\x11\x16wH\xb1\xaf\xb0\xa9\xf2\xb9\xf3\xfb\x90\xff\"Lrl\xb4\xb4\x13\x8e\xc9\x1e\xfd\x14^\x15\xc0\xeb\xb9.k9\xe7>$hY\xf9y\xfeo\xc6i.\x11c\xd5w\x93YU\x0f\x92ǚ\xe2\xf5\xfe\x1aYF\xe7\xcc\xce\xe1\x06\x94\xe3Һ\xd8\x1afs\x9eW\xcdTj\x9f\xa9GQ\xd3\x0f\xef\x14\xd0\xe2UH\xeb~\x81\x92\x9a\xe7\xf9\x88\xfeZ\x8b\xc0\xb3\x91W\xe7\x15\xc8\xdd\xf9\x9a\xe9\xed\xcb\xdf\xe6\xb2\xcfJ=o \xcd\xfaF\xa9\"䇔\xaa\xa6\xb4\xd4y\xf5\xb7f\xa1\xd2\xf6\xf2\xee4H\xde\xf4\xcb\xf4W\xb3\x8c\xe1*\x85j\x05,63|w\x11\x9eh`s\x98\x02\xfe'\x00\x00\xff\xff\xef\xf8\xa6>\x10\f\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcWM\x8f\xdb8\x12\xbd\xfbW\x14\x92k$o\xb0\xd8\xc5·F\xef\x1c\x82I\x06\x8d\xb8\xa7\xef4Y\xb2\x19S\xa4\xa6\xaa(\xc7\xf3\xf1\xdf\a$%\xb7-\xcb\xe9\xf4`0\xbat\x8b\"\x1f\xeb\xe3իrUU\v\xd5\xd9'$\xb6\xc1\xaf@u\x16\xbf\n\xfa\xf4\xc6\xf5\xfe\x7f\\۰\xec\xdf/\xf6֛\x15\xdcG\x96\xd0~F\x0e\x914\xfe\x1f\x1b\xeb\xad\xd8\xe0\x17-\x8a2J\xd4j\x01\xa0\xbc\x0f\xa2\xd22\xa7W\x00\x1d\xbcPp\x0e\xa9ڢ\xaf\xf7q\x83\x9bh\x9dA\xca\xe0\xe3\xd5\xfd\xbf\xea\xf7\xff\xad\xff\xb3\x00\xf0\xaa\xc5\x15\xf4\xc1\xc5\x16٫\x8ewA\\\xd0\x05\xb3\xee\xd1!\x85چ\x05w\xa8\xd3\x15[\n\xb1[\xc1\xf3\x87\x021\\_L\x7f\xcah\xeb\x01\xed〖78\xcb\xf2\xe376}\xb4,yc\xe7\")wӲ\xbc\x87w\x81\xe4\xa7\xe7\xdb+\xe8ٕ/\xd6o\xa3St\xeb\xfc\x02\x80u\xe8p\x05\xf9x\xa74\x9a\x05\xc0\x10\x9f\fW\x812&G\\\xb9\a\xb2^\x90\xee\x13\x96?]f\x905\xd9NrD\x1f(\xf4\xd6 \x81e\x90\x1dB7\xbe\x87&\xbf\x17;\x80%\x90\xdabF\x00\xf8\xc2\xc1?(٭\xa0N\xf1\xad\xc7C\xc3璛\x87\xcbE9&\xb3Y\xc8\xfa\xed\x9c!%\xae0\x06\x16\xc6\xc8\x02\x8b\x92\xc8\xc0Q\xef@1\xdc\xf5\xca:\xb5q\xb8\xfc٫\xf1\xff\x19\xbb\xf2\xa9\xba\xdb)\xc6K\xb3\xceVfl:\x83\x18\t[k\xc2lʣm\x91E\xb5\xdd\x05\xe0\xdd\xf6\x12\xce()\v\x03Eߗ\xcc\xea\x1d\xb6j5\xec\f\x1d\xfa\xbb\x87\x0fO\xff^_,\xc3\\H\xa6TK\x99R02\x02\x0e;$\x84\xa7\xcc\xeb\x9c&\xe4!i'P\x80\x91G\\\x9f\x16;\n\x1d\x92ؑ\x84\xe59\xab\xf3\xb3Չ]\xbfW\x17\xdf\x00\x92+\xe5\x14\x98T\xf0X\xb84\xd0\x12\xcd\xe0}\xe1\x94e \xec\b\x19}\x91\x80\xb4\xac<\x84\xcd\x17\xd4RO\xa0\xd7H\t&\xd5Lt&\xe9D\x8f$@\xa8\xc3\xd6\xdb_O\xd8\f\x12\xf2\xa5N\t\xb2@&\xbeW\x0ez\xe5\"\xbe\x03\xe5\xcd\x04\xb9UG LwB\xf4gx\xf9\x00O\xed\xf8\x14\b\xc1\xfa&\xac`'\xd2\xf1j\xb9\xdcZ\x19\xd5O\x87\xb6\x8d\xde\xcaq\x99\x85\xccn\xa2\x04\xe2\xa5\xc1\x1eݒ\xed\xb6R\xa4wVPK$\\\xaa\xceV\xd9\x11_Ԫ5oi\xd0K\xbe\xb8\xf6\x8a\x9f\xe5\xc9j\xf5\x8a\xf4$\xe1*\xac)P\xc5\xc5\xe7,\xa4\xa5\x14\xba\xcf?\xac\x1fa\xb4\xa4d\xaa$\xe5y\xebU\\\xc6\xfc\xa4hZ\xdf \x95s\r\x856c\xa27]\xb0^\xf2\x8bv\x16\xbd\x00\xc7Mk%\xd1\xe0\x97\x88,)uS\xd8\xfb\xdc!`\x83\x10\xbbTPf\xbaჇ{բ\xbbW\x8c\xffp\xaeRV\xb8JI\xf8\xael\x9d\xf7\xbd\xe9\xe6\x12\xde\xf3B\x1d\xdaՍ\xd4\xce+ºC}Qx\t\xc56vP\x88&\xd0$@jԋy\xbc\xcbx\xce\v\x05\x94\xa6\xdd\xd8\xedt\x15.\x1aЭ\xb3\xdf\b،\xdf\xf7\xf9\xa6\xc4\xe1&ЩGU\xa3\x9f\x83%\x91\x06\x87-:s\xc5ԛ1Ϯ\x10\x9a\x94b\xe5\xae\r\xbd\xb4\xe4\xb41\xcf,\xca\xfa\x12\xf2g\x80\xcc\x7fC\xfe%1\xa6\xc1\xd9\x06\xf5Q;,\x80\x10\x9a\x19\xee\xbd\xca\xe4\xf4\xa0\x8f\xed\x1c\x11\xef&?|ο]\xff,\x9a\xa6m&\xf9\xb3\xf9\xbcZ\xe44\xec\x99\x15\bł=\xb0\xec|%nN\xb3\xec\n~\xfbc\xf1g\x00\x00\x00\xff\xff+\xf2\xd32>\x10\x00\x00"), -} - -var CRDs = crds() - -func crds() []*apiextv1.CustomResourceDefinition { - apiextinstall.Install(scheme.Scheme) - decode := scheme.Codecs.UniversalDeserializer().Decode - var objs []*apiextv1.CustomResourceDefinition - for _, crd := range rawCRDs { - gzr, err := gzip.NewReader(bytes.NewReader(crd)) - if err != nil { - panic(err) - } - bytes, err := io.ReadAll(gzr) - if err != nil { - panic(err) - } - gzr.Close() - - obj, _, err := decode(bytes, nil, nil) - if err != nil { - panic(err) - } - objs = append(objs, obj.(*apiextv1.CustomResourceDefinition)) - } - return objs -} diff --git a/config/crd/v1/crds/doc.go b/config/crd/v1/crds/doc.go deleted file mode 100644 index 9eed410f6..000000000 --- a/config/crd/v1/crds/doc.go +++ /dev/null @@ -1,4 +0,0 @@ -// Package crds embeds the controller-tools generated CRD manifests -package crds - -//go:generate go run ../../../../hack/crd-gen/v1/main.go diff --git a/config/crd/v2alpha1/crds.go b/config/crd/v2alpha1/crds.go new file mode 100644 index 000000000..111b68cdc --- /dev/null +++ b/config/crd/v2alpha1/crds.go @@ -0,0 +1,58 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package crds embeds the controller-tools generated CRD manifests from +// ./bases into the binary via go:embed. +package crds + +import ( + "embed" + + apiextinstall "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install" + apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/client-go/kubernetes/scheme" +) + +//go:embed bases/*.yaml +var basesFS embed.FS + +var CRDs = crds() + +func crds() []*apiextv1.CustomResourceDefinition { + apiextinstall.Install(scheme.Scheme) + decode := scheme.Codecs.UniversalDeserializer().Decode + + entries, err := basesFS.ReadDir("bases") + if err != nil { + panic(err) + } + + objs := make([]*apiextv1.CustomResourceDefinition, 0, len(entries)) + for _, entry := range entries { + data, err := basesFS.ReadFile("bases/" + entry.Name()) + if err != nil { + panic(err) + } + + obj, _, err := decode(data, nil, nil) + if err != nil { + panic(err) + } + objs = append(objs, obj.(*apiextv1.CustomResourceDefinition)) + } + + return objs +} diff --git a/config/crd/v2alpha1/crds/crds.go b/config/crd/v2alpha1/crds/crds.go deleted file mode 100644 index 96990c557..000000000 --- a/config/crd/v2alpha1/crds/crds.go +++ /dev/null @@ -1,60 +0,0 @@ -/* -Copyright the Velero contributors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by crds_generate.go; DO NOT EDIT. - -package crds - -import ( - "bytes" - "compress/gzip" - "io" - - apiextinstall "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install" - apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "k8s.io/client-go/kubernetes/scheme" -) - -var rawCRDs = [][]byte{ - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcYK\x93\xe3\xb6\x11\xbeϯ\xe8\xda\x1c\xf6\xb2\xd2d\xf3p\xa5t\xdb\xd1\xc4US\xf1Ϊ\xac\xc9\xdcA\xb2I\xc1\v\x02\b\x1e\x92\xe5$\xff\xdd\xd5\x00IA$4z\xd8^\xdd\x044\xba\xbf~\xa0\x1f\xe0l6\xbbc\x9a\xbf\xa2\xb1\\\xc9\x050\xcd\xf1g\x87\x92\xfe\xd9\xf9\xd7\x7f\xd89W\xf7ۏw_\xb9\xac\x16\xb0\xf4֩\xf6G\xb4ʛ\x12\x1f\xb1\xe6\x92;\xae\xe4]\x8b\x8eU̱\xc5\x1d\x00\x93R9F˖\xfe\x02\x94J:\xa3\x84@3kPο\xfa\x02\v\xcfE\x85&0\xefEo\xff<\xff\xf8\xdd\xfc\xefw\x00\x92\xb5\xb8\x00\xe2W\xa9\x9d\x14\x8aUv\xbeE\x81F\u0379\xba\xb3\x1aKb\xdc\x18\xe5\xf5\x02\x0e\x1b\xf1`'4\x02~d\x8e=v<²\xe0\xd6\xfdk\xb2\xf5\x03\xb7.lk\xe1\r\x13#\xd9a\xc7n\x94q\xcf\a\xfe3\xa8\"G\xcbe\xe3\x053LJ\xee\x00l\xa94. \x9cѬDZ\xeb\x94\r*yl\x98\aZ\x85d9\"!/5h\xb2\xd6Q\x8e\x89\xdf\x02\xc4\x11\x83\x87\xe4|D\x12\xf9\xa6\xebg\xa1Pȁ\xaa\xc1m\x10\x1eX\xf9\xd5kX;eX\x83\xf0\x83*\xa3\xfbv\x1b4\x18(\x8aHA\xd1\v\x9c|\xa7L\xd6u\x1a\xcby\xa4\xed\x98\xf5\xbcF\xfe;\x16\xf4\xbb\xc7Vi\x90ec\xab\xcfA\xf3@\xc1\x95\xcc\aا\x06/\n\xaeԈRU\x98X\xec\b\x13\xb7\xa0\x8d*\xd1\xda7\x02\x9e\x18\x1c\xa1x>,LL\x13)\xb6\x7faBo\xd8ǘd\xca\r\xb6lѝP\x1a\xe5\xa7\xd5\xd3\xeb_\xd7G\xcb\xf0F\xc2`\xa5\xb3\x94)\b\xbe6ʩR\t(\xd0\xed\x10et}\xab\xb6h(\x016\\ځ#\xa5\xf3*%8$s\x8a\xef\xc0\x8fv\xe3\xa6\xc1\x10=\x04Ф\xde\a\x92\xa9\xd18ާώ\xf7\xa1\xf2$\xab#=\xfe7;\xda\x03 \xd5\xe3)\xa8\xa8\x04aT\xab˭Xu֊\xce\xe3\x16\fj\x83\x16e,J\xb4\xcc$\xa8\xe2',\xdd|\xc4z\x8d\x86\xd8P\xb6\xf7\xa2\"e\xb7h\x1c\x18,U#\xf9/\x03o\vN\x05\xa1\x829\xb4.\\F#\x99\x80-\x13\x1e?\x90\xd1F\x9c[\xb6\a\x83$\x13\xbcL\xf8\x85\x03v\x8c\xe33Y\x91\xcbZ-`㜶\x8b\xfb\xfb\x86\xbb\xbe\x1e\x97\xaam\xbd\xe4n\x7f\x1f\xbc\xc1\v\uf531\xf7\x15nQ\xdc[\xde̘)7\xdca\xe9\xbc\xc1{\xa6\xf9,(\"CM\x9e\xb7՟LW\xc1\xed\x91\xd8I \xc6_\xa8\xa4W\xb8\x87\xca+\xdd\nֱ\x8a*\x1e\xbc@Kd\xba\x1f\xff\xb9~\x81\x1eI\xf4Ttʁtb\x97\xde?dM.k4\xf1\\mT\x1bx\xa2\xac\xb4\xe2҅?\xa5\xe0(\x1dX_\xb4\xdcQ\x18\xfcǣu\xe4\xba1\xdbe\xe8Y\xa0@\xf0\x9a\xf2A5&x\x92\xb0d-\x8a%\xb3\xf8\x8d}E^\xb13r\xc2E\xdeJ;\xb11q4o\xb2ѷR'\\\x9bf\x90\xb5ƒ\xbcJ\x86\xa5c\xbc\xe6]%\xa14\xc0\x8eh\x8f-\x94\xbf\xfa\xf4\xcbV\x931ѹp\xa3\xdfC\x8eQ\x8fV&\x89\xbc\xabu\xb6+R\xe2\xb8H\xa5\xbfI}4\xa8\x95\xe5N\x99\xfd\xa1J\x8eC\xe1\xa4W\xe8W2Y\xa2\xb8E\xbde8\t\\Vds\x1cB\x99\x92P\xe4\x1a\x80*\xd9(\xba\\G\xae\x80'G4\x14\xdb\x16]^Q\x99\xadj\\¡\xa7\x84\xb4w\x1c\xab[(%\x90\x8d\xadHQ\xf8\x99\xca\xc2Rɚ7S\xc5\xd3\xf6\xf7T\x88\x9c\xb1i&`\x13\x91\xa4\x05E'!\x99\x85\n5\xebC\x97R{\xcd\x1boN\xf9\xbf\xe6(\xaaI\xfe9y\x93z\x85\x83\x94[|<@\xefoWWՒ\xd2\xebT\xc8P6\xf4\xbbIhNA\x02<\xd5\tGn\xe1\xdd;P\x06\xde\xc5a\xe9݇x\xdas\xe1f\\BmS1;.D/\xe8\xaa\x00\xa7&\xe7\xcb\xfa\x8c\xf2ρ\x88 }Y_\xdb^MѠ\xf4\xedT\xe0\f\x98w*\xb3,\xb8\xf4?g\xd6w\\Vjg\xafQvhq\xa8\xcbT\xde\xdd\xe2\xf3/#\x1e#\xd7;ꉃ\xbb\x9d\x82\x1d\xe3I\x9b1H\xb7\x1f2|\v\xac\xa9&\x19t\xdeH\xca\bh\f%i\x1bX*?i{\xde\xd4\xd4J\xa6\xedF\xb9\xa7\xc73:\xae\a\xc2>\xf5>=\xf6.~\r\x817\xe4ߎ\x122^\"\xf8}#Y\x85\xca~\x13\xda5\xff\x05/\xc4K\xa4=b\xa1\x1a^2\x016\xac\xc9n\x0e\xec\x94\xe8yO\x01\xe5F\xbd1\xdct`K\xf0\x86\xf6gx$\xb8%\x8c\xd6\xc7,zU\x94\xe1\r\xa7`\x91\xc3\xce\xe1\x8em\x95\xf0m %\x97`\x05^\x9f\xb05P\x05\xa1~\xab@\xa8x]\xa3\xa1\xa6*t\\Q\xf0\xeau\xf9\xde&Bx\x9d\xfe\xa1b\xd52\xad\xb1\xa2\U0004e0b1\xf3\xedU^u\xcc4\xe8^\x03\xe83&zIH{SPwF\x0e\xea\xda\xffp\xb9\x02\x19\xac^\x97\x99f\x9d~\xab\xd7)\xc2ӭ\f\xfdj\xfbB\x1advF\x10\xbf_\x13a\x0f\xae\xe6\x02\xc1\xee\xad\xc36\x98`\x840z*\xe7\x973\x95\x11\x0en\xb8\x00\xd3$|:\xf1\x03\x8f[\x00\xe8\xed\x05\x92W\xaf\xb9Nm\xf0\x0f\xb8\rsD\xd1\r\xfeP\xec\xb3<\xa1\xcf1]|݆\xb7\xbc\b\xf0\xf2M\xc4\xcb1\xe4\x13x\x8b\xfdo\x86L\x8d 7X\xe5j\xe0i\xcf\xcd@o\xb3\x8b\xe5\xe5\xedN^\xf2,\xdfӏhƵs\xb4}(8\xe3\x8d\xe3D7\xdaMs\xc4E\xc3Ox\x9a\xb9t\xfc\x89\x0f\xae\x9d\xdbKoB\x16\xec\x9eaU}\xe3\x00\xc4\xca\x12\xb5\xc3\xeaaOm\xd1\x05\x9d\x13\x01\x90o?L\xfd[\x1f\xfa&\xd4\xec\xda)\xa5\x874<\x9e\xddR\x91>\x8d\x99\x84\x17\x14S%}\xcd\x14nloO\x83\x06x\xa1\x1a\x1c^\x00\xde\xc7V\x86\x8e\x85\x06\x89\xba\xfc\x89ГU\x9aF\xfc\x19\x9d\x9fPH/\x04+\x04.\xc0\x19\x7fj\xdc\xc9Ow\xf1-:}v\xbciԛ\xb2\x99ڎ\r\x0fm\xe1A\xb4\x7f\x05ϙ\xec\xc0o0Xd\x87\x15\xe0\x16%\xd0\x00ϸ\xc0\xaa癙y\xceY>\x03z\xdaK\xff\x91\xc6o\xd1Z֜\xbb@\x9f#U|\x9b\xea\x8e\x00+\xa8\xf1\x1e\x8f\x1d\xefmw\xb7\xaf\x1e\x80~\x9fK|\xe1\xf8\xf3\x06\x960\xaf\x9f\x01\xb3\"\x9a\\N\x1b\xa0\x9dNj\xf0\xc6\xf4\xf5\x8c\xbb\xccj\x7f?3[\xab\xee\xd2g\xb6&\x9f\xb5\xd2\xcd\xf80\x92+\x8c\xfd^\x96\xe7\xf0\xdd(\xb3\xf7}\xb8\fWY\xba\xc3w\xcbu\x1f\x9eW6J\xf47<|\uf47e-А\x1b\x8a\xdc\x04\x12^\xe5\x13\xaf嚿\x81\xc30L\x05Vsx\xd9Pk\x12߄\xfa\xf1\xb2\xe2V\v\xb6\x1f\x94I[\xe6\f\xf3í\x99<\xf9_\xdb5\x0f\xdf\xdf\xf2\x9d\xd7ۓ\x15\x9c\x99\xae\xc2\xfe\xf0]폑\xf0Ƌ\xd0\xf1wΛf\xbb#\x0e\xe7JA\xf7\xdd\xf5\xfa\f~,\xe6[&\xef\xac\xf5&\x8b\x01y\x95\xf0\xee^p\xd3\x15_\f\x9f5\x16\xf0\xdf\xff\xdf\xfd\x1a\x00\x00\xff\xff_zG\xb9\xdb \x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcZI\xb3\xdb6\x12\xbe\xbf_\xd1\xe59\xe4b\xe9ų\xa4\xa6t\xb3\xe5Iի\x89\xedW\x96\xe7\xdd!\xb2)\"&\x01\x0e\x16)\x9a忧\x1a\v\t\x92\xd0\x1a'<\xb8\xfc\xb04zC\xf7\xd7\r-\x16\x8b\a\xd6\xf1\x17T\x9aK\xb1\x02\xd6q\xfcŠ\xa0\xbf\xf4\xf2\xeb\xdf\xf5\x92\xcb\xc7\xfd\x9b\x87\xaf\\\x94+X[md\xfb\x19\xb5\xb4\xaa\xc0\xf7Xq\xc1\r\x97\xe2\xa1E\xc3Jf\xd8\xea\x01\x80\t!\r\xa3aM\x7f\x02\x14R\x18%\x9b\x06\xd5b\x87b\xf9\xd5nqkyS\xa2r\xc4\xe3\xd1\xfb\xef\x97o~X\xfe\xed\x01@\xb0\x16W@\xf4l\xd7HV\xea\xe5\x1e\x1bTr\xc9\xe5\x83\xee\xb0 \xb2;%m\xb7\x82a\xc2o\vGzv\xdf3\xc3\xfe\xe5(\xb8\xc1\x86k\xf3\xcf\xc9\xc4O\\\x1b7\xd95V\xb1ft\xaa\x1b\u05f5T\xe6\xe3@y\x01\xa5\xf5\x13\\\xecl\xc3T\xba\xe5\x01@\x17\xb2\xc3\x15\xb8\x1d\x1d+\x90Ƃ\x88\x8e\xc2\x02XY:\xa5\xb1\xe6YqaP\xadec[1\xd0G](\xde\x19\xa7\x94\x81SІ\x19\xabAۢ\x06\xa6\xe1#\x1e\x1e\x9fij\x92;\x85\xda\xf3\n\xf0\xb3\x96♙z\x05K\xbf|\xd9\xd5Lc\x98\xf5zݸ\x890d\x8eĭ6\x8a\x8b]\xee\xfc/\xbcE(\xadr\xf6$\x99\v\x04Ss\x9d2v`\x9a\x98S\x06˓l\xb8y\"\xa6\rk\xbb)?\xc9V\xcfP\xc9\f\xe6\xd8Y˶k\xd0`\tۣ\xc1(D%U\xcb\xcc\n\xb80?\xfc\xf5\xb4&\x82\xaa\x96n\xeb{)\xc6jyG\xa3\x90\f{N\xc8B;TY\xddHÚ\xdf\u0088!\x02\xef\x92\xfd\x9e\x13O7\x1d\xbf\xc8ʓ(\x14\xb6(\xeec\x88\x0f\xbb\xe7ܤ\xa4\xd3\xd9Nq\xa9\xb89\xae\xe0\xcd\xf7ײI\xb7\x02d\x05\xa6FxNJ\xaf\xb6\x83\x8d\x91\x8a\xed\x10~\x92\x85\xf7\xb1C\x8d*\xf8\xd8\xd6/ѵ\xb4M\t\xdbh\x18\x00m\xa4\xca:[\x87\xc5\xd2\xef\nt#ىǍ\xcf\xfc\xc6w\xa1PȲw!Fɥ[\xc1\xa5\xc8_\x88\xb7;\xbc\xea2\xa4\xda\x14\xb2\xc4^u\x98r\xc45tJ\x16\xa8\xf5\x99\xebI\xdbG<|\x1c\x06fj\xf1+\xf6\x7ffMW\xb37>\x18\x165\xb6l\x15v\xc8\x0e\xc5\xdb秗\xbflF\xc3p2\xb4\xb1\xc2h\x8ai\xc4z\xa7\xa4\x91\x85l`\x8b\xe6\x80(\\x\x85V\xeeQQ\x90\xdeq\xa1\x81\x89\xb2\xa7\t\xe9\x82!Ր\xeb;z4\xeb'\x83;\xc9\x0eUjvre\x1a3<\xc6x\xff%i1\x19\x9d\b\xf1\xbf\xc5h\x0e\x80\xe4\xf6\xbb\xa0\xa4\xfc\x88^\xaa\x90\x02\xb0\f\xaa\xf2v\xe3\x1a\x14v\n5]/\xe7U\xb2\x02&@n\x7f\xc6\xc2,'\xa47\xa8\x88L\xbc\x0f\x85\x14{T\x06\x14\x16r'\xf8\x7fz\xda\x1a\x8ct\x876̠6\xeeB*\xc1\x1aس\xc6\xe2\xeb\x89\xf6\xe8k\xd9\x11\x14ҙ`EB\xcfm\xd0S>>H\x85\xc0E%WP\x1b\xd3\xe9\xd5\xe3㎛\b\x16\nٶVps|t\xc6\xe0[k\xa4ҏ%\xee\xb1y\xd4|\xb7`\xaa\xa8\xb9\xc1\xc2X\x85\x8f\xac\xe3\v'\x88p\x80aٖ\x7fR\x01^\xe8ѱ3/\xf4\x9fK\xf47\x98\x87\xf2?]\t\x16Hy\x11\a+\xd0\x10\xa9\xee\xf3?6_ r\xe2-\xe5\x8d2,\x9d\xe9%ڇ\xb4\xc9E\x85\xca䀹l\x1dM\x14e'\xb90\ue3e2\xe1(\fh\xbbm\xb9!7\xf8\xb7Em\xc8tS\xb2k\a\xa8`\x8b`;\n\x05\xe5t\xc1\x93\x805k\xb1Y3\x8d\x7f\xb0\xad\xc8*zAF\xb8\xcaZ)L\x9c.\xf6\xeaM&\"\xd2;a\xda!|l:,Ȧ\xa4V\xda\xc4+\x1er\t\xc5\x00\x96\xac\x1ck'\x7f\xed\xe9˦\x90\xe9\xa2K\xaeF\u07fb\x1c\xa1ȫH\xe2wLu!35\xe3̔~C\x90\x0f{\x14vRs#Ց\b\xfb\xd48u\x83\x93\x16\xa1\xaf`\xa2\xc0\xe6\x1e\xf1\xd6n'pQ\x92Ʊwc\n@\x9e\xaacT\x8a\x9d\xa4\x8b\x95\x18\x02\x9e\f\xad \xaf\xd6h\xf2b\x8aL*\xe3\x02\x06\xd0\v)\xb8\x9d\x8a\xba\x95\xb2A6\xd5`\xa1\xf9F\xb0N\xd7\xd2\\\x10\xf8\xa9\x82\xb8\xf2˱C:|\xbdyzM\xff\xc4q\xf2\xa0=/C\x88\xa7[Fh+o\xb6`\xe7\xf5\xe6\tt\xd8>7\x92\xb0Mö\r\xae\xc0(;\x17\xec\xb4\xc3:\xee\x15ߣ\xca\xcdLo\x8e[\x18\xbd\xd0o\x03\xab\x1d\xa8vC/T\x90`\x94r-\x85A\x91\xb3\xd1Y\xaf\xa2/J\xban\x98\xce\xf2<\xe1l\x93\xae\xcf]\x93H\x10\n\xb7\xc2\xd4,\xcf\x17\xf8\xa4\xeb\xe4\x186\xf1\x1e\x9b\xc1\x81\x9b\xfa.\x89\xfc\x05\xbdZ\xa0dyV\x9ep߽8\xb2:#\xcc\xf3\xcb\xda\xc9{I2J7\xf7H\xb6\x1f\x19\xfd\n\xd9\xc6^\x92\x93n\xc2\xe5)\xe1$E\x01\nfX\x82\xedn睂\x0eWX\xcey^\x8c앙\x1e\v}\"\x92\xcc2\x13\x04\xd0\xf9\x81`\xe5Z\x8a\x8a\xef\xe6g\xa7e\xfe\xb9k{V\xb4Y\xc6K\x8e$\x8dS\x82#N\x16\x0e\xe1.b\xf6#lX\xf1\x9dU\xa7\xa2Qű)g\x00\xe6b\x00\xba\xa0\x0f\xc7\xc4=y\xa4\x97,\xe6\xef\x10R\x13d\xef\xbd$\x8dR>\xfd\xcde\x00\n\xdd\x03E\xae\xe1\xd5+\x90\n^\xf9^ѫ\xd7~\xb7\xe5\x8dYp\x01\x95N\x8f9\xf0\xa6\x89\aݔD\xfb\xaa\x82j:i/e\x97\xac\x1a>MhL\xb4a\xa8\xfet\x1a0\x12\x0e\x8c'Ⱦ?]\xbf\xce\xd0\xddbE0P\xa1\xb1JP\"F\xa5\b\x19iGR\xdaL&:#i\xc7\x14\nse\x16\xcd\xca\xf9<\xa20\x91ғ\x1fB\x9b\x8by\x85Un4@\x1e\xd7\x1b EHq\xc2\xfe\x84\xa9=\xb4\x1f\xecϬ\x89\xd6O,^\x11xu\x83\n\x8b\xe4\x8c\x18\xa1)\x9e\x85@\xc6t\xe0\xee\xaaC\x85\x148?\xce9X)\x81Ae\xc9\xd5\xdcaW\x90c=\xb4\xedU\xf3\xf4\xfe\x8c0\xb3\xd5\xe7\xb8?cm\x9d`\xa0\v\xb6\x9e\xc2%\xe7\xb3\xf4\xffi\xf2N#~F\xf4ܥ>ǡ\xab\xd1~\xdc\\\xc3a\xb24rX\xf1\x06A\x1f\xb5\xc1v̭/\xfd\xbc\xe9\xef`\xa8\xef\x00\xdfsC6c\x12\x91W\xa9\xf8\x8e\xd3}\x17\xfd\xccP\x0e\x04'\r}3\x97K\x1d\x18\xc8:k\x9f\xaf\x9d\x7f\x0f\xe4(\xa1\xf8\xc3\to0Q:\xc4\xdaϗ!\xf8gR\xc7E\x85<\xbf\xac\xaf2\x0f\x1d\x9c\x01\x134|\xa8yQ\x8f}\x89\xcf\xd3:\x80a_\xd1U\x7f7\xb0\x99G\x11\x8b|-8Y3\r\xfe\x93\xe9\xf4\x0eM\xa7Ɔ\xce\xce>\xbf\xac\xaf\xaa\x97]+ﺊ\xd9?%\x04-\xc7\xe0\x1a\x1e\x18duW\xcd̊\x02;\x83\xe5\xbb\xe3GY^r\xfa\xb7\xa3\xc5Ĉ\xb8\xa6\x99\x991\xb5ko\"\x05\xb6\xdb\xf2ud\xb7o\xc1\xdesM\xdfN\x89\xb8f\x9c*\x93|=/a}\xf4;\xcd4\xc0\x17rp\xd7L\xfaΧh\xda\xe6\x12?]\xcf١3\n\xb1\xeb_2\x83\v\xda\x7f\x1f\xce\xcb7\v\xfc\vLڼ\xbe\xabs0'3\xd7\x1d\x8b\xb9\xd8u\xd5\xe3\xd3ONc\x03\xb9^_\x9e\x1a\x96\x80{\x14 \x05T\x8c7\x84\x1e\x1d\xc9L\x00;O%`(\xff\xce\x17\xbb\x84\x11*d۵\x97-\x99Q\xc2<\x9a\xfd\x9e\xc6싘Ϩm\x93\xc1r\xbfc\x11\xe3\x8f\xf4\xfd*\x9d-b\xce7T\x18a\"剄\xb8q*h]\xad\xa4le3}\x1d\xbb\xd47\x9a,\x87Z6\xc1\xa9\x85m\xb7\xa8\x88[\xf7F\a\x02\x0f\x04L\x8b\x9a\x89]\x16\t\xc57&\x84\x86is\n,\xe6\x1e\xf9\xa6\x92\xa5\x8fr\xc3ע\xd6lw)X\x7f\xf0\xab<\n\r[\x80m\xa9@\x19k\xfd;\x1dr\xc8M\x91X\\N\x177%\x89ы\xd7͜|\xda\\\xc1˧\r\x1d\xf2i\xf3[yAa\xdb\\ׂ*\x95\xccpÅ\xfd%3~\u0894\x87y\xe88[ę\xfa\x82\xa0\xcf\xcc\xd4=H\xa6Z\x85\xf6̰|@\x9d[\xa4\x98\xf8\xad \xbd\xeb\xeb^b\x8f\xd6\xe4 \f^\x13\x0eNi\xfe#\x1e2\xa31\xe5f\xa6\x9eC\x1e\xcfL\xcd~\x9d\x91N\xfa\xd6y.\\ƹ,\xcd\xfe\a\x10\x99\xb9\x1f]\x82\xbbIρ\xbf\xbb\x8a\xf8\u0604\x1f\xe2\x9b\xfb=\xc3,ʍ\x9b\x81TR$\x16\xcb\x10N\xf6\xf7u\x8c\xa3\xb4\x84/5\xd7\xf1\xd9 6BJ\xae\xbb\x86\x1d{Y.\xa5\x8d>nM\x9f\x83\xe7Nr\xbe\xdf\xde\xff\x8c$\xdf+=\x1f\x95\xe1Bdv\xf3\xf2t\xca\xf9\x16'\x9c\xc9yC\x8b\xe1ʚ\xff\xe9}\xbc\x8a\xbcDaxœ'\xf8\xa1XsO:9]N\x9f\xb2n\xab/G?.\xba\xab\xde\x1eQ\xb8\x80D\xc3o\x9drxoC\xc1\x80B\x90{\xf4]O\x7f\xe6\xf1\xba\xcf\xe8̄֎O\xfe\xb9\"V\n\x827\x0e\x1e\xdd\x0e-\xc7\x02\xfd\x91\xa82\xebU\xb3A\xc7y\x99\xd0\x0e\x8d\xfat\xc4n\xfb\x9f\x02\xac\xe0\xbf\xff\x7f\xf85\x00\x00\xff\xff \xad\x88\xba\xac(\x00\x00"), -} - -var CRDs = crds() - -func crds() []*apiextv1.CustomResourceDefinition { - apiextinstall.Install(scheme.Scheme) - decode := scheme.Codecs.UniversalDeserializer().Decode - var objs []*apiextv1.CustomResourceDefinition - for _, crd := range rawCRDs { - gzr, err := gzip.NewReader(bytes.NewReader(crd)) - if err != nil { - panic(err) - } - bytes, err := io.ReadAll(gzr) - if err != nil { - panic(err) - } - gzr.Close() - - obj, _, err := decode(bytes, nil, nil) - if err != nil { - panic(err) - } - objs = append(objs, obj.(*apiextv1.CustomResourceDefinition)) - } - return objs -} diff --git a/config/crd/v2alpha1/crds/doc.go b/config/crd/v2alpha1/crds/doc.go deleted file mode 100644 index 9eed410f6..000000000 --- a/config/crd/v2alpha1/crds/doc.go +++ /dev/null @@ -1,4 +0,0 @@ -// Package crds embeds the controller-tools generated CRD manifests -package crds - -//go:generate go run ../../../../hack/crd-gen/v1/main.go diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index ea669c709..f8f27a521 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -4,6 +4,16 @@ kind: ClusterRole metadata: name: velero-perms rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - create + - delete + - get + - list - apiGroups: - "" resources: diff --git a/go.mod b/go.mod index 3aa6ea020..3e6bd840c 100644 --- a/go.mod +++ b/go.mod @@ -50,7 +50,7 @@ require ( golang.org/x/text v0.37.0 google.golang.org/api v0.283.0 google.golang.org/grpc v1.82.1 - google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af + google.golang.org/protobuf v1.36.12 k8s.io/api v0.36.0 k8s.io/apiextensions-apiserver v0.36.0 k8s.io/apimachinery v0.36.0 diff --git a/go.sum b/go.sum index 63cf28c46..55741d1c5 100644 --- a/go.sum +++ b/go.sum @@ -566,8 +566,8 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1: google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= -google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= -google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/hack/crd-gen/v1/main.go b/hack/crd-gen/v1/main.go deleted file mode 100644 index 5f45b04e0..000000000 --- a/hack/crd-gen/v1/main.go +++ /dev/null @@ -1,134 +0,0 @@ -/* -Copyright the Velero contributors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// This code embeds the CRD manifests in ../bases in ../crds/crds.go - -package main - -import ( - "bytes" - "compress/gzip" - "fmt" - "io" - "log" - "os" - "text/template" -) - -// This is relative to config/crd/crds -const goHeaderFile = "../../../../hack/boilerplate.go.txt" - -const tpl = `{{.GoHeader}} -// Code generated by crds_generate.go; DO NOT EDIT. - -package crds - -import ( - "bytes" - "compress/gzip" - "io" - - apiextinstall "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install" - apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "k8s.io/client-go/kubernetes/scheme" -) - -var rawCRDs = [][]byte{ -{{- range .RawCRDs }} - []byte({{ . }}), -{{- end }} -} - -var CRDs = crds() - -func crds() []*apiextv1.CustomResourceDefinition { - apiextinstall.Install(scheme.Scheme) - decode := scheme.Codecs.UniversalDeserializer().Decode - var objs []*apiextv1.CustomResourceDefinition - for _, crd := range rawCRDs { - gzr, err := gzip.NewReader(bytes.NewReader(crd)) - if err != nil { - panic(err) - } - bytes, err := io.ReadAll(gzr) - if err != nil { - panic(err) - } - gzr.Close() - - obj, _, err := decode(bytes, nil, nil) - if err != nil { - panic(err) - } - objs = append(objs, obj.(*apiextv1.CustomResourceDefinition)) - } - return objs -} -` - -type templateData struct { - GoHeader string - RawCRDs []string -} - -func main() { - headerBytes, err := os.ReadFile(goHeaderFile) - if err != nil { - log.Fatalln(err) - } - - data := templateData{ - GoHeader: string(headerBytes), - } - - // This is relative to config/crd/crds - manifests, err := os.ReadDir("../bases") - if err != nil { - log.Fatalln(err) - } - - for _, crd := range manifests { - file, err := os.Open("../bases/" + crd.Name()) - if err != nil { - log.Fatalln(err) - } - - // gzip compress manifest - var buf bytes.Buffer - gzw := gzip.NewWriter(&buf) - if _, err := io.Copy(gzw, file); err != nil { - log.Fatalln(err) - } - file.Close() - gzw.Close() - - data.RawCRDs = append(data.RawCRDs, fmt.Sprintf("%q", buf.Bytes())) - } - - t, err := template.New("crd").Parse(tpl) - if err != nil { - log.Fatalln(err) - } - - out, err := os.Create("crds.go") - if err != nil { - log.Fatalln(err) - } - - if err := t.Execute(out, data); err != nil { - log.Fatalln(err) - } -} diff --git a/hack/update-3generated-crd-code.sh b/hack/update-3generated-crd-code.sh index 720639a40..cc7b92eda 100755 --- a/hack/update-3generated-crd-code.sh +++ b/hack/update-3generated-crd-code.sh @@ -55,6 +55,6 @@ controller-gen \ paths=./pkg/controller/... \ rbac:roleName=velero-perms -go generate ./config/crd/v1/crds - -go generate ./config/crd/v2alpha1/crds +# The CRD manifests above are embedded directly into the binary via +# go:embed (see config/crd/v1/crds.go and config/crd/v2alpha1/crds.go), +# so no further code generation step is required. diff --git a/hack/verify-generated-crd-code.sh b/hack/verify-generated-crd-code.sh deleted file mode 100755 index 1d9f23cab..000000000 --- a/hack/verify-generated-crd-code.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -e -# -# Copyright the Velero contributors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -HACK_DIR=$(dirname "${BASH_SOURCE}") - -${HACK_DIR}/update-3generated-crd-code.sh - -# ensure no changes to generated CRDs -if ! git diff --exit-code config/crd/v1/crds/crds.go config/crd/v2alpha1/crds/crds.go &> /dev/null; then - # revert changes to state before running CRD generation to stay consistent - # with code-generator `--verify-only` option which discards generated changes - git checkout config/crd - - echo "CRD verification - failed! Generated CRDs are out-of-date, please run 'make update' and 'git add' the generated file(s)." - exit 1 -fi diff --git a/internal/delete/actions/csi/volumesnapshotcontent_action.go b/internal/delete/actions/csi/volumesnapshotcontent_action.go index c57a0eb1a..4b7895341 100644 --- a/internal/delete/actions/csi/volumesnapshotcontent_action.go +++ b/internal/delete/actions/csi/volumesnapshotcontent_action.go @@ -86,7 +86,7 @@ func (p *volumeSnapshotContentDeleteItemAction) Execute( // This handles legacy (pre-1.15) backups where the original VSC // with DeletionPolicy=Retain still exists in the cluster. originalVSCName := snapCont.Name - if cleaned := p.tryDeleteOriginalVSC(context.TODO(), originalVSCName); cleaned { + if cleaned := p.tryDeleteOriginalVSC(context.TODO(), originalVSCName, input.Backup.Name); cleaned { p.log.Infof("Successfully deleted original VolumeSnapshotContent %s from cluster, skipping temp VSC creation", originalVSCName) return nil } @@ -149,10 +149,11 @@ func (p *volumeSnapshotContentDeleteItemAction) Execute( // the cluster (legacy pre-1.15 backups). It patches the DeletionPolicy to // Delete so the CSI driver also removes the cloud snapshot, then deletes // the VSC object itself. -// Returns true if the original VSC was found and deletion was initiated. +// Returns true if the original VSC was found, carries the backup label, and deletion was initiated. func (p *volumeSnapshotContentDeleteItemAction) tryDeleteOriginalVSC( ctx context.Context, vscName string, + backupName string, ) bool { existing := new(snapshotv1api.VolumeSnapshotContent) if err := p.crClient.Get(ctx, crclient.ObjectKey{Name: vscName}, existing); err != nil { @@ -164,6 +165,15 @@ func (p *volumeSnapshotContentDeleteItemAction) tryDeleteOriginalVSC( return false } + if !kubeutil.HasBackupLabel(&existing.ObjectMeta, backupName) { + p.log.Warnf( + "Original VolumeSnapshotContent %s in cluster does not belong to backup %s, skipping direct deletion", + vscName, + backupName, + ) + return false + } + p.log.Debugf("Found original VolumeSnapshotContent %s in cluster (legacy backup), cleaning up directly", vscName) // Patch DeletionPolicy to Delete so the CSI driver removes the cloud snapshot diff --git a/internal/delete/actions/csi/volumesnapshotcontent_action_test.go b/internal/delete/actions/csi/volumesnapshotcontent_action_test.go index e8a0b5865..25cc69b82 100644 --- a/internal/delete/actions/csi/volumesnapshotcontent_action_test.go +++ b/internal/delete/actions/csi/volumesnapshotcontent_action_test.go @@ -122,7 +122,29 @@ func TestVSCExecute(t *testing.T) { backup: builder.ForBackup("velero", "backup").Result(), expectErr: false, preExistingVSC: &snapshotv1api.VolumeSnapshotContent{ - ObjectMeta: metav1.ObjectMeta{Name: "bar"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "bar", + Labels: map[string]string{ + velerov1api.BackupNameLabel: "backup", + }, + }, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + DeletionPolicy: snapshotv1api.VolumeSnapshotContentRetain, + Driver: "disk.csi.azure.com", + Source: snapshotv1api.VolumeSnapshotContentSource{SnapshotHandle: stringPtr("snap-123")}, + VolumeSnapshotRef: corev1api.ObjectReference{Name: "vs-1", Namespace: "default"}, + }, + }, + }, + { + name: "Original VSC exists in cluster without backup label, falls through to temp VSC flow", + vsc: builder.ForVolumeSnapshotContent("bar").ObjectMeta(builder.WithLabelsMap(map[string]string{velerov1api.BackupNameLabel: "backup"})).Status(&snapshotv1api.VolumeSnapshotContentStatus{SnapshotHandle: &snapshotHandleStr}).Result(), + backup: builder.ForBackup("velero", "backup").Result(), + expectErr: false, + preExistingVSC: &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "bar", + }, Spec: snapshotv1api.VolumeSnapshotContentSpec{ DeletionPolicy: snapshotv1api.VolumeSnapshotContentRetain, Driver: "disk.csi.azure.com", @@ -200,22 +222,51 @@ func TestNewVolumeSnapshotContentDeleteItemAction(t *testing.T) { func TestTryDeleteOriginalVSC(t *testing.T) { tests := []struct { - name string - vscName string - existing *snapshotv1api.VolumeSnapshotContent - createIt bool - expectRet bool + name string + vscName string + backupName string + existing *snapshotv1api.VolumeSnapshotContent + createIt bool + expectRet bool }{ { - name: "VSC not found in cluster, returns false", - vscName: "not-found", + name: "VSC not found in cluster, returns false", + vscName: "not-found", + backupName: "test-backup", + expectRet: false, + }, + { + name: "VSC found in cluster without backup label, returns false", + vscName: "unlabeled-vsc", + backupName: "test-backup", + existing: &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{Name: "unlabeled-vsc"}, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + DeletionPolicy: snapshotv1api.VolumeSnapshotContentRetain, + Driver: "disk.csi.azure.com", + Source: snapshotv1api.VolumeSnapshotContentSource{ + SnapshotHandle: stringPtr("snap-123"), + }, + VolumeSnapshotRef: corev1api.ObjectReference{ + Name: "vs-1", + Namespace: "default", + }, + }, + }, + createIt: true, expectRet: false, }, { - name: "VSC found with Retain policy, patches and deletes", - vscName: "legacy-vsc", + name: "VSC found with Retain policy and matching backup label, patches and deletes", + vscName: "legacy-vsc", + backupName: "test-backup", existing: &snapshotv1api.VolumeSnapshotContent{ - ObjectMeta: metav1.ObjectMeta{Name: "legacy-vsc"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "legacy-vsc", + Labels: map[string]string{ + velerov1api.BackupNameLabel: "test-backup", + }, + }, Spec: snapshotv1api.VolumeSnapshotContentSpec{ DeletionPolicy: snapshotv1api.VolumeSnapshotContentRetain, Driver: "disk.csi.azure.com", @@ -232,10 +283,16 @@ func TestTryDeleteOriginalVSC(t *testing.T) { expectRet: true, }, { - name: "VSC found with Delete policy already, just deletes", - vscName: "already-delete-vsc", + name: "VSC found with Delete policy and matching backup label, just deletes", + vscName: "already-delete-vsc", + backupName: "test-backup", existing: &snapshotv1api.VolumeSnapshotContent{ - ObjectMeta: metav1.ObjectMeta{Name: "already-delete-vsc"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "already-delete-vsc", + Labels: map[string]string{ + velerov1api.BackupNameLabel: "test-backup", + }, + }, Spec: snapshotv1api.VolumeSnapshotContentSpec{ DeletionPolicy: snapshotv1api.VolumeSnapshotContentDelete, Driver: "disk.csi.azure.com", @@ -266,7 +323,7 @@ func TestTryDeleteOriginalVSC(t *testing.T) { require.NoError(t, crClient.Create(t.Context(), test.existing)) } - result := p.tryDeleteOriginalVSC(t.Context(), test.vscName) + result := p.tryDeleteOriginalVSC(t.Context(), test.vscName, test.backupName) require.Equal(t, test.expectRet, result) // If cleanup succeeded, verify the VSC is gone @@ -289,13 +346,18 @@ func TestTryDeleteOriginalVSC(t *testing.T) { log: logrus.StandardLogger(), crClient: errClient, } - require.False(t, p.tryDeleteOriginalVSC(t.Context(), "some-vsc")) + require.False(t, p.tryDeleteOriginalVSC(t.Context(), "some-vsc", "test-backup")) }) t.Run("Patch fails, returns false", func(t *testing.T) { realClient := velerotest.NewFakeControllerRuntimeClient(t) vsc := &snapshotv1api.VolumeSnapshotContent{ - ObjectMeta: metav1.ObjectMeta{Name: "patch-fail-vsc"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "patch-fail-vsc", + Labels: map[string]string{ + velerov1api.BackupNameLabel: "test-backup", + }, + }, Spec: snapshotv1api.VolumeSnapshotContentSpec{ DeletionPolicy: snapshotv1api.VolumeSnapshotContentRetain, Driver: "disk.csi.azure.com", @@ -313,13 +375,18 @@ func TestTryDeleteOriginalVSC(t *testing.T) { log: logrus.StandardLogger(), crClient: errClient, } - require.False(t, p.tryDeleteOriginalVSC(t.Context(), "patch-fail-vsc")) + require.False(t, p.tryDeleteOriginalVSC(t.Context(), "patch-fail-vsc", "test-backup")) }) t.Run("Delete fails, returns false", func(t *testing.T) { realClient := velerotest.NewFakeControllerRuntimeClient(t) vsc := &snapshotv1api.VolumeSnapshotContent{ - ObjectMeta: metav1.ObjectMeta{Name: "delete-fail-vsc"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "delete-fail-vsc", + Labels: map[string]string{ + velerov1api.BackupNameLabel: "test-backup", + }, + }, Spec: snapshotv1api.VolumeSnapshotContentSpec{ DeletionPolicy: snapshotv1api.VolumeSnapshotContentDelete, Driver: "disk.csi.azure.com", @@ -337,7 +404,7 @@ func TestTryDeleteOriginalVSC(t *testing.T) { log: logrus.StandardLogger(), crClient: errClient, } - require.False(t, p.tryDeleteOriginalVSC(t.Context(), "delete-fail-vsc")) + require.False(t, p.tryDeleteOriginalVSC(t.Context(), "delete-fail-vsc", "test-backup")) }) } diff --git a/internal/hook/item_hook_handler.go b/internal/hook/item_hook_handler.go index bed48c5ea..a2a58fc4a 100644 --- a/internal/hook/item_hook_handler.go +++ b/internal/hook/item_hook_handler.go @@ -365,6 +365,12 @@ func getPodExecHookFromAnnotations(annotations map[string]string, phase HookPhas func parseStringToCommand(commandValue string) []string { var command []string + // An empty command means the container image's own entrypoint should be used. + // Callers that require a command already return early; getInitContainerFromAnnotation + // deliberately allows this case, so return nil rather than indexing an empty string. + if commandValue == "" { + return nil + } // check for json array if commandValue[0] == '[' { if err := json.Unmarshal([]byte(commandValue), &command); err != nil { @@ -419,7 +425,7 @@ func getInitContainerFromAnnotation(podName string, annotations map[string]strin return nil } if command == "" { - log.Infof("RestoreHook init container for pod %s is using container's default entrypoint", podName, containerImage) + log.Infof("RestoreHook init container for pod %s is using the default entrypoint of image %s", podName, containerImage) } if containerName == "" { uid, err := uuid.NewRandom() diff --git a/internal/hook/item_hook_handler_test.go b/internal/hook/item_hook_handler_test.go index 1f2df9469..792bcd66e 100644 --- a/internal/hook/item_hook_handler_test.go +++ b/internal/hook/item_hook_handler_test.go @@ -1287,6 +1287,25 @@ func TestGetInitContainerFromAnnotations(t *testing.T) { podRestoreHookInitContainerCommandAnnotationKey: "[foobarbaz", }, }, + { + name: "should use the image's default entrypoint when the command annotation is empty", + expectNil: false, + expected: builder.ForContainer("restore-init1", "busy-box").Result(), + inputAnnotations: map[string]string{ + podRestoreHookInitContainerImageAnnotationKey: "busy-box", + podRestoreHookInitContainerNameAnnotationKey: "restore-init", + podRestoreHookInitContainerCommandAnnotationKey: "", + }, + }, + { + name: "should use the image's default entrypoint when the command annotation is missing", + expectNil: false, + expected: builder.ForContainer("restore-init1", "busy-box").Result(), + inputAnnotations: map[string]string{ + podRestoreHookInitContainerImageAnnotationKey: "busy-box", + podRestoreHookInitContainerNameAnnotationKey: "restore-init", + }, + }, } for _, tc := range testCases { diff --git a/internal/volume/snapshotlocation.go b/internal/volume/snapshotlocation.go index 594fbf3a5..f8adab7fd 100644 --- a/internal/volume/snapshotlocation.go +++ b/internal/volume/snapshotlocation.go @@ -29,6 +29,10 @@ func UpdateVolumeSnapshotLocationWithCredentialConfig(location *velerov1api.Volu if location.Spec.Config == nil { location.Spec.Config = make(map[string]string) } + + // Delete any user-provided credentialsFile to prevent path traversal vulnerabilities + delete(location.Spec.Config, "credentialsFile") + // If the VSL specifies a credential, fetch its path on disk and pass to // plugin via the config. if location.Spec.Credential != nil && credentialStore != nil { diff --git a/internal/volume/volumes_information.go b/internal/volume/volumes_information.go index ad8993447..69214ef45 100644 --- a/internal/volume/volumes_information.go +++ b/internal/volume/volumes_information.go @@ -36,6 +36,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/features" "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/kuberesource" + "github.com/vmware-tanzu/velero/pkg/util/stringptr" ) type Method string @@ -494,7 +495,8 @@ func (v *BackupVolumesInformation) generateVolumeInfoForCSIVolumeSnapshot() { tmpVolumeInfos = append(tmpVolumeInfos, volumeInfo) } else { - v.logger.Warnf("cannot find info for PVC %s/%s", volumeSnapshot.Namespace, volumeSnapshot.Spec.Source.PersistentVolumeClaimName) + v.logger.Warnf("cannot find info for PVC %s/%s", volumeSnapshot.Namespace, + stringptr.GetString(volumeSnapshot.Spec.Source.PersistentVolumeClaimName)) continue } } diff --git a/pkg/apis/velero/shared/constants.go b/pkg/apis/velero/shared/constants.go index 12f8b51ee..d497d59b1 100644 --- a/pkg/apis/velero/shared/constants.go +++ b/pkg/apis/velero/shared/constants.go @@ -17,6 +17,6 @@ limitations under the License. package shared const ( - DataUploadParentSnapshotNone = "none" - DataUploadParentSnapshotAuto = "auto" + ParentSnapshotNone = "none" + ParentSnapshotAuto = "auto" ) diff --git a/pkg/apis/velero/v1/download_request_types.go b/pkg/apis/velero/v1/download_request_types.go index 5e93862e6..1eb9d5474 100644 --- a/pkg/apis/velero/v1/download_request_types.go +++ b/pkg/apis/velero/v1/download_request_types.go @@ -56,7 +56,7 @@ type DownloadTarget struct { } // DownloadRequestPhase represents the lifecycle phase of a DownloadRequest. -// +kubebuilder:validation:Enum=New;Processed +// +kubebuilder:validation:Enum=New;Processed;Failed type DownloadRequestPhase string const ( @@ -64,18 +64,30 @@ const ( // DownloadRequestController yet. DownloadRequestPhaseNew DownloadRequestPhase = "New" - // DownloadRequestPhaseProcessed means the DownloadRequest has been processed by the - // DownloadRequestController. + // DownloadRequestPhaseProcessed means the DownloadRequestController has signed a URL + // into Status.DownloadURL. The controller signs the key by convention and does not + // check that the object is present, so this phase does not imply the file exists. DownloadRequestPhaseProcessed DownloadRequestPhase = "Processed" + + // DownloadRequestPhaseFailed means the controller will not sign a URL for this request + // and no retry will change that. Status.Message carries the reason. A caller waiting on + // Status.DownloadURL should stop when it sees this phase rather than poll until its own + // timeout, which would report a storage problem that is not the cause. + DownloadRequestPhaseFailed DownloadRequestPhase = "Failed" ) // DownloadRequestStatus is the current status of a DownloadRequest. type DownloadRequestStatus struct { - // Phase is the current state of the DownloadRequest. + // Phase is the current state of the DownloadRequest. Processed means a URL has been + // signed into DownloadURL. It does not mean the target object exists in object storage, + // so a request whose target never produced a file still reaches Processed and the URL + // returns 404. Callers should check that the backup or restore is in a phase that + // produces the target before relying on the download. // +optional Phase DownloadRequestPhase `json:"phase,omitempty"` - // DownloadURL contains the pre-signed URL for the target file. + // DownloadURL contains the pre-signed URL for the target file. It is signed for a fixed + // lifetime and expires at Expiration, so it should be used promptly and not cached. // +optional DownloadURL string `json:"downloadURL,omitempty"` @@ -83,6 +95,10 @@ type DownloadRequestStatus struct { // +optional // +nullable Expiration *metav1.Time `json:"expiration,omitempty"` + + // Message explains a Failed phase. It is empty in every other phase. + // +optional + Message string `json:"message,omitempty"` } // TODO(2.0) After converting all resources to use the runtime-controller client, @@ -93,6 +109,10 @@ type DownloadRequestStatus struct { // +kubebuilder:object:generate=true // +kubebuilder:storageversion // +kubebuilder:resource:shortName=dreq +// +kubebuilder:printcolumn:name="Target Kind",type="string",JSONPath=".spec.target.kind",description="The type of file to download" +// +kubebuilder:printcolumn:name="Target Name",type="string",JSONPath=".spec.target.name",description="The name of the resource the file is associated with" +// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.phase",description="The status of the download request" +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" // DownloadRequest is a request to download an artifact from backup object storage, such as a backup // log file. diff --git a/pkg/apis/velero/v1/pod_volume_backup_types.go b/pkg/apis/velero/v1/pod_volume_backup_types.go index 5ad725df1..566ba3b29 100644 --- a/pkg/apis/velero/v1/pod_volume_backup_types.go +++ b/pkg/apis/velero/v1/pod_volume_backup_types.go @@ -61,6 +61,12 @@ type PodVolumeBackupSpec struct { // Cancel indicates request to cancel the ongoing PodVolumeBackup. It can be set // when the PodVolumeBackup is in InProgress phase Cancel bool `json:"cancel,omitempty"` + + // ParentSnapshot specifies the parent snapshot that current backup is based on. + // If its value is "" or "auto", the data mover finds the recent backup of the same volume as parent. + // If its value is "none", the data mover will do a full backup + // If its value is a specific snapshotID, the data mover finds the specific snapshot as parent. + ParentSnapshot string `json:"parentSnapshot,omitempty"` } // PodVolumeBackupPhase represents the lifecycle phase of a PodVolumeBackup. diff --git a/pkg/apis/velero/v1/server_status_request_types.go b/pkg/apis/velero/v1/server_status_request_types.go index 98e15a0b5..26742e0a1 100644 --- a/pkg/apis/velero/v1/server_status_request_types.go +++ b/pkg/apis/velero/v1/server_status_request_types.go @@ -28,6 +28,10 @@ import ( // +kubebuilder:resource:shortName=ssr // +kubebuilder:object:generate=true // +kubebuilder:storageversion +// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.phase",description="The status of the server status request" +// +kubebuilder:printcolumn:name="Server Version",type="string",JSONPath=".status.serverVersion",description="The Velero server version" +// +kubebuilder:printcolumn:name="Processed",type="date",JSONPath=".status.processedTimestamp",description="The time the request was processed by the controller" +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" // ServerStatusRequest is a request to access current status information about // the Velero server. diff --git a/pkg/archive/extractor.go b/pkg/archive/extractor.go index aae9a7a85..15cacde22 100644 --- a/pkg/archive/extractor.go +++ b/pkg/archive/extractor.go @@ -32,14 +32,27 @@ import ( // Extractor unzips/extracts a backup tarball to a local // temp directory. type Extractor struct { - log logrus.FieldLogger - fs filesystem.Interface + log logrus.FieldLogger + fs filesystem.Interface + maxExtractionSize int64 + totalExtractedSize int64 +} + +var maxExtractionSize = int64(16) << 30 + +// SetMaxExtractionSize sets the maximum extraction size. It is normally called at server startup. +func SetMaxExtractionSize(size int64) { + if size > 0 { + maxExtractionSize = size + } } func NewExtractor(log logrus.FieldLogger, fs filesystem.Interface) *Extractor { return &Extractor{ - log: log, - fs: fs, + log: log, + fs: fs, + maxExtractionSize: maxExtractionSize, + totalExtractedSize: 0, } } @@ -96,6 +109,15 @@ func (e *Extractor) readBackup(tarRdr *tar.Reader) (string, error) { return "", err } + // Enforce maximum extraction size to prevent memory/storage exhaustion and zip bombs. + maxSize := e.maxExtractionSize + e.totalExtractedSize += header.Size + if e.totalExtractedSize > maxSize { + err := fmt.Errorf("decompressed backup exceeds maximum allowed size of %d bytes", maxSize) + e.log.Infof("error checking extracted size: %v", err) + return "", err + } + target, err := sanitizeArchivePath(dir, header.Name) if err != nil { e.log.Infof("error sanitizing archive path: %s", err.Error()) diff --git a/pkg/archive/extractor_test.go b/pkg/archive/extractor_test.go index a4daf02ca..d87f787c3 100644 --- a/pkg/archive/extractor_test.go +++ b/pkg/archive/extractor_test.go @@ -20,6 +20,7 @@ import ( "archive/tar" "bytes" "compress/gzip" + "fmt" "io" "os" "testing" @@ -113,6 +114,66 @@ func TestUnzipAndExtractBackupRejectsPathTraversal(t *testing.T) { require.Contains(t, err.Error(), "invalid archive path") } +func TestUnzipAndExtractBackupRejectsLargeFile(t *testing.T) { + SetMaxExtractionSize(1024) + defer SetMaxExtractionSize(16 * 1024 * 1024 * 1024) + ext := NewExtractor(test.NewLogger(), test.NewFakeFileSystem()) + + var buf bytes.Buffer + gzw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gzw) + + data := make([]byte, 2048) // 2KB data + err := tw.WriteHeader(&tar.Header{ + Name: "large.txt", + Mode: 0600, + Typeflag: tar.TypeReg, + Size: int64(len(data)), + }) + require.NoError(t, err) + + _, err = tw.Write(data) + require.NoError(t, err) + require.NoError(t, tw.Close()) + require.NoError(t, gzw.Close()) + + _, err = ext.UnzipAndExtractBackup(&buf) + require.Error(t, err) + require.Contains(t, err.Error(), "decompressed backup exceeds maximum allowed size") +} + +func TestUnzipAndExtractBackupRejectsManySmallFiles(t *testing.T) { + SetMaxExtractionSize(1024) + defer SetMaxExtractionSize(16 * 1024 * 1024 * 1024) + ext := NewExtractor(test.NewLogger(), test.NewFakeFileSystem()) + + var buf bytes.Buffer + gzw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gzw) + + // Create 100 files of 20 bytes each (total 2000 bytes, exceeding the 1024 byte limit) + for i := 0; i < 100; i++ { + data := make([]byte, 20) + err := tw.WriteHeader(&tar.Header{ + Name: fmt.Sprintf("small_%d.txt", i), + Mode: 0600, + Typeflag: tar.TypeReg, + Size: int64(len(data)), + }) + require.NoError(t, err) + + _, err = tw.Write(data) + require.NoError(t, err) + } + + require.NoError(t, tw.Close()) + require.NoError(t, gzw.Close()) + + _, err := ext.UnzipAndExtractBackup(&buf) + require.Error(t, err) + require.Contains(t, err.Error(), "decompressed backup exceeds maximum allowed size") +} + func createArchive(files []string, fs filesystem.Interface) (string, error) { outName := "output.tar.gz" out, err := fs.Create(outName) diff --git a/pkg/backup/actions/csi/pvc_action.go b/pkg/backup/actions/csi/pvc_action.go index b9debe031..817844bfa 100644 --- a/pkg/backup/actions/csi/pvc_action.go +++ b/pkg/backup/actions/csi/pvc_action.go @@ -212,6 +212,7 @@ func (p *pvcBackupItemAction) validatePVCAndPV( } func (p *pvcBackupItemAction) createVolumeSnapshot( + ctx context.Context, pvc corev1api.PersistentVolumeClaim, backup *velerov1api.Backup, policySnapshotClass string, @@ -222,7 +223,7 @@ func (p *pvcBackupItemAction) createVolumeSnapshot( p.log.Debugf("Fetching storage class for PV %s", *pvc.Spec.StorageClassName) storageClass := new(storagev1api.StorageClass) if err := p.crClient.Get( - context.TODO(), crclient.ObjectKey{Name: *pvc.Spec.StorageClassName}, + ctx, crclient.ObjectKey{Name: *pvc.Spec.StorageClassName}, storageClass, ); err != nil { return nil, errors.Wrap(err, "error getting storage class") @@ -230,6 +231,7 @@ func (p *pvcBackupItemAction) createVolumeSnapshot( p.log.Debugf("Fetching VolumeSnapshotClass for %s", storageClass.Provisioner) vsClass, err := csi.GetVolumeSnapshotClass( + ctx, storageClass.Provisioner, backup, &pvc, @@ -266,7 +268,7 @@ func (p *pvcBackupItemAction) createVolumeSnapshot( }, } - if err := p.crClient.Create(context.TODO(), vs); err != nil { + if err := p.crClient.Create(ctx, vs); err != nil { return nil, errors.Wrapf( err, "error creating volume snapshot", ) @@ -295,6 +297,8 @@ func (p *pvcBackupItemAction) Execute( ) { p.log.Info("Starting PVCBackupItemAction") + ctx := context.Background() + if valid := p.validateBackup(*backup); !valid { return item, nil, "", nil, nil } @@ -319,7 +323,7 @@ func (p *pvcBackupItemAction) Execute( } // Ensure PVC-to-Pod cache is built for this namespace (lazy per-namespace caching) - if err := p.ensurePVCPodCacheForNamespace(context.TODO(), pvc.Namespace); err != nil { + if err := p.ensurePVCPodCacheForNamespace(ctx, pvc.Namespace); err != nil { return nil, nil, "", nil, err } @@ -347,7 +351,7 @@ func (p *pvcBackupItemAction) Execute( // created but never processed (the DataUpload controller runs inside node-agent), // causing the backup to hang until itemOperationTimeout expires. if boolptr.IsSetToTrue(backup.Spec.SnapshotMoveData) && datamover.IsBuiltInDataMover(backup.Spec.DataMover) { - if err := nodeagent.IsReady(context.TODO(), backup.Namespace, p.crClient, p.log); err != nil { + if err := nodeagent.IsReady(ctx, backup.Namespace, p.crClient, p.log); err != nil { p.log.WithError(err).Error("cannot perform snapshot data movement without running node-agent pods") return nil, nil, "", nil, errors.Wrap(err, "CSI PVC BIA cannot proceed: node-agent is not ready for snapshot data movement") } @@ -360,7 +364,7 @@ func (p *pvcBackupItemAction) Execute( p.log.Infof("Volume policy specifies snapshotClass=%s for PVC %s/%s", policySnapshotClass, pvc.Namespace, pvc.Name) } - vs, err := p.getVolumeSnapshotReference(context.TODO(), pvc, backup, policySnapshotClass) + vs, err := p.getVolumeSnapshotReference(ctx, pvc, backup, policySnapshotClass) if err != nil { return nil, nil, "", nil, err } @@ -376,7 +380,7 @@ func (p *pvcBackupItemAction) Execute( if err != nil { p.log.Errorf("Failed to wait for VolumeSnapshot %s/%s to become ReadyToUse within timeout %v: %s", vs.Namespace, vs.Name, backup.Spec.CSISnapshotTimeout.Duration, err.Error()) - csi.CleanupVolumeSnapshot(vs, p.crClient, p.log) + csi.CleanupVolumeSnapshot(ctx, vs, p.crClient, p.log) return nil, nil, "", nil, errors.WithStack(err) } @@ -427,7 +431,7 @@ func (p *pvcBackupItemAction) Execute( // TODO: need to use DeleteVolumeSnapshotIfAny, after data mover // adopting the controller-runtime client. - if deleteErr := p.crClient.Delete(context.TODO(), vs); deleteErr != nil { + if deleteErr := p.crClient.Delete(ctx, vs); deleteErr != nil { if !apierrors.IsNotFound(deleteErr) { dataUploadLog.WithError(deleteErr).Error("fail to delete VolumeSnapshot") } @@ -565,7 +569,7 @@ func newDataUpload( parentSnapshot := "" if backup.Spec.BackupType == velerov1api.BackupTypeFull { - parentSnapshot = veleroshared.DataUploadParentSnapshotNone + parentSnapshot = veleroshared.ParentSnapshotNone } dataMover := backup.Spec.DataMover @@ -841,7 +845,7 @@ func (p *pvcBackupItemAction) getVolumeSnapshotReference( } // Legacy fallback: create individual VS - return p.createVolumeSnapshot(pvc, backup, policySnapshotClass) + return p.createVolumeSnapshot(ctx, pvc, backup, policySnapshotClass) } func (p *pvcBackupItemAction) findExistingVSForBackup( @@ -1230,7 +1234,7 @@ func setPVCRequestSizeToVSRestoreSize( logger logrus.FieldLogger, ) { if vsc.Status.RestoreSize != nil { - logger.Debugf("Patching PVC request size to fit the volumesnapshot restore size %d", vsc.Status.RestoreSize) + logger.Debugf("Patching PVC request size to fit the volumesnapshot restore size %d", *vsc.Status.RestoreSize) restoreSize := *resource.NewQuantity(*vsc.Status.RestoreSize, resource.BinarySI) // It is possible that the volume provider allocated a larger diff --git a/pkg/backup/actions/csi/pvc_action_test.go b/pkg/backup/actions/csi/pvc_action_test.go index 61141f2d5..4e021c7d1 100644 --- a/pkg/backup/actions/csi/pvc_action_test.go +++ b/pkg/backup/actions/csi/pvc_action_test.go @@ -131,7 +131,7 @@ func TestExecute(t *testing.T) { vsClass: builder.ForVolumeSnapshotClass("testVSClass").Driver("hostpath").ObjectMeta(builder.WithLabels(velerov1api.VolumeSnapshotClassSelectorLabel, "")).Result(), extraObjects: []runtime.Object{ &corev1api.Node{ - ObjectMeta: metav1.ObjectMeta{Name: "linux-node", Labels: map[string]string{"kubernetes.io/os": "linux"}}, + ObjectMeta: metav1.ObjectMeta{Name: "linux-node", Labels: map[string]string{corev1api.LabelOSStable: "linux"}}, }, &appsv1api.DaemonSet{ ObjectMeta: metav1.ObjectMeta{Namespace: "velero", Name: "node-agent"}, @@ -186,7 +186,7 @@ func TestExecute(t *testing.T) { vsClass: builder.ForVolumeSnapshotClass("tescVSClass").Driver("hostpath").ObjectMeta(builder.WithLabels(velerov1api.VolumeSnapshotClassSelectorLabel, "")).Result(), extraObjects: []runtime.Object{ &corev1api.Node{ - ObjectMeta: metav1.ObjectMeta{Name: "linux-node", Labels: map[string]string{"kubernetes.io/os": "linux"}}, + ObjectMeta: metav1.ObjectMeta{Name: "linux-node", Labels: map[string]string{corev1api.LabelOSStable: "linux"}}, }, &appsv1api.DaemonSet{ ObjectMeta: metav1.ObjectMeta{Namespace: "velero", Name: "node-agent"}, diff --git a/pkg/backup/actions/csi/volumesnapshot_action.go b/pkg/backup/actions/csi/volumesnapshot_action.go index 49e690e93..eb302f2f2 100644 --- a/pkg/backup/actions/csi/volumesnapshot_action.go +++ b/pkg/backup/actions/csi/volumesnapshot_action.go @@ -78,6 +78,8 @@ func (p *volumeSnapshotBackupItemAction) Execute( ) { p.log.Infof("Executing VolumeSnapshotBackupItemAction") + ctx := context.Background() + vs := new(snapshotv1api.VolumeSnapshot) if err := runtime.DefaultUnstructuredConverter.FromUnstructured( item.UnstructuredContent(), vs); err != nil { @@ -90,7 +92,7 @@ func (p *volumeSnapshotBackupItemAction) Execute( WithField("Backup", fmt.Sprintf("%s/%s", backup.Namespace, backup.Name)). WithField("BackupPhase", backup.Status.Phase).Debugf("Cleaning VolumeSnapshots.") - csi.DeleteReadyVolumeSnapshot(*vs, p.crClient, p.log) + csi.DeleteReadyVolumeSnapshot(ctx, *vs, p.crClient, p.log) return item, nil, "", nil, nil } @@ -115,11 +117,9 @@ func (p *volumeSnapshotBackupItemAction) Execute( p.log.Infof("Getting VolumesnapshotContent for Volumesnapshot %s/%s", vs.Namespace, vs.Name) - ctx := context.TODO() - vsc, err := csi.GetVSCForVS(ctx, vs, p.crClient) if err != nil { - csi.CleanupVolumeSnapshot(vs, p.crClient, p.log) + csi.CleanupVolumeSnapshot(ctx, vs, p.crClient, p.log) return nil, nil, "", nil, errors.WithStack(err) } @@ -187,7 +187,7 @@ func (p *volumeSnapshotBackupItemAction) Execute( ) if vscPatchError := p.crClient.Patch( - context.TODO(), + ctx, vsc, crclient.MergeFrom(originVSC), ); vscPatchError != nil { @@ -203,7 +203,7 @@ func (p *volumeSnapshotBackupItemAction) Execute( originVS := vs.DeepCopy() kubeutil.AddAnnotations(&vs.ObjectMeta, annotations) if err := p.crClient.Patch( - context.TODO(), + ctx, vs, crclient.MergeFrom(originVS), ); err != nil { @@ -269,8 +269,8 @@ func (p *volumeSnapshotBackupItemAction) Progress( } var err error if progress.Started, err = time.Parse(time.RFC3339, operationIDParts[2]); err != nil { - p.log.Errorf("error parsing operation ID's StartedTime", - "part into time %s: %s", operationID, err.Error()) + p.log.Errorf("error parsing operation ID's StartedTime part into time %s: %s", + operationID, err.Error()) return progress, errors.WithStack(err) } diff --git a/pkg/backup/actions/csi/volumesnapshotcontent_action.go b/pkg/backup/actions/csi/volumesnapshotcontent_action.go index f184230d1..fc93cfb87 100644 --- a/pkg/backup/actions/csi/volumesnapshotcontent_action.go +++ b/pkg/backup/actions/csi/volumesnapshotcontent_action.go @@ -107,8 +107,7 @@ func (p *volumeSnapshotContentBackupItemAction) Execute( } p.log.Infof( - "Returning from VolumeSnapshotContentBackupItemAction", - "with %d additionalItems to backup", + "Returning from VolumeSnapshotContentBackupItemAction with %d additionalItems to backup", len(additionalItems), ) return &unstructured.Unstructured{Object: snapContMap}, additionalItems, "", nil, nil diff --git a/pkg/backup/backup.go b/pkg/backup/backup.go index 30eb26a36..038b85cc1 100644 --- a/pkg/backup/backup.go +++ b/pkg/backup/backup.go @@ -1263,21 +1263,12 @@ func buildFinalTarball(tr *tar.Reader, tw tarWriter, updateFiles map[string]File return errors.WithStack(err) } delete(updateFiles, header.Name) - // skip over file contents from old tarball - _, err := io.ReadAll(tr) - if err != nil { - return errors.WithStack(err) - } } else { // Add original content to new tarball, as item wasn't updated - oldContents, err := io.ReadAll(tr) - if err != nil { - return errors.WithStack(err) - } if err := tw.WriteHeader(header); err != nil { return errors.WithStack(err) } - if _, err := tw.Write(oldContents); err != nil { + if _, err := io.Copy(tw, tr); err != nil { return errors.WithStack(err) } } diff --git a/pkg/backup/backup_test.go b/pkg/backup/backup_test.go index b116d5376..9574aa288 100644 --- a/pkg/backup/backup_test.go +++ b/pkg/backup/backup_test.go @@ -2931,7 +2931,6 @@ func (*fakeVolumeSnapshotter) DeleteSnapshot(snapshotID string) error { // looking at the backup request's VolumeSnapshots field. This test uses the fakeVolumeSnapshotter // struct in place of real volume snapshotters. func TestBackupWithSnapshots(t *testing.T) { - // TODO: add more verification for skippedPVTracker itemBlockPool := StartItemBlockWorkerPool(t.Context(), 1, logrus.StandardLogger()) defer itemBlockPool.Stop() tests := []struct { @@ -2941,6 +2940,7 @@ func TestBackupWithSnapshots(t *testing.T) { apiResources []*test.APIResource snapshotterGetter volumeSnapshotterGetter want []*volume.Snapshot + wantSkippedPVs []SkippedPV }{ { name: "persistent volume with no zone annotation creates a snapshot", @@ -2977,6 +2977,7 @@ func TestBackupWithSnapshots(t *testing.T) { }, }, }, + wantSkippedPVs: []SkippedPV{}, }, { name: "persistent volume with deprecated zone annotation creates a snapshot", @@ -2991,7 +2992,7 @@ func TestBackupWithSnapshots(t *testing.T) { }, apiResources: []*test.APIResource{ test.PVs( - builder.ForPersistentVolume("pv-1").ObjectMeta(builder.WithLabels("failure-domain.beta.kubernetes.io/zone", "zone-1")).Result(), + builder.ForPersistentVolume("pv-1").ObjectMeta(builder.WithLabels(corev1api.LabelFailureDomainBetaZone, "zone-1")).Result(), ), }, snapshotterGetter: map[string]vsv1.VolumeSnapshotter{ @@ -3014,6 +3015,7 @@ func TestBackupWithSnapshots(t *testing.T) { }, }, }, + wantSkippedPVs: []SkippedPV{}, }, { name: "persistent volume with GA zone annotation creates a snapshot", @@ -3028,7 +3030,7 @@ func TestBackupWithSnapshots(t *testing.T) { }, apiResources: []*test.APIResource{ test.PVs( - builder.ForPersistentVolume("pv-1").ObjectMeta(builder.WithLabels("topology.kubernetes.io/zone", "zone-1")).Result(), + builder.ForPersistentVolume("pv-1").ObjectMeta(builder.WithLabels(corev1api.LabelTopologyZone, "zone-1")).Result(), ), }, snapshotterGetter: map[string]vsv1.VolumeSnapshotter{ @@ -3051,6 +3053,7 @@ func TestBackupWithSnapshots(t *testing.T) { }, }, }, + wantSkippedPVs: []SkippedPV{}, }, { name: "persistent volume with both GA and deprecated zone annotation creates a snapshot and should use the GA", @@ -3065,7 +3068,7 @@ func TestBackupWithSnapshots(t *testing.T) { }, apiResources: []*test.APIResource{ test.PVs( - builder.ForPersistentVolume("pv-1").ObjectMeta(builder.WithLabelsMap(map[string]string{"failure-domain.beta.kubernetes.io/zone": "zone-1-deprecated", "topology.kubernetes.io/zone": "zone-1-ga"})).Result(), + builder.ForPersistentVolume("pv-1").ObjectMeta(builder.WithLabelsMap(map[string]string{corev1api.LabelFailureDomainBetaZone: "zone-1-deprecated", corev1api.LabelTopologyZone: "zone-1-ga"})).Result(), ), }, snapshotterGetter: map[string]vsv1.VolumeSnapshotter{ @@ -3088,6 +3091,7 @@ func TestBackupWithSnapshots(t *testing.T) { }, }, }, + wantSkippedPVs: []SkippedPV{}, }, { name: "error returned from CreateSnapshot results in a failed snapshot", @@ -3123,6 +3127,7 @@ func TestBackupWithSnapshots(t *testing.T) { }, }, }, + wantSkippedPVs: []SkippedPV{}, }, { name: "backup with SnapshotVolumes=false does not create any snapshots", @@ -3144,6 +3149,17 @@ func TestBackupWithSnapshots(t *testing.T) { "default": new(fakeVolumeSnapshotter).WithVolume("pv-1", "vol-1", "", "type-1", 100, false), }, want: nil, + wantSkippedPVs: []SkippedPV{ + { + Name: "pv-1", + Reasons: []PVSkipReason{ + { + Approach: volumeSnapshotApproach, + Reason: "not satisfy the criteria for VolumePolicy or the legacy snapshot way", + }, + }, + }, + }, }, { name: "backup with no volume snapshot locations does not create any snapshots", @@ -3162,6 +3178,17 @@ func TestBackupWithSnapshots(t *testing.T) { "default": new(fakeVolumeSnapshotter).WithVolume("pv-1", "vol-1", "", "type-1", 100, false), }, want: nil, + wantSkippedPVs: []SkippedPV{ + { + Name: "pv-1", + Reasons: []PVSkipReason{ + { + Approach: volumeSnapshotApproach, + Reason: "no applicable volumesnapshotter found", + }, + }, + }, + }, }, { name: "backup with no volume snapshotters does not create any snapshots", @@ -3181,6 +3208,17 @@ func TestBackupWithSnapshots(t *testing.T) { }, snapshotterGetter: map[string]vsv1.VolumeSnapshotter{}, want: nil, + wantSkippedPVs: []SkippedPV{ + { + Name: "pv-1", + Reasons: []PVSkipReason{ + { + Approach: volumeSnapshotApproach, + Reason: "no applicable volumesnapshotter found", + }, + }, + }, + }, }, { name: "unsupported persistent volume type does not create any snapshots", @@ -3202,6 +3240,17 @@ func TestBackupWithSnapshots(t *testing.T) { "default": new(fakeVolumeSnapshotter), }, want: nil, + wantSkippedPVs: []SkippedPV{ + { + Name: "pv-1", + Reasons: []PVSkipReason{ + { + Approach: volumeSnapshotApproach, + Reason: "no applicable volumesnapshotter found", + }, + }, + }, + }, }, { name: "when there are multiple volumes, snapshot locations, and snapshotters, volumes are matched to the right snapshotters", @@ -3255,6 +3304,7 @@ func TestBackupWithSnapshots(t *testing.T) { }, }, }, + wantSkippedPVs: []SkippedPV{}, }, } @@ -3273,6 +3323,7 @@ func TestBackupWithSnapshots(t *testing.T) { require.NoError(t, err) assert.Equal(t, tc.want, tc.req.VolumeSnapshots.Get()) + assert.Equal(t, tc.wantSkippedPVs, tc.req.SkippedPVTracker.Summary()) }) } } diff --git a/pkg/backup/item_backupper.go b/pkg/backup/item_backupper.go index c180092a5..16ba0fe9b 100644 --- a/pkg/backup/item_backupper.go +++ b/pkg/backup/item_backupper.go @@ -569,9 +569,9 @@ func (ib *itemBackupper) executeActions( // zoneLabel is the label that stores availability-zone info // on PVs const ( - zoneLabelDeprecated = "failure-domain.beta.kubernetes.io/zone" + zoneLabelDeprecated = corev1api.LabelFailureDomainBetaZone // this is reused for nodeAffinity requirements - zoneLabel = "topology.kubernetes.io/zone" + zoneLabel = corev1api.LabelTopologyZone awsEbsCsiZoneKey = "topology.ebs.csi.aws.com/zone" azureCsiZoneKey = "topology.disk.csi.azure.com/zone" diff --git a/pkg/backup/item_collector.go b/pkg/backup/item_collector.go index 3aade5fad..1733e9ac9 100644 --- a/pkg/backup/item_collector.go +++ b/pkg/backup/item_collector.go @@ -346,7 +346,15 @@ func getOrderedResourcesForType( if !ok || len(orderStr) == 0 { return nil } - orders := strings.Split(orderStr, ",") + parts := strings.Split(orderStr, ",") + orders := make([]string, 0, len(parts)) + for _, part := range parts { + name := strings.TrimSpace(part) + if name == "" { + continue + } + orders = append(orders, name) + } return orders } diff --git a/pkg/backup/item_collector_test.go b/pkg/backup/item_collector_test.go index 084d5b5ff..47a1d7be5 100644 --- a/pkg/backup/item_collector_test.go +++ b/pkg/backup/item_collector_test.go @@ -445,3 +445,24 @@ func TestGetResourceItems(t *testing.T) { }) } } + +func TestGetOrderedResourcesForTypeTrimsSpaces(t *testing.T) { + // Spaces after commas are common in CLI input and should not break ordering. + orders := getOrderedResourcesForType(map[string]string{ + "pods": "ns1/pod2, ns1/pod1", + }, "pods") + require.Equal(t, []string{"ns1/pod2", "ns1/pod1"}, orders) + + log := logrus.StandardLogger() + podResources := []*kubernetesResource{ + {namespace: "ns1", name: "pod3"}, + {namespace: "ns1", name: "pod1"}, + {namespace: "ns1", name: "pod2"}, + } + sorted := sortResourcesByOrder(log, podResources, orders) + require.Equal(t, []*kubernetesResource{ + {namespace: "ns1", name: "pod2", orderedResource: true}, + {namespace: "ns1", name: "pod1", orderedResource: true}, + {namespace: "ns1", name: "pod3"}, + }, sorted) +} diff --git a/pkg/backup/snapshots_test.go b/pkg/backup/snapshots_test.go new file mode 100644 index 000000000..e3ad65833 --- /dev/null +++ b/pkg/backup/snapshots_test.go @@ -0,0 +1,241 @@ +/* +Copyright The Velero Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package backup + +import ( + "testing" + + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/features" + velerotest "github.com/vmware-tanzu/velero/pkg/test" + "github.com/vmware-tanzu/velero/pkg/util/boolptr" +) + +func TestGetBackupCSIResources(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, snapshotv1api.AddToScheme(scheme)) + require.NoError(t, velerov1api.AddToScheme(scheme)) + + tests := []struct { + name string + backup *velerov1api.Backup + csiFeatureEnabled bool + existingObjects []kbclient.Object + wantSnapshots int + wantSnapshotContents int + wantSnapshotClasses int + }{ + { + name: "SnapshotMoveData is true, skip CSI resources", + backup: &velerov1api.Backup{ + ObjectMeta: metav1.ObjectMeta{Name: "test-backup"}, + Spec: velerov1api.BackupSpec{ + SnapshotMoveData: boolptr.True(), + }, + }, + csiFeatureEnabled: true, + existingObjects: []kbclient.Object{ + &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs-1", + Namespace: "ns-1", + Labels: map[string]string{ + velerov1api.BackupNameLabel: "test-backup", + }, + }, + }, + &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vsc-1", + Labels: map[string]string{ + velerov1api.BackupNameLabel: "test-backup", + }, + }, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + VolumeSnapshotClassName: func(s string) *string { return &s }("vsc-class-1"), + }, + }, + &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vsc-class-1", + }, + }, + }, + wantSnapshots: 0, + wantSnapshotContents: 0, + wantSnapshotClasses: 0, + }, + { + name: "CSIFeatureFlag is false, skip CSI resources", + backup: &velerov1api.Backup{ + ObjectMeta: metav1.ObjectMeta{Name: "test-backup"}, + Spec: velerov1api.BackupSpec{ + SnapshotMoveData: boolptr.False(), + }, + }, + csiFeatureEnabled: false, + existingObjects: []kbclient.Object{ + &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs-1", + Namespace: "ns-1", + Labels: map[string]string{ + velerov1api.BackupNameLabel: "test-backup", + }, + }, + }, + &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vsc-1", + Labels: map[string]string{ + velerov1api.BackupNameLabel: "test-backup", + }, + }, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + VolumeSnapshotClassName: func(s string) *string { return &s }("vsc-class-1"), + }, + }, + &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vsc-class-1", + }, + }, + }, + wantSnapshots: 0, + wantSnapshotContents: 0, + wantSnapshotClasses: 0, + }, + { + name: "CSIFeatureFlag enabled, retrieve CSI resources", + backup: &velerov1api.Backup{ + ObjectMeta: metav1.ObjectMeta{Name: "test-backup"}, + Spec: velerov1api.BackupSpec{ + SnapshotMoveData: boolptr.False(), + }, + }, + csiFeatureEnabled: true, + existingObjects: []kbclient.Object{ + &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs-1", + Namespace: "ns-1", + Labels: map[string]string{ + velerov1api.BackupNameLabel: "test-backup", + }, + }, + }, + &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vsc-1", + Labels: map[string]string{ + velerov1api.BackupNameLabel: "test-backup", + }, + }, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + VolumeSnapshotClassName: func(s string) *string { return &s }("vsc-class-1"), + }, + }, + &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vsc-class-1", + }, + }, + }, + wantSnapshots: 1, + wantSnapshotContents: 1, + wantSnapshotClasses: 1, + }, + { + name: "CSIFeatureFlag enabled, multiple contents referencing same class", + backup: &velerov1api.Backup{ + ObjectMeta: metav1.ObjectMeta{Name: "test-backup"}, + Spec: velerov1api.BackupSpec{ + SnapshotMoveData: boolptr.False(), + }, + }, + csiFeatureEnabled: true, + existingObjects: []kbclient.Object{ + &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vsc-1", + Labels: map[string]string{ + velerov1api.BackupNameLabel: "test-backup", + }, + }, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + VolumeSnapshotClassName: func(s string) *string { return &s }("vsc-class-1"), + }, + }, + &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vsc-2", + Labels: map[string]string{ + velerov1api.BackupNameLabel: "test-backup", + }, + }, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + VolumeSnapshotClassName: func(s string) *string { return &s }("vsc-class-1"), + }, + }, + &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vsc-class-1", + }, + }, + }, + wantSnapshots: 0, + wantSnapshotContents: 2, + wantSnapshotClasses: 1, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + defer features.NewFeatureFlagSet() + + if tc.csiFeatureEnabled { + features.Enable(velerov1api.CSIFeatureFlag) + } else { + features.Disable(velerov1api.CSIFeatureFlag) + } + + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(tc.existingObjects...).Build() + logger := velerotest.NewLogger() + + snaps, contents, classes := GetBackupCSIResources(client, client, tc.backup, logger) + + assert.Len(t, snaps, tc.wantSnapshots) + assert.Len(t, contents, tc.wantSnapshotContents) + assert.Len(t, classes, tc.wantSnapshotClasses) + + // If we expect CSI resources to be pulled, ensure the attempts count was updated on the backup object + if tc.csiFeatureEnabled && !boolptr.IsSetToTrue(tc.backup.Spec.SnapshotMoveData) { + assert.Equal(t, tc.wantSnapshots, tc.backup.Status.CSIVolumeSnapshotsAttempted) + } else { + assert.Equal(t, 0, tc.backup.Status.CSIVolumeSnapshotsAttempted) + } + }) + } +} diff --git a/pkg/builder/container_builder.go b/pkg/builder/container_builder.go index 762462c86..e25002629 100644 --- a/pkg/builder/container_builder.go +++ b/pkg/builder/container_builder.go @@ -18,10 +18,12 @@ package builder import ( "encoding/json" + "fmt" "strings" corev1api "k8s.io/api/core/v1" apimachineryRuntime "k8s.io/apimachinery/pkg/runtime" + utilrand "k8s.io/apimachinery/pkg/util/rand" "github.com/vmware-tanzu/velero/pkg/label" ) @@ -42,15 +44,22 @@ func ForContainer(name, image string) *ContainerBuilder { } // ForPluginContainer is a helper builder specifically for plugin init containers -func ForPluginContainer(image string, pullPolicy corev1api.PullPolicy) *ContainerBuilder { +func ForPluginContainer(image string, pullPolicy corev1api.PullPolicy, existingContainers []corev1api.Container) *ContainerBuilder { volumeMount := ForVolumeMount("plugins", "/target").Result() - return ForContainer(getName(image), image).PullPolicy(pullPolicy).VolumeMounts(volumeMount) + return ForContainer(getName(image, existingContainers), image).PullPolicy(pullPolicy).VolumeMounts(volumeMount) } // getName returns the 'name' component of a docker image that includes the entire string // except the registry name, and transforms the combined string into a DNS-1123 compatible name // that fits within the 63-character limit for Kubernetes container names. -func getName(image string) string { +// It appends a random string if there is a collision with existing container names. +func getName(image string, existingContainers []corev1api.Container) string { + // Convert existingContainers to a map for O(1) collision lookups + existingNames := make(map[string]bool, len(existingContainers)) + for _, c := range existingContainers { + existingNames[c.Name] = true + } + slashIndex := strings.Index(image, "/") slashCount := 0 if slashIndex >= 0 { @@ -88,7 +97,20 @@ func getName(image string) string { name := re.Replace(image[start:end]) // Ensure the name doesn't exceed Kubernetes container name length limit - return label.GetValidName(name) + name = label.GetValidName(name) + + for existingNames[name] { + name = re.Replace(image[start:end]) + if len(name) > 57 { + // Leave 6 characters for "-xxxxx" random string + name = name[:57] + name = strings.TrimSuffix(name, "-") + } + name = fmt.Sprintf("%s-%s", name, utilrand.String(5)) + name = label.GetValidName(name) + } + + return name } // Result returns the built Container. diff --git a/pkg/builder/container_builder_test.go b/pkg/builder/container_builder_test.go index b23cbddfd..e0af71f75 100644 --- a/pkg/builder/container_builder_test.go +++ b/pkg/builder/container_builder_test.go @@ -16,16 +16,19 @@ limitations under the License. package builder import ( + "strings" "testing" "github.com/stretchr/testify/assert" + corev1api "k8s.io/api/core/v1" ) func TestGetName(t *testing.T) { tests := []struct { - name string - image string - expected string + name string + image string + existingContainers []corev1api.Container + expected string }{ { name: "image name with registry hostname and tag", @@ -92,11 +95,25 @@ func TestGetName(t *testing.T) { image: "quay.io/vmware-tanzu/velero@sha256:a75f9e8c3ced3943515f249597be389f8233e1258d289b11184796edceaa7dab", expected: "vmware-tanzu-velero", }, + { + name: "duplicate plugin name", + image: "gcr.io/my-repo/my-image:latest", + existingContainers: []corev1api.Container{ + {Name: "my-repo-my-image"}, + }, + expected: "my-repo-my-image-", // we will check it has the prefix + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - assert.Equal(t, test.expected, getName(test.image)) + if test.name == "duplicate plugin name" { + result := getName(test.image, test.existingContainers) + assert.True(t, strings.HasPrefix(result, test.expected), "expected prefix %s in %s", test.expected, result) + assert.Len(t, result, len(test.expected)+5) + } else { + assert.Equal(t, test.expected, getName(test.image, test.existingContainers)) + } }) } } @@ -117,7 +134,7 @@ func TestGetNameWithLongPaths(t *testing.T) { // Should be exactly 63 characters (truncated with hash) assert.Len(t, result, 63) // Should be deterministic - result2 := getName("arohcpsvcdev.azurecr.io/redhat-user-workloads/ocp-art-tenant/oadp-hypershift-oadp-plugin-main@sha256:adb840bf3890b4904a8cdda1a74c82cf8d96c52eba9944ac10e795335d6fd450") + result2 := getName("arohcpsvcdev.azurecr.io/redhat-user-workloads/ocp-art-tenant/oadp-hypershift-oadp-plugin-main@sha256:adb840bf3890b4904a8cdda1a74c82cf8d96c52eba9944ac10e795335d6fd450", nil) assert.Equal(t, result, result2) }, }, @@ -142,7 +159,7 @@ func TestGetNameWithLongPaths(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - result := getName(test.image) + result := getName(test.image, nil) test.validate(t, result) }) } diff --git a/pkg/cmd/cli/backup/create.go b/pkg/cmd/cli/backup/create.go index a8d988d72..2150061ab 100644 --- a/pkg/cmd/cli/backup/create.go +++ b/pkg/cmd/cli/backup/create.go @@ -416,8 +416,19 @@ func ParseOrderedResources(orderMapStr string) (map[string]string, error) { return nil, fmt.Errorf("invalid OrderedResources '%s'", entry) } kind := strings.TrimSpace(kv[0]) - order := strings.TrimSpace(kv[1]) - orderedResources[kind] = order + orderParts := strings.Split(kv[1], ",") + cleaned := make([]string, 0, len(orderParts)) + for _, part := range orderParts { + name := strings.TrimSpace(part) + if name == "" { + continue + } + cleaned = append(cleaned, name) + } + if kind == "" || len(cleaned) == 0 { + return nil, fmt.Errorf("invalid OrderedResources '%s'", entry) + } + orderedResources[kind] = strings.Join(cleaned, ",") } return orderedResources, nil } diff --git a/pkg/cmd/cli/backup/create_test.go b/pkg/cmd/cli/backup/create_test.go index 528e76943..90e9e8c8a 100644 --- a/pkg/cmd/cli/backup/create_test.go +++ b/pkg/cmd/cli/backup/create_test.go @@ -234,6 +234,14 @@ func TestCreateOptions_OrderedResources(t *testing.T) { "persistentvolumes": "pv1,pv2", } assert.Equal(t, expectedMixedResources, orderedResources) + + // Spaces after commas in the resource list must be trimmed. + orderedResources, err = ParseOrderedResources("pods=ns1/p1, ns1/p2 ; persistentvolumeclaims= ns2/pvc1, ns2/pvc2") + require.NoError(t, err) + assert.Equal(t, map[string]string{ + "pods": "ns1/p1,ns1/p2", + "persistentvolumeclaims": "ns2/pvc1,ns2/pvc2", + }, orderedResources) } func TestCreateCommand(t *testing.T) { diff --git a/pkg/cmd/cli/plugin/add.go b/pkg/cmd/cli/plugin/add.go index 45a112a46..553a217dc 100644 --- a/pkg/cmd/cli/plugin/add.go +++ b/pkg/cmd/cli/plugin/add.go @@ -111,7 +111,7 @@ func NewAddCommand(f client.Factory) *cobra.Command { } // add the plugin as an init container - plugin := *builder.ForPluginContainer(args[0], corev1api.PullPolicy(imagePullPolicyFlag.String())).Result() + plugin := *builder.ForPluginContainer(args[0], corev1api.PullPolicy(imagePullPolicyFlag.String()), veleroDeploy.Spec.Template.Spec.InitContainers).Result() veleroDeploy.Spec.Template.Spec.InitContainers = append(veleroDeploy.Spec.Template.Spec.InitContainers, plugin) diff --git a/pkg/cmd/cli/uninstall/uninstall.go b/pkg/cmd/cli/uninstall/uninstall.go index 93e0118c6..033d0603c 100644 --- a/pkg/cmd/cli/uninstall/uninstall.go +++ b/pkg/cmd/cli/uninstall/uninstall.go @@ -57,13 +57,11 @@ var resToDelete = []kbclient.ObjectList{} // uninstallOptions collects all the options for uninstalling Velero from a Kubernetes cluster. type uninstallOptions struct { - wait bool // deprecated force bool } // BindFlags adds command line values to the options struct. func (o *uninstallOptions) BindFlags(flags *pflag.FlagSet) { - flags.BoolVar(&o.wait, "wait", o.wait, "Wait for Velero uninstall to be ready. Optional. Deprecated.") flags.BoolVar(&o.force, "force", o.force, "Forces the Velero uninstall. Optional.") } @@ -81,10 +79,6 @@ Use '--force' to skip the prompt confirming if you want to uninstall Velero. `, Example: ` # velero uninstall --namespace staging`, Run: func(c *cobra.Command, args []string) { - if o.wait { - fmt.Println("Warning: the \"--wait\" option is deprecated and will be removed in a future release. The uninstall command always waits for the uninstall to complete.") - } - // Confirm if not asked to force-skip confirmation if !o.force { fmt.Println("You are about to uninstall Velero.") diff --git a/pkg/cmd/server/config/config.go b/pkg/cmd/server/config/config.go index 08b58a1bd..5198adcbc 100644 --- a/pkg/cmd/server/config/config.go +++ b/pkg/cmd/server/config/config.go @@ -28,6 +28,11 @@ const ( defaultPodVolumeOperationTimeout = 240 * time.Minute defaultResourceTerminatingTimeout = 10 * time.Minute + // DefaultResourceTimeout is the default for --resource-timeout. It matches + // defaultResourceTerminatingTimeout so controller fallbacks stay aligned with + // server defaults (see pkg/cmd/server/config/config.go). + DefaultResourceTimeout = defaultResourceTerminatingTimeout + // server's client default qps and burst defaultClientQPS float32 = 100.0 defaultClientBurst int = 100 @@ -41,7 +46,7 @@ const ( defaultCSISnapshotTimeout = 10 * time.Minute defaultItemOperationTimeout = 4 * time.Hour - resourceTimeout = 10 * time.Minute + resourceTimeout = defaultResourceTerminatingTimeout defaultMaxConcurrentK8SConnections = 30 defaultDisableInformerCache = false @@ -183,6 +188,7 @@ type Config struct { ConcurrentBackups int GlobalBackupVolumePoliciesConfigMap string DefaultResourceModifierConfigMap string + MaxBackupExtractionSize int } func GetDefaultConfig() *Config { @@ -289,4 +295,10 @@ func (c *Config) BindFlags(flags *pflag.FlagSet) { c.DefaultResourceModifierConfigMap, "The name of a ConfigMap in the Velero namespace containing default resource modifier rules applied to all restores. Ignored when a per-restore resource modifier is specified.", ) + flags.IntVar( + &c.MaxBackupExtractionSize, + "max-backup-extraction-size", + c.MaxBackupExtractionSize, + "Maximum size of a backup extraction in megabytes. If not set, default value (16GB) will be used.", + ) } diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index 7aff5e946..665577672 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -61,6 +61,7 @@ import ( "github.com/vmware-tanzu/velero/internal/storage" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" + "github.com/vmware-tanzu/velero/pkg/archive" "github.com/vmware-tanzu/velero/pkg/backup" "github.com/vmware-tanzu/velero/pkg/buildinfo" "github.com/vmware-tanzu/velero/pkg/client" @@ -935,6 +936,11 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string } } + if s.config.MaxBackupExtractionSize > 0 { + s.logger.Infof("Setting backup data extraction cap as %v MB", s.config.MaxBackupExtractionSize) + archive.SetMaxExtractionSize(int64(s.config.MaxBackupExtractionSize) * 1024 * 1024) + } + s.logger.Info("Server starting...") if err := s.mgr.Start(s.ctx); err != nil { diff --git a/pkg/cmd/util/downloadrequest/downloadrequest.go b/pkg/cmd/util/downloadrequest/downloadrequest.go index f0956b1cb..2c90a2894 100644 --- a/pkg/cmd/util/downloadrequest/downloadrequest.go +++ b/pkg/cmd/util/downloadrequest/downloadrequest.go @@ -40,6 +40,12 @@ import ( // not found var ErrNotFound = errors.New("file not found") var ErrDownloadRequestDownloadURLTimeout = errors.New("download request download url timeout, check velero server logs for errors. backup storage location may not be available") +var unzipLimit int64 = 1024 * 1024 * 1024 // 1GB limit + +// ErrDownloadRequestFailed is returned when the server refused the request and gave no +// reason. The controller sets a message in every path that fails today, so this is a +// fallback rather than the usual case. +var ErrDownloadRequestFailed = errors.New("download request failed, check velero server logs for errors") func Stream( ctx context.Context, @@ -114,6 +120,16 @@ func getDownloadURL( if updated.Status.DownloadURL != "" { return updated.Status.DownloadURL, nil } + + // Failed is terminal. Waiting for a URL that will never be signed would end in + // ErrDownloadRequestDownloadURLTimeout, which blames the storage location for + // something the status already explains. + if updated.Status.Phase == veleroV1api.DownloadRequestPhaseFailed { + if updated.Status.Message != "" { + return "", errors.New(updated.Status.Message) + } + return "", ErrDownloadRequestFailed + } } } } @@ -202,17 +218,35 @@ func download( return errors.Errorf("request failed: %v", string(body)) } - reader := resp.Body + var r io.Reader = resp.Body + var gzipReader *gzip.Reader if kind != veleroV1api.DownloadTargetKindBackupContents { // need to decompress logs - gzipReader, err := gzip.NewReader(resp.Body) + var err error + gzipReader, err = gzip.NewReader(resp.Body) if err != nil { return err } defer gzipReader.Close() - reader = gzipReader + + r = io.LimitReader(gzipReader, unzipLimit) } - _, err = io.Copy(w, reader) - return err + _, err = io.Copy(w, r) + if err != nil { + return err + } + + if gzipReader != nil { + var buf [1]byte + n, err := gzipReader.Read(buf[:]) + if n > 0 || err == nil { + return errors.Errorf("decompressed data exceeds the limit") + } + if err != io.EOF { + return err + } + } + + return nil } diff --git a/pkg/cmd/util/downloadrequest/downloadrequest_test.go b/pkg/cmd/util/downloadrequest/downloadrequest_test.go index 995e83dc6..36a02413a 100644 --- a/pkg/cmd/util/downloadrequest/downloadrequest_test.go +++ b/pkg/cmd/util/downloadrequest/downloadrequest_test.go @@ -463,6 +463,7 @@ func TestDownload(t *testing.T) { expectedContent string expectedError bool errorType error + expectedErrMsg string }{ { name: "successful download with gzip for logs", @@ -474,6 +475,16 @@ func TestDownload(t *testing.T) { expectedContent: testContent, expectedError: false, }, + { + name: "error decompressed data exceeds the limit", + serverHandler: func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write(compressedContent.Bytes()) + }, + target: velerov1api.DownloadTargetKindBackupLog, + expectedError: true, + expectedErrMsg: "decompressed data exceeds the limit", + }, { name: "successful download without gzip for backup contents", serverHandler: func(w http.ResponseWriter, r *http.Request) { @@ -506,6 +517,12 @@ func TestDownload(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { + originalLimit := unzipLimit + if tc.expectedErrMsg == "decompressed data exceeds the limit" { + unzipLimit = 10 + } + defer func() { unzipLimit = originalLimit }() + server := httptest.NewServer(tc.serverHandler) defer server.Close() @@ -525,6 +542,9 @@ func TestDownload(t *testing.T) { if tc.errorType != nil { assert.Equal(t, tc.errorType, err) } + if tc.expectedErrMsg != "" { + assert.Contains(t, err.Error(), tc.expectedErrMsg) + } } else { require.NoError(t, err) assert.Equal(t, tc.expectedContent, buf.String()) diff --git a/pkg/cmd/util/output/backup_printer.go b/pkg/cmd/util/output/backup_printer.go index 53a950828..420b73129 100644 --- a/pkg/cmd/util/output/backup_printer.go +++ b/pkg/cmd/util/output/backup_printer.go @@ -90,8 +90,12 @@ func printBackup(backup *velerov1api.Backup) []metav1.TableRow { if backup.Status.Expiration != nil { expiration = backup.Status.Expiration.Time } - if expiration.IsZero() && backup.Spec.TTL.Duration > 0 { - expiration = backup.CreationTimestamp.Add(backup.Spec.TTL.Duration) + // Only estimate expiration from TTL after the backup has started. Backups + // stalled in New have no Status.Expiration yet; using CreationTimestamp + // would incorrectly show them as already expired (issue #3555). + if expiration.IsZero() && backup.Spec.TTL.Duration > 0 && + backup.Status.StartTimestamp != nil && !backup.Status.StartTimestamp.Time.IsZero() { + expiration = backup.Status.StartTimestamp.Time.Add(backup.Spec.TTL.Duration) } status := string(backup.Status.Phase) diff --git a/pkg/cmd/util/output/printer_timestamp_test.go b/pkg/cmd/util/output/printer_timestamp_test.go index f967b3b3f..e38d487c4 100644 --- a/pkg/cmd/util/output/printer_timestamp_test.go +++ b/pkg/cmd/util/output/printer_timestamp_test.go @@ -76,6 +76,26 @@ func TestPrintBackupWithoutStartTimestamp(t *testing.T) { assert.Equal(t, string(velerov1api.BackupPhaseFailedValidation), rows[0].Cells[1]) } +func TestPrintBackupExpiresForStalledNewBackup(t *testing.T) { + created := metav1.NewTime(time.Now().Add(-20 * 24 * time.Hour)) + backup := &velerov1api.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "clusterstate-20210128123759", + CreationTimestamp: created, + }, + Spec: velerov1api.BackupSpec{ + TTL: metav1.Duration{Duration: 10 * 24 * time.Hour}, + }, + Status: velerov1api.BackupStatus{ + Phase: velerov1api.BackupPhaseNew, + }, + } + + rows := printBackup(backup) + require.Len(t, rows, 1) + assert.Equal(t, "n/a", rows[0].Cells[5], "stalled New backup should not show expiration in the past") +} + func TestPrintBackupWithStartTimestamp(t *testing.T) { started := metav1.NewTime(time.Date(2026, 8, 8, 21, 6, 28, 0, time.UTC)) backup := &velerov1api.Backup{ diff --git a/pkg/controller/backup_deletion_controller.go b/pkg/controller/backup_deletion_controller.go index cd74a3a27..416c28cd4 100644 --- a/pkg/controller/backup_deletion_controller.go +++ b/pkg/controller/backup_deletion_controller.go @@ -321,6 +321,10 @@ func (r *backupDeletionReconciler) Reconcile(ctx context.Context, req ctrl.Reque volumeSnapshotters[snapshot.Spec.Location] = volumeSnapshotter } + if snapshot.Status.ProviderSnapshotID == "" { + log.WithField("volumeSnapshot", snapshot.Spec.PersistentVolumeName).Warn("Skipping snapshot deletion: empty ProviderSnapshotID") + continue + } if err := volumeSnapshotter.DeleteSnapshot(snapshot.Status.ProviderSnapshotID); err != nil { errs = append(errs, errors.Wrapf(err, "error deleting snapshot %s", snapshot.Status.ProviderSnapshotID).Error()) } @@ -531,7 +535,7 @@ func (r *backupDeletionReconciler) deleteCSIVolumeSnapshotsIfAny(ctx context.Con } for _, item := range vsList.Items { vs := item - csi.CleanupVolumeSnapshot(&vs, r.Client, log) + csi.CleanupVolumeSnapshot(ctx, &vs, r.Client, log) } } diff --git a/pkg/controller/backup_deletion_controller_test.go b/pkg/controller/backup_deletion_controller_test.go index 24cc65846..d358fbe5e 100644 --- a/pkg/controller/backup_deletion_controller_test.go +++ b/pkg/controller/backup_deletion_controller_test.go @@ -397,6 +397,74 @@ func TestBackupDeletionControllerReconcile(t *testing.T) { // Make sure snapshot was deleted assert.Equal(t, 0, td.volumeSnapshotter.SnapshotsTaken.Len()) }) + t.Run("empty ProviderSnapshotID skips DeleteSnapshot call", func(t *testing.T) { + input := defaultTestDbr() + + backup := builder.ForBackup(velerov1api.DefaultNamespace, input.Spec.BackupName).Result() + backup.UID = "uid" + backup.Spec.StorageLocation = "primary" + + restore1 := builder.ForRestore(backup.Namespace, "restore-1"). + Phase(velerov1api.RestorePhaseCompleted). + Backup(backup.Name). + Result() + + location := &velerov1api.BackupStorageLocation{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: backup.Namespace, + Name: "primary", + }, + Spec: velerov1api.BackupStorageLocationSpec{ + Provider: "objStoreProvider", + StorageType: velerov1api.StorageType{ + ObjectStorage: &velerov1api.ObjectStorageLocation{ + Bucket: "bucket", + }, + }, + }, + Status: velerov1api.BackupStorageLocationStatus{ + Phase: velerov1api.BackupStorageLocationPhaseAvailable, + }, + } + + snapshotLocation := &velerov1api.VolumeSnapshotLocation{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: backup.Namespace, + Name: "vsl-1", + }, + Spec: velerov1api.VolumeSnapshotLocationSpec{ + Provider: "provider-1", + }, + } + td := setupBackupDeletionControllerTest(t, input, backup, restore1, location, snapshotLocation) + + snapshots := []*volume.Snapshot{ + { + Spec: volume.SnapshotSpec{ + Location: "vsl-1", + PersistentVolumeName: "pv-1", + }, + Status: volume.SnapshotStatus{ + ProviderSnapshotID: "", + }, + }, + } + + pluginManager := &pluginmocks.Manager{} + pluginManager.On("GetVolumeSnapshotter", "provider-1").Return(td.volumeSnapshotter, nil) + pluginManager.On("GetDeleteItemActions").Return(nil, nil) + pluginManager.On("CleanupClients") + td.controller.newPluginManager = func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager } + + td.backupStore.On("GetBackupVolumeSnapshots", input.Spec.BackupName).Return(snapshots, nil) + td.backupStore.On("GetBackupContents", input.Spec.BackupName).Return(io.NopCloser(bytes.NewReader([]byte("hello world"))), nil) + td.backupStore.On("DeleteBackup", input.Spec.BackupName).Return(nil) + + _, err := td.controller.Reconcile(t.Context(), td.req) + require.NoError(t, err) + + td.backupStore.AssertCalled(t, "DeleteBackup", input.Spec.BackupName) + }) t.Run("full delete, no errors, with backup name greater than 63 chars", func(t *testing.T) { backup := defaultBackup(). ObjectMeta( diff --git a/pkg/controller/backup_sync_controller.go b/pkg/controller/backup_sync_controller.go index ce9af902f..38d79c727 100644 --- a/pkg/controller/backup_sync_controller.go +++ b/pkg/controller/backup_sync_controller.go @@ -164,17 +164,37 @@ func (b *backupSyncReconciler) Reconcile(ctx context.Context, req ctrl.Request) continue } - if backup.Status.Phase == velerov1api.BackupPhaseWaitingForPluginOperations || - backup.Status.Phase == velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed || - backup.Status.Phase == velerov1api.BackupPhaseFinalizing || - backup.Status.Phase == velerov1api.BackupPhaseFinalizingPartiallyFailed { + // Only sync backup metadata that has reached a phase Velero itself writes to + // object storage. Anything else (including an empty or New phase) would be + // created in the cluster as a backup that still looks pending, which the backup + // queue controller would then pick up and run as if it were a newly requested + // backup. + switch backup.Status.Phase { + case velerov1api.BackupPhaseCompleted, + velerov1api.BackupPhasePartiallyFailed, + velerov1api.BackupPhaseFailed: + // finished backups are synced as-is + case velerov1api.BackupPhaseWaitingForPluginOperations, + velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed, + velerov1api.BackupPhaseFinalizing, + velerov1api.BackupPhaseFinalizingPartiallyFailed: if backup.Status.Expiration == nil || backup.Status.Expiration.After(time.Now()) { log.Debugf("Skipping non-expired incomplete backup %v", backup.Name) continue } log.Debugf("%v Backup is past expiration, syncing for garbage collection", backup.Status.Phase) backup.Status.Phase = velerov1api.BackupPhasePartiallyFailed + default: + log.Infof("Skipping backup %v, phase %q in the backup store is not a phase that can be synced", backup.Name, backup.Status.Phase) + continue } + + // A synced backup is a record of a backup that already ran somewhere else, not + // a backup to run here. Hooks are only read while a backup is being executed, + // so they have no consumer for a synced backup and are dropped rather than + // stored as an executable payload. + backup.Spec.Hooks = velerov1api.BackupHooks{} + backup.Namespace = b.namespace backup.ResourceVersion = "" diff --git a/pkg/controller/backup_sync_controller_test.go b/pkg/controller/backup_sync_controller_test.go index fbfe65457..75f9c5205 100644 --- a/pkg/controller/backup_sync_controller_test.go +++ b/pkg/controller/backup_sync_controller_test.go @@ -204,10 +204,10 @@ var _ = Describe("Backup Sync Reconciler", func() { location: defaultLocation("ns-1"), cloudBackups: []*cloudBackupData{ { - backup: builder.ForBackup("ns-1", "backup-1").Result(), + backup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhaseCompleted).Result(), }, { - backup: builder.ForBackup("ns-1", "backup-2").Result(), + backup: builder.ForBackup("ns-1", "backup-2").Phase(velerov1api.BackupPhaseCompleted).Result(), }, }, }, @@ -309,10 +309,10 @@ var _ = Describe("Backup Sync Reconciler", func() { location: defaultLocation("velero"), cloudBackups: []*cloudBackupData{ { - backup: builder.ForBackup("ns-1", "backup-1").Result(), + backup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhaseCompleted).Result(), }, { - backup: builder.ForBackup("ns-1", "backup-2").Result(), + backup: builder.ForBackup("ns-1", "backup-2").Phase(velerov1api.BackupPhaseCompleted).Result(), }, }, }, @@ -322,10 +322,10 @@ var _ = Describe("Backup Sync Reconciler", func() { location: defaultLocation("ns-1"), cloudBackups: []*cloudBackupData{ { - backup: builder.ForBackup("ns-1", "backup-1").Result(), + backup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhaseCompleted).Result(), }, { - backup: builder.ForBackup("ns-1", "backup-2").Result(), + backup: builder.ForBackup("ns-1", "backup-2").Phase(velerov1api.BackupPhaseCompleted).Result(), }, }, existingBackups: []*velerov1api.Backup{ @@ -341,7 +341,7 @@ var _ = Describe("Backup Sync Reconciler", func() { location: defaultLocation("ns-1"), cloudBackups: []*cloudBackupData{ { - backup: builder.ForBackup("ns-1", "backup-1").Result(), + backup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhaseCompleted).Result(), }, }, existingBackups: []*velerov1api.Backup{ @@ -356,10 +356,10 @@ var _ = Describe("Backup Sync Reconciler", func() { location: defaultLocation("ns-1"), cloudBackups: []*cloudBackupData{ { - backup: builder.ForBackup("ns-1", "backup-1").StorageLocation("foo").ObjectMeta(builder.WithLabels(velerov1api.StorageLocationLabel, "foo")).Result(), + backup: builder.ForBackup("ns-1", "backup-1").StorageLocation("foo").ObjectMeta(builder.WithLabels(velerov1api.StorageLocationLabel, "foo")).Phase(velerov1api.BackupPhaseCompleted).Result(), }, { - backup: builder.ForBackup("ns-1", "backup-2").Result(), + backup: builder.ForBackup("ns-1", "backup-2").Phase(velerov1api.BackupPhaseCompleted).Result(), }, }, }, @@ -370,10 +370,10 @@ var _ = Describe("Backup Sync Reconciler", func() { longLocationNameEnabled: true, cloudBackups: []*cloudBackupData{ { - backup: builder.ForBackup("ns-1", "backup-1").StorageLocation("foo").ObjectMeta(builder.WithLabels(velerov1api.StorageLocationLabel, "foo")).Result(), + backup: builder.ForBackup("ns-1", "backup-1").StorageLocation("foo").ObjectMeta(builder.WithLabels(velerov1api.StorageLocationLabel, "foo")).Phase(velerov1api.BackupPhaseCompleted).Result(), }, { - backup: builder.ForBackup("ns-1", "backup-2").Result(), + backup: builder.ForBackup("ns-1", "backup-2").Phase(velerov1api.BackupPhaseCompleted).Result(), }, }, }, @@ -383,13 +383,13 @@ var _ = Describe("Backup Sync Reconciler", func() { location: defaultLocation("ns-1"), cloudBackups: []*cloudBackupData{ { - backup: builder.ForBackup("ns-1", "backup-1").Result(), + backup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhaseCompleted).Result(), podVolumeBackups: []*velerov1api.PodVolumeBackup{ builder.ForPodVolumeBackup("ns-1", "pvb-1").Result(), }, }, { - backup: builder.ForBackup("ns-1", "backup-2").Result(), + backup: builder.ForBackup("ns-1", "backup-2").Phase(velerov1api.BackupPhaseCompleted).Result(), podVolumeBackups: []*velerov1api.PodVolumeBackup{ builder.ForPodVolumeBackup("ns-1", "pvb-2").Result(), }, @@ -402,13 +402,13 @@ var _ = Describe("Backup Sync Reconciler", func() { location: defaultLocation("ns-1"), cloudBackups: []*cloudBackupData{ { - backup: builder.ForBackup("ns-1", "backup-1").Result(), + backup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhaseCompleted).Result(), podVolumeBackups: []*velerov1api.PodVolumeBackup{ builder.ForPodVolumeBackup("ns-1", "pvb-1").Result(), }, }, { - backup: builder.ForBackup("ns-1", "backup-2").Result(), + backup: builder.ForBackup("ns-1", "backup-2").Phase(velerov1api.BackupPhaseCompleted).Result(), podVolumeBackups: []*velerov1api.PodVolumeBackup{ builder.ForPodVolumeBackup("ns-1", "pvb-3").Result(), }, @@ -557,6 +557,184 @@ var _ = Describe("Backup Sync Reconciler", func() { } }) + It("Test synced backups are never picked up by the backup queue controller", func() { + fakeClock := testclocks.NewFakeClock(time.Now()) + hooks := velerov1api.BackupHooks{ + Resources: []velerov1api.BackupResourceHookSpec{ + { + Name: "hook-1", + PreHooks: []velerov1api.BackupResourceHook{ + { + Exec: &velerov1api.ExecHook{ + Container: "container-1", + Command: []string{"/bin/sh", "-c", "echo hello"}, + }, + }, + }, + }, + }, + } + + tests := []struct { + name string + cloudBackup *velerov1api.Backup + expectSynced bool + // phase expected in the cluster after the sync and queue reconciles have run. + // only checked when expectSynced is true. + expectPhase velerov1api.BackupPhase + }{ + { + name: "backup metadata with an empty phase is not synced", + cloudBackup: builder.ForBackup("ns-1", "backup-1").Hooks(hooks).Result(), + expectSynced: false, + }, + { + name: "backup metadata in phase New is not synced", + cloudBackup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhaseNew).Hooks(hooks).Result(), + expectSynced: false, + }, + { + name: "backup metadata in phase Queued is not synced", + cloudBackup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhaseQueued).Hooks(hooks).Result(), + expectSynced: false, + }, + { + name: "backup metadata in phase ReadyToStart is not synced", + cloudBackup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhaseReadyToStart).Hooks(hooks).Result(), + expectSynced: false, + }, + { + name: "backup metadata in phase InProgress is not synced", + cloudBackup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhaseInProgress).Hooks(hooks).Result(), + expectSynced: false, + }, + { + name: "backup metadata in phase Deleting is not synced", + cloudBackup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhaseDeleting).Hooks(hooks).Result(), + expectSynced: false, + }, + { + name: "backup metadata in phase Completed is synced and stays Completed", + cloudBackup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhaseCompleted).Hooks(hooks).Result(), + expectSynced: true, + expectPhase: velerov1api.BackupPhaseCompleted, + }, + { + name: "backup metadata in phase PartiallyFailed is synced and stays PartiallyFailed", + cloudBackup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhasePartiallyFailed).Result(), + expectSynced: true, + expectPhase: velerov1api.BackupPhasePartiallyFailed, + }, + { + name: "backup metadata in phase Failed is synced and stays Failed", + cloudBackup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhaseFailed).Result(), + expectSynced: true, + expectPhase: velerov1api.BackupPhaseFailed, + }, + { + name: "non-expired backup waiting for plugin operations is not synced", + cloudBackup: builder.ForBackup("ns-1", "backup-1"). + Phase(velerov1api.BackupPhaseWaitingForPluginOperations). + Expiration(fakeClock.Now().Add(time.Hour)).Result(), + expectSynced: false, + }, + { + name: "expired backup waiting for plugin operations is synced as PartiallyFailed", + cloudBackup: builder.ForBackup("ns-1", "backup-1"). + Phase(velerov1api.BackupPhaseWaitingForPluginOperations). + Expiration(fakeClock.Now().Add(-time.Hour)).Result(), + expectSynced: true, + expectPhase: velerov1api.BackupPhasePartiallyFailed, + }, + { + name: "expired backup waiting for plugin operations partially failed is synced as PartiallyFailed", + cloudBackup: builder.ForBackup("ns-1", "backup-1"). + Phase(velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed). + Expiration(fakeClock.Now().Add(-time.Hour)).Result(), + expectSynced: true, + expectPhase: velerov1api.BackupPhasePartiallyFailed, + }, + { + name: "expired finalizing backup is synced as PartiallyFailed", + cloudBackup: builder.ForBackup("ns-1", "backup-1"). + Phase(velerov1api.BackupPhaseFinalizing). + Expiration(fakeClock.Now().Add(-time.Hour)).Result(), + expectSynced: true, + expectPhase: velerov1api.BackupPhasePartiallyFailed, + }, + { + name: "expired finalizing partially failed backup is synced as PartiallyFailed", + cloudBackup: builder.ForBackup("ns-1", "backup-1"). + Phase(velerov1api.BackupPhaseFinalizingPartiallyFailed). + Expiration(fakeClock.Now().Add(-time.Hour)).Result(), + expectSynced: true, + expectPhase: velerov1api.BackupPhasePartiallyFailed, + }, + } + + queueScheme := runtime.NewScheme() + Expect(velerov1api.AddToScheme(queueScheme)).ShouldNot(HaveOccurred()) + + for _, test := range tests { + var ( + client = ctrlfake.NewClientBuilder().Build() + pluginManager = &pluginmocks.Manager{} + backupStores = make(map[string]*persistencemocks.BackupStore) + location = defaultLocation("ns-1") + ) + + pluginManager.On("CleanupClients").Return(nil) + syncReconciler := backupSyncReconciler{ + client: client, + namespace: "ns-1", + defaultBackupSyncPeriod: time.Second * 10, + newPluginManager: func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager }, + backupStoreGetter: NewFakeObjectBackupStoreGetter(backupStores), + logger: velerotest.NewLogger(), + } + + Expect(client.Create(ctx, location)).ShouldNot(HaveOccurred(), test.name) + backupStore := &persistencemocks.BackupStore{} + backupStores[location.Name] = backupStore + backupStore.On("ListBackups").Return([]string{test.cloudBackup.Name}, nil) + backupStore.On("BackupExists", "bucket-1", test.cloudBackup.Name).Return(true, nil) + backupStore.On("GetBackupMetadata", test.cloudBackup.Name).Return(test.cloudBackup, nil) + backupStore.On("GetPodVolumeBackups", test.cloudBackup.Name).Return(nil, nil) + + _, err := syncReconciler.Reconcile(ctx, ctrl.Request{ + NamespacedName: types.NamespacedName{Namespace: location.Namespace, Name: location.Name}, + }) + Expect(err).ShouldNot(HaveOccurred(), test.name) + + backupKey := types.NamespacedName{Namespace: "ns-1", Name: test.cloudBackup.Name} + synced := &velerov1api.Backup{} + err = client.Get(ctx, backupKey, synced) + + if !test.expectSynced { + Expect(apierrors.IsNotFound(err)).To(BeTrue(), test.name) + continue + } + Expect(err).ShouldNot(HaveOccurred(), test.name) + + // Reconcile the synced backup with the queue controller twice: the first + // reconcile would move a New/empty-phase backup to Queued, the second one + // would move it on to ReadyToStart, which is what hands it to the backup + // controller for execution. + queueReconciler := NewBackupQueueReconciler(client, queueScheme, velerotest.NewLogger(), 1, NewBackupTracker()) + for range 2 { + _, err = queueReconciler.Reconcile(ctx, ctrl.Request{NamespacedName: backupKey}) + Expect(err).ShouldNot(HaveOccurred(), test.name) + } + + after := &velerov1api.Backup{} + Expect(client.Get(ctx, backupKey, after)).ShouldNot(HaveOccurred(), test.name) + Expect(after.Status.Phase).To(BeEquivalentTo(test.expectPhase), test.name) + // Hooks are dropped on sync, so the stored metadata cannot carry a payload + // that a later code path could execute. + Expect(after.Spec.Hooks.Resources).To(BeEmpty(), test.name) + } + }) + It("Test deleting orphaned backups.", func() { longLabelName := "the-really-long-location-name-that-is-much-more-than-63-characters" diff --git a/pkg/controller/data_download_controller.go b/pkg/controller/data_download_controller.go index 422879d6e..1e867fed5 100644 --- a/pkg/controller/data_download_controller.go +++ b/pkg/controller/data_download_controller.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "strings" + "sync" "time" "github.com/cockroachdb/errors" @@ -74,7 +75,7 @@ type DataDownloadReconciler struct { podResources corev1api.ResourceRequirements preparingTimeout time.Duration metrics *metrics.ServerMetrics - cancelledDataDownload map[string]time.Time + cancelledDataDownload sync.Map dataMovePriorityClass string repoConfigMgr repository.ConfigManager podLabels map[string]string @@ -118,7 +119,6 @@ func NewDataDownloadReconciler( podResources: podResources, preparingTimeout: preparingTimeout, metrics: metrics, - cancelledDataDownload: make(map[string]time.Time), dataMovePriorityClass: dataMovePriorityClass, repoConfigMgr: repoConfigMgr, podLabels: podLabels, @@ -131,6 +131,7 @@ func NewDataDownloadReconciler( // +kubebuilder:rbac:groups="",resources=pods,verbs=get // +kubebuilder:rbac:groups="",resources=persistentvolumes,verbs=get // +kubebuilder:rbac:groups="",resources=persistentvolumerclaims,verbs=get +// +kubebuilder:rbac:groups="",resources=secrets;configmaps,verbs=get;list;create;delete func (r *DataDownloadReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.logger.WithFields(logrus.Fields{ @@ -198,7 +199,7 @@ func (r *DataDownloadReconciler) Reconcile(ctx context.Context, req ctrl.Request } } } else { - delete(r.cancelledDataDownload, dd.Name) + r.cancelledDataDownload.Delete(dd.Name) // put the finalizer remove action here for all cr will goes to the final status, we could check finalizer and do remove action in final status // instead of intermediate state. @@ -223,9 +224,9 @@ func (r *DataDownloadReconciler) Reconcile(ctx context.Context, req ctrl.Request } if dd.Spec.Cancel { - if spotted, found := r.cancelledDataDownload[dd.Name]; !found { - r.cancelledDataDownload[dd.Name] = r.Clock.Now() - } else { + v, loaded := r.cancelledDataDownload.LoadOrStore(dd.Name, r.Clock.Now()) + if loaded { + spotted := v.(time.Time) delay := cancelDelayOthers if dd.Status.Phase == velerov2alpha1api.DataDownloadPhaseInProgress { delay = cancelDelayInProgress @@ -234,7 +235,7 @@ func (r *DataDownloadReconciler) Reconcile(ctx context.Context, req ctrl.Request if time.Since(spotted) > delay { log.Infof("Data download %s is canceled in Phase %s but not handled in rasonable time", dd.GetName(), dd.Status.Phase) if r.tryCancelDataDownload(ctx, dd, "") { - delete(r.cancelledDataDownload, dd.Name) + r.cancelledDataDownload.Delete(dd.Name) } return ctrl.Result{}, nil @@ -556,7 +557,7 @@ func (r *DataDownloadReconciler) OnDataDownloadCancelled(ctx context.Context, na log.WithError(err).Error("error updating data download status") } else { r.metrics.RegisterDataDownloadCancel(r.nodeName) - delete(r.cancelledDataDownload, dd.Name) + r.cancelledDataDownload.Delete(dd.Name) } } diff --git a/pkg/controller/data_download_controller_test.go b/pkg/controller/data_download_controller_test.go index a605fcaaa..4ef79b823 100644 --- a/pkg/controller/data_download_controller_test.go +++ b/pkg/controller/data_download_controller_test.go @@ -19,9 +19,12 @@ package controller import ( "context" "fmt" + "sync" "testing" "time" + clocktesting "k8s.io/utils/clock/testing" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" @@ -507,7 +510,7 @@ func TestDataDownloadReconcile(t *testing.T) { } if test.sportTime != nil { - r.cancelledDataDownload[test.dd.Name] = test.sportTime.Time + r.cancelledDataDownload.Store(test.dd.Name, test.sportTime.Time) } if test.constrained { @@ -624,9 +627,15 @@ func TestDataDownloadReconcile(t *testing.T) { } if test.expectCancelRecord { - assert.Contains(t, r.cancelledDataDownload, test.dd.Name) + _, ok := r.cancelledDataDownload.Load(test.dd.Name) + assert.True(t, ok) } else { - assert.Empty(t, r.cancelledDataDownload) + empty := true + r.cancelledDataDownload.Range(func(key, value any) bool { + empty = false + return false + }) + assert.True(t, empty) } if isDataDownloadInFinalState(&dd) || dd.Status.Phase == velerov2alpha1api.DataDownloadPhaseInProgress { @@ -1437,3 +1446,50 @@ func TestDataDownloadSetupExposeParam(t *testing.T) { }) } } + +type sequenceClock struct { + *clocktesting.FakeClock + mu sync.Mutex +} + +func (c *sequenceClock) Now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + c.FakeClock.Step(time.Second) + return c.FakeClock.Now() +} + +func TestDataDownloadCancelConcurrency(t *testing.T) { + ctx := t.Context() + dd := dataDownloadBuilder().Cancel(true).Phase(velerov2alpha1api.DataDownloadPhaseInProgress).Result() + + r, err := initDataDownloadReconciler(t, nil) + require.NoError(t, err) + + err = r.client.Create(ctx, dd) + require.NoError(t, err) + + firstTime := time.Now() + // manually store the initial time + r.cancelledDataDownload.Store(dd.Name, firstTime) + + // Custom clock that returns a different time each call + r.Clock = &sequenceClock{FakeClock: clocktesting.NewFakeClock(firstTime)} + + var wg sync.WaitGroup + routines := 50 + wg.Add(routines) + + for i := 0; i < routines; i++ { + go func() { + defer wg.Done() + _, _ = r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: dd.Name, Namespace: dd.Namespace}}) + }() + } + + wg.Wait() + + v, ok := r.cancelledDataDownload.Load(dd.Name) + assert.True(t, ok) + assert.Equal(t, firstTime, v.(time.Time), "The initially recorded timestamp should be preserved") +} diff --git a/pkg/controller/data_upload_controller.go b/pkg/controller/data_upload_controller.go index 61ccfef58..ae50a7741 100644 --- a/pkg/controller/data_upload_controller.go +++ b/pkg/controller/data_upload_controller.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "strings" + "sync" "time" "github.com/cockroachdb/errors" @@ -81,7 +82,7 @@ type DataUploadReconciler struct { podResources corev1api.ResourceRequirements preparingTimeout time.Duration metrics *metrics.ServerMetrics - cancelledDataUpload map[string]time.Time + cancelledDataUpload sync.Map dataMovePriorityClass string podLabels map[string]string podAnnotations map[string]string @@ -130,7 +131,6 @@ func NewDataUploadReconciler( podResources: podResources, preparingTimeout: preparingTimeout, metrics: metrics, - cancelledDataUpload: make(map[string]time.Time), dataMovePriorityClass: dataMovePriorityClass, podLabels: podLabels, podAnnotations: podAnnotations, @@ -143,6 +143,7 @@ func NewDataUploadReconciler( // +kubebuilder:rbac:groups="",resources=pods,verbs=get // +kubebuilder:rbac:groups="",resources=persistentvolumes,verbs=get // +kubebuilder:rbac:groups="",resources=persistentvolumerclaims,verbs=get +// +kubebuilder:rbac:groups="",resources=secrets;configmaps,verbs=get;list;create;delete func (r *DataUploadReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.logger.WithFields(logrus.Fields{ @@ -207,7 +208,7 @@ func (r *DataUploadReconciler) Reconcile(ctx context.Context, req ctrl.Request) } } } else { - delete(r.cancelledDataUpload, du.Name) + r.cancelledDataUpload.Delete(du.Name) // put the finalizer remove action here for all cr will goes to the final status, we could check finalizer and do remove action in final status // instead of intermediate state. @@ -232,9 +233,9 @@ func (r *DataUploadReconciler) Reconcile(ctx context.Context, req ctrl.Request) } if du.Spec.Cancel { - if spotted, found := r.cancelledDataUpload[du.Name]; !found { - r.cancelledDataUpload[du.Name] = r.Clock.Now() - } else { + v, loaded := r.cancelledDataUpload.LoadOrStore(du.Name, r.Clock.Now()) + if loaded { + spotted := v.(time.Time) delay := cancelDelayOthers if du.Status.Phase == velerov2alpha1api.DataUploadPhaseInProgress { delay = cancelDelayInProgress @@ -243,7 +244,7 @@ func (r *DataUploadReconciler) Reconcile(ctx context.Context, req ctrl.Request) if time.Since(spotted) > delay { log.Infof("Data upload %s is canceled in Phase %s but not handled in reasonable time", du.GetName(), du.Status.Phase) if r.tryCancelDataUpload(ctx, du, "") { - delete(r.cancelledDataUpload, du.Name) + r.cancelledDataUpload.Delete(du.Name) } return ctrl.Result{}, nil @@ -577,7 +578,7 @@ func (r *DataUploadReconciler) OnDataUploadCancelled(ctx context.Context, namesp log.WithError(err).Error("error updating DataUpload status") } else { r.metrics.RegisterDataUploadCancel(r.nodeName) - delete(r.cancelledDataUpload, du.Name) + r.cancelledDataUpload.Delete(du.Name) } } diff --git a/pkg/controller/data_upload_controller_test.go b/pkg/controller/data_upload_controller_test.go index ec819f8eb..30b5926ac 100644 --- a/pkg/controller/data_upload_controller_test.go +++ b/pkg/controller/data_upload_controller_test.go @@ -19,6 +19,7 @@ package controller import ( "context" "fmt" + "sync" "testing" "time" @@ -672,7 +673,7 @@ func TestReconcile(t *testing.T) { } if test.sportTime != nil { - r.cancelledDataUpload[test.du.Name] = test.sportTime.Time + r.cancelledDataUpload.Store(test.du.Name, test.sportTime.Time) } if test.constrained { @@ -752,9 +753,15 @@ func TestReconcile(t *testing.T) { } if test.expectCancelRecord { - assert.Contains(t, r.cancelledDataUpload, test.du.Name) + _, ok := r.cancelledDataUpload.Load(test.du.Name) + assert.True(t, ok) } else { - assert.Empty(t, r.cancelledDataUpload) + empty := true + r.cancelledDataUpload.Range(func(key, value any) bool { + empty = false + return false + }) + assert.True(t, empty) } if isDataUploadInFinalState(&du) || du.Status.Phase == velerov2alpha1api.DataUploadPhaseInProgress { @@ -1561,3 +1568,50 @@ func TestDataUploadSetupExposeParam(t *testing.T) { }) } } + +type dataUploadSequenceClock struct { + *testclocks.FakeClock + mu sync.Mutex +} + +func (c *dataUploadSequenceClock) Now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + c.FakeClock.Step(time.Second) + return c.FakeClock.Now() +} + +func TestDataUploadCancelConcurrency(t *testing.T) { + ctx := t.Context() + du := dataUploadBuilder().Cancel(true).Phase(velerov2alpha1api.DataUploadPhaseInProgress).Result() + + r, err := initDataUploaderReconciler() + require.NoError(t, err) + + err = r.client.Create(ctx, du) + require.NoError(t, err) + + firstTime := time.Now() + // manually store the initial time + r.cancelledDataUpload.Store(du.Name, firstTime) + + // Custom clock that returns a different time each call + r.Clock = &dataUploadSequenceClock{FakeClock: testclocks.NewFakeClock(firstTime)} + + var wg sync.WaitGroup + routines := 50 + wg.Add(routines) + + for i := 0; i < routines; i++ { + go func() { + defer wg.Done() + _, _ = r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: du.Name, Namespace: du.Namespace}}) + }() + } + + wg.Wait() + + v, ok := r.cancelledDataUpload.Load(du.Name) + assert.True(t, ok) + assert.Equal(t, firstTime, v.(time.Time), "The initially recorded timestamp should be preserved") +} diff --git a/pkg/controller/download_request_controller.go b/pkg/controller/download_request_controller.go index 02d385bec..a2a865251 100644 --- a/pkg/controller/download_request_controller.go +++ b/pkg/controller/download_request_controller.go @@ -18,6 +18,7 @@ package controller import ( "context" + "fmt" "time" "github.com/cockroachdb/errors" @@ -132,11 +133,13 @@ func (r *downloadRequestReconciler) Reconcile(ctx context.Context, req ctrl.Requ // Update the expiration. downloadRequest.Status.Expiration = &metav1.Time{Time: r.clock.Now().Add(persistence.DownloadURLTTL)} - if downloadRequest.Spec.Target.Kind == velerov1api.DownloadTargetKindRestoreLog || + isRestoreTarget := downloadRequest.Spec.Target.Kind == velerov1api.DownloadTargetKindRestoreLog || downloadRequest.Spec.Target.Kind == velerov1api.DownloadTargetKindRestoreResults || downloadRequest.Spec.Target.Kind == velerov1api.DownloadTargetKindRestoreResourceList || downloadRequest.Spec.Target.Kind == velerov1api.DownloadTargetKindRestoreItemOperations || - downloadRequest.Spec.Target.Kind == velerov1api.DownloadTargetKindRestoreVolumeInfo { + downloadRequest.Spec.Target.Kind == velerov1api.DownloadTargetKindRestoreVolumeInfo + + if isRestoreTarget { restore := &velerov1api.Restore{} if err := r.client.Get(ctx, kbclient.ObjectKey{ Namespace: downloadRequest.Namespace, @@ -149,6 +152,16 @@ func (r *downloadRequestReconciler) Reconcile(ctx context.Context, req ctrl.Requ log.Warnf("fail to get restore for DownloadRequest %s. Retry later.", err.Error()) return ctrl.Result{}, errors.WithStack(err) } + + if restorePhaseHasNoArtifacts(restore.Status.Phase) { + msg := fmt.Sprintf("restore %q is in phase %q and has not written any artifacts", + restore.Name, restore.Status.Phase) + log.Infof("%s, not signing a URL", msg) + downloadRequest.Status.Phase = velerov1api.DownloadRequestPhaseFailed + downloadRequest.Status.Message = msg + return ctrl.Result{}, nil + } + backupName = restore.Spec.BackupName } @@ -165,6 +178,15 @@ func (r *downloadRequestReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{}, errors.WithStack(err) } + if !isRestoreTarget && backupPhaseHasNoArtifacts(backup.Status.Phase) { + msg := fmt.Sprintf("backup %q is in phase %q and has not written any artifacts", + backup.Name, backup.Status.Phase) + log.Infof("%s, not signing a URL", msg) + downloadRequest.Status.Phase = velerov1api.DownloadRequestPhaseFailed + downloadRequest.Status.Message = msg + return ctrl.Result{}, nil + } + location := &velerov1api.BackupStorageLocation{} if err := r.client.Get(ctx, kbclient.ObjectKey{ Namespace: backup.Namespace, @@ -236,3 +258,32 @@ func (r *downloadRequestReconciler) SetupWithManager(mgr ctrl.Manager) error { WatchesRawSource(downloadRequestSource). Complete(r) } + +// backupPhaseHasNoArtifacts reports whether a backup in this phase is known to have +// written nothing to object storage yet, so no DownloadTargetKind can exist for it. +// +// Only pre-execution phases are listed. InProgress and everything after it may have a +// partial log or other artifacts, and Deleting may still have all of them, so those are +// left alone: signing there preserves the behavior callers have today. +func backupPhaseHasNoArtifacts(phase velerov1api.BackupPhase) bool { + switch phase { + case velerov1api.BackupPhaseNew, + velerov1api.BackupPhaseQueued, + velerov1api.BackupPhaseReadyToStart, + velerov1api.BackupPhaseFailedValidation: + return true + default: + return false + } +} + +// restorePhaseHasNoArtifacts is the same test for a restore. +func restorePhaseHasNoArtifacts(phase velerov1api.RestorePhase) bool { + switch phase { + case velerov1api.RestorePhaseNew, + velerov1api.RestorePhaseFailedValidation: + return true + default: + return false + } +} diff --git a/pkg/controller/download_request_phase_test.go b/pkg/controller/download_request_phase_test.go new file mode 100644 index 000000000..ac3b44e35 --- /dev/null +++ b/pkg/controller/download_request_phase_test.go @@ -0,0 +1,277 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/runtime" + testclocks "k8s.io/utils/clock/testing" + ctrl "sigs.k8s.io/controller-runtime" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + v1crds "github.com/vmware-tanzu/velero/config/crd/v1" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/builder" + persistencemocks "github.com/vmware-tanzu/velero/pkg/persistence/mocks" + "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" + pluginmocks "github.com/vmware-tanzu/velero/pkg/plugin/mocks" + velerotest "github.com/vmware-tanzu/velero/pkg/test" +) + +// Expectations are declared once and reused by both the behavior tests and the coverage +// tests below, so a phase can only be tested by being classified here first. +var backupPhaseExpectations = map[velerov1api.BackupPhase]bool{ + // Pre-execution: nothing has been written for any target kind. + velerov1api.BackupPhaseNew: true, + velerov1api.BackupPhaseQueued: true, + velerov1api.BackupPhaseReadyToStart: true, + velerov1api.BackupPhaseFailedValidation: true, + + // From here on there may be a partial log or other artifacts, and Deleting may + // still have all of them, so these keep the behavior callers have today. + velerov1api.BackupPhaseInProgress: false, + velerov1api.BackupPhaseWaitingForPluginOperations: false, + velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed: false, + velerov1api.BackupPhaseFinalizing: false, + velerov1api.BackupPhaseFinalizingPartiallyFailed: false, + velerov1api.BackupPhaseCompleted: false, + velerov1api.BackupPhasePartiallyFailed: false, + velerov1api.BackupPhaseFailed: false, + velerov1api.BackupPhaseDeleting: false, +} + +var restorePhaseExpectations = map[velerov1api.RestorePhase]bool{ + velerov1api.RestorePhaseNew: true, + velerov1api.RestorePhaseFailedValidation: true, + + velerov1api.RestorePhaseInProgress: false, + velerov1api.RestorePhaseWaitingForPluginOperations: false, + velerov1api.RestorePhaseWaitingForPluginOperationsPartiallyFailed: false, + velerov1api.RestorePhaseFinalizing: false, + velerov1api.RestorePhaseFinalizingPartiallyFailed: false, + velerov1api.RestorePhaseCompleted: false, + velerov1api.RestorePhasePartiallyFailed: false, + velerov1api.RestorePhaseFailed: false, +} + +func TestBackupPhaseHasNoArtifacts(t *testing.T) { + for phase, want := range backupPhaseExpectations { + t.Run(string(phase), func(t *testing.T) { + assert.Equal(t, want, backupPhaseHasNoArtifacts(phase)) + }) + } + + // A backup that has not been reconciled yet has an empty phase. It is left alone + // deliberately: the state is transient and the caller can retry. + assert.False(t, backupPhaseHasNoArtifacts(velerov1api.BackupPhase(""))) +} + +func TestRestorePhaseHasNoArtifacts(t *testing.T) { + for phase, want := range restorePhaseExpectations { + t.Run(string(phase), func(t *testing.T) { + assert.Equal(t, want, restorePhaseHasNoArtifacts(phase)) + }) + } + + assert.False(t, restorePhaseHasNoArtifacts(velerov1api.RestorePhase(""))) +} + +// The two tests below read the phase enum out of the generated CRDs, which come from the +// same kubebuilder markers as the Go constants. Adding a phase to the API without +// classifying it here fails, which a hand-written list of phases cannot do. +func TestBackupPhaseExpectationsCoverTheCRD(t *testing.T) { + phases := statusPhaseEnum(t, "backups.velero.io") + require.NotEmpty(t, phases, "no status.phase enum found in the Backup CRD") + + for _, phase := range phases { + _, ok := backupPhaseExpectations[velerov1api.BackupPhase(phase)] + assert.True(t, ok, "BackupPhase %q is served by the CRD but not classified by backupPhaseHasNoArtifacts", phase) + } + assert.Len(t, backupPhaseExpectations, len(phases), "backupPhaseExpectations and the CRD enum have drifted apart") +} + +func TestRestorePhaseExpectationsCoverTheCRD(t *testing.T) { + phases := statusPhaseEnum(t, "restores.velero.io") + require.NotEmpty(t, phases, "no status.phase enum found in the Restore CRD") + + for _, phase := range phases { + _, ok := restorePhaseExpectations[velerov1api.RestorePhase(phase)] + assert.True(t, ok, "RestorePhase %q is served by the CRD but not classified by restorePhaseHasNoArtifacts", phase) + } + assert.Len(t, restorePhaseExpectations, len(phases), "restorePhaseExpectations and the CRD enum have drifted apart") +} + +// statusPhaseEnum returns the allowed values of status.phase for a generated CRD. +func statusPhaseEnum(t *testing.T, crdName string) []string { + t.Helper() + + for _, crd := range v1crds.CRDs { + if crd.Name != crdName { + continue + } + for _, version := range crd.Spec.Versions { + if version.Schema == nil || version.Schema.OpenAPIV3Schema == nil { + continue + } + status, ok := version.Schema.OpenAPIV3Schema.Properties["status"] + if !ok { + continue + } + phase, ok := status.Properties["phase"] + if !ok { + continue + } + + values := make([]string, 0, len(phase.Enum)) + for _, raw := range phase.Enum { + var value string + require.NoError(t, json.Unmarshal(raw.Raw, &value)) + values = append(values, value) + } + return values + } + } + + t.Fatalf("CRD %q not found", crdName) + return nil +} + +// The guard is only useful if a caller can tell why it fired. These reconcile a real +// request against a fake client and assert on what a client would actually observe. +func TestGuardSetsFailedPhaseWithReason(t *testing.T) { + tests := []struct { + name string + targetKind velerov1api.DownloadTargetKind + backupPhase velerov1api.BackupPhase + wantPhase velerov1api.DownloadRequestPhase + wantMessage string + }{ + { + name: "backup that never ran fails with the phase named", + targetKind: velerov1api.DownloadTargetKindBackupLog, + backupPhase: velerov1api.BackupPhaseFailedValidation, + wantPhase: velerov1api.DownloadRequestPhaseFailed, + wantMessage: `backup "a-backup" is in phase "FailedValidation" and has not written any artifacts`, + }, + { + name: "backup that ran is untouched by the guard", + targetKind: velerov1api.DownloadTargetKindBackupLog, + backupPhase: velerov1api.BackupPhaseCompleted, + wantPhase: velerov1api.DownloadRequestPhaseProcessed, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + harness := newDownloadRequestHarness(t, tc.targetKind, tc.backupPhase) + got := harness.reconcile(t) + + assert.Equal(t, tc.wantPhase, got.Status.Phase) + assert.Equal(t, tc.wantMessage, got.Status.Message, + "the message is the only thing telling a caller why no URL arrived") + + if tc.wantPhase == velerov1api.DownloadRequestPhaseFailed { + assert.Empty(t, got.Status.DownloadURL, + "a failed request must not carry a URL that would 404") + } + }) + } +} + +// A message is set only on failure. An empty message alongside Failed would put the CLI +// back on its generic storage-location error, which is the thing this replaces. +func TestFailedPhaseAlwaysCarriesAMessage(t *testing.T) { + harness := newDownloadRequestHarness(t, + velerov1api.DownloadTargetKindBackupLog, velerov1api.BackupPhaseNew) + + got := harness.reconcile(t) + + require.Equal(t, velerov1api.DownloadRequestPhaseFailed, got.Status.Phase) + assert.NotEmpty(t, got.Status.Message) +} + +// downloadRequestHarness builds the smallest cluster a DownloadRequest reconcile needs: +// the request, its backup, and a storage location whose store returns a URL. +type downloadRequestHarness struct { + client kbclient.Client + reqName string + r *downloadRequestReconciler +} + +func newDownloadRequestHarness( + t *testing.T, + kind velerov1api.DownloadTargetKind, + backupPhase velerov1api.BackupPhase, +) *downloadRequestHarness { + t.Helper() + + s := runtime.NewScheme() + require.NoError(t, velerov1api.AddToScheme(s)) + + backup := builder.ForBackup(velerov1api.DefaultNamespace, "a-backup"). + StorageLocation("a-location").Phase(backupPhase).Result() + location := builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "a-location").Result() + request := builder.ForDownloadRequest(velerov1api.DefaultNamespace, "a-request"). + Target(kind, "a-backup").Result() + + c := fake.NewClientBuilder().WithScheme(s). + WithObjects(request, backup, location).Build() + + store := &persistencemocks.BackupStore{} + store.On("GetDownloadURL", request.Spec.Target).Return("a-url", nil) + + pluginManager := &pluginmocks.Manager{} + pluginManager.On("CleanupClients").Return(nil) + + r := NewDownloadRequestReconciler( + c, + testclocks.NewFakeClock(time.Now()), + func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager }, + NewFakeObjectBackupStoreGetter(map[string]*persistencemocks.BackupStore{"a-location": store}), + velerotest.NewLogger(), + nil, + nil, + ) + + return &downloadRequestHarness{client: c, reqName: request.Name, r: r} +} + +func (h *downloadRequestHarness) reconcile(t *testing.T) *velerov1api.DownloadRequest { + t.Helper() + + _, err := h.r.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: kbclient.ObjectKey{ + Namespace: velerov1api.DefaultNamespace, + Name: h.reqName, + }, + }) + require.NoError(t, err) + + got := &velerov1api.DownloadRequest{} + require.NoError(t, h.client.Get(context.Background(), kbclient.ObjectKey{ + Namespace: velerov1api.DefaultNamespace, Name: h.reqName, + }, got)) + return got +} diff --git a/pkg/controller/pod_volume_backup_controller.go b/pkg/controller/pod_volume_backup_controller.go index 2372bf25b..13dbd5d79 100644 --- a/pkg/controller/pod_volume_backup_controller.go +++ b/pkg/controller/pod_volume_backup_controller.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "strings" + "sync" "time" "github.com/cockroachdb/errors" @@ -89,7 +90,6 @@ func NewPodVolumeBackupReconciler( preparingTimeout: preparingTimeout, resourceTimeout: resourceTimeout, exposer: exposer.NewPodVolumeExposer(kubeClient, logger), - cancelledPVB: make(map[string]time.Time), dataMovePriorityClass: dataMovePriorityClass, privileged: privileged, podLabels: podLabels, @@ -112,7 +112,7 @@ type PodVolumeBackupReconciler struct { vgdpCounter *exposer.VgdpCounter preparingTimeout time.Duration resourceTimeout time.Duration - cancelledPVB map[string]time.Time + cancelledPVB sync.Map dataMovePriorityClass string privileged bool podLabels map[string]string @@ -183,7 +183,7 @@ func (r *PodVolumeBackupReconciler) Reconcile(ctx context.Context, req ctrl.Requ } } } else { - delete(r.cancelledPVB, pvb.Name) + r.cancelledPVB.Delete(pvb.Name) if controllerutil.ContainsFinalizer(pvb, PodVolumeFinalizer) { if err := UpdatePVBWithRetry(ctx, r.client, req.NamespacedName, log, func(pvb *velerov1api.PodVolumeBackup) bool { @@ -204,9 +204,9 @@ func (r *PodVolumeBackupReconciler) Reconcile(ctx context.Context, req ctrl.Requ } if pvb.Spec.Cancel { - if spotted, found := r.cancelledPVB[pvb.Name]; !found { - r.cancelledPVB[pvb.Name] = r.clock.Now() - } else { + v, loaded := r.cancelledPVB.LoadOrStore(pvb.Name, r.clock.Now()) + if loaded { + spotted := v.(time.Time) delay := cancelDelayOthers if pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseInProgress { delay = cancelDelayInProgress @@ -215,7 +215,7 @@ func (r *PodVolumeBackupReconciler) Reconcile(ctx context.Context, req ctrl.Requ if time.Since(spotted) > delay { log.Infof("PVB %s is canceled in Phase %s but not handled in reasonable time", pvb.GetName(), pvb.Status.Phase) if r.tryCancelPodVolumeBackup(ctx, pvb, "") { - delete(r.cancelledPVB, pvb.Name) + r.cancelledPVB.Delete(pvb.Name) } return ctrl.Result{}, nil @@ -620,7 +620,7 @@ func (r *PodVolumeBackupReconciler) OnDataPathCancelled(ctx context.Context, nam }); err != nil { log.WithError(err).Error("error updating PVB status on cancel") } else { - delete(r.cancelledPVB, pvb.Name) + r.cancelledPVB.Delete(pvb.Name) } } diff --git a/pkg/controller/pod_volume_backup_controller_test.go b/pkg/controller/pod_volume_backup_controller_test.go index 8b05f0e3b..21e30d5db 100644 --- a/pkg/controller/pod_volume_backup_controller_test.go +++ b/pkg/controller/pod_volume_backup_controller_test.go @@ -19,9 +19,12 @@ package controller import ( "context" "fmt" + "sync" "testing" "time" + clocktesting "k8s.io/utils/clock/testing" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" @@ -489,7 +492,7 @@ func TestPVBReconcile(t *testing.T) { } if test.sportTime != nil { - r.cancelledPVB[test.pvb.Name] = test.sportTime.Time + r.cancelledPVB.Store(test.pvb.Name, test.sportTime.Time) } if test.constrained { @@ -567,9 +570,15 @@ func TestPVBReconcile(t *testing.T) { } if test.expectCancelRecord { - assert.Contains(t, r.cancelledPVB, test.pvb.Name) + _, ok := r.cancelledPVB.Load(test.pvb.Name) + assert.True(t, ok) } else { - assert.Empty(t, r.cancelledPVB) + empty := true + r.cancelledPVB.Range(func(key, value any) bool { + empty = false + return false + }) + assert.True(t, empty) } if isPVBInFinalState(&pvb) || pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseInProgress { @@ -1308,3 +1317,50 @@ func TestPodVolumeBackupSetupExposeParam(t *testing.T) { }) } } + +type pvbSequenceClock struct { + *clocktesting.FakeClock + mu sync.Mutex +} + +func (c *pvbSequenceClock) Now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + c.FakeClock.Step(time.Second) + return c.FakeClock.Now() +} + +func TestPodVolumeBackupCancelConcurrency(t *testing.T) { + ctx := t.Context() + pvb := builder.ForPodVolumeBackup(velerov1api.DefaultNamespace, "pvb-1").Cancel(true).Phase(velerov1api.PodVolumeBackupPhaseInProgress).Result() + + r, err := initPVBReconciler() + require.NoError(t, err) + + err = r.client.Create(ctx, pvb) + require.NoError(t, err) + + firstTime := time.Now() + // manually store the initial time + r.cancelledPVB.Store(pvb.Name, firstTime) + + // Custom clock that returns a different time each call + r.clock = &pvbSequenceClock{FakeClock: clocktesting.NewFakeClock(firstTime)} + + var wg sync.WaitGroup + routines := 50 + wg.Add(routines) + + for i := 0; i < routines; i++ { + go func() { + defer wg.Done() + _, _ = r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: pvb.Name, Namespace: pvb.Namespace}}) + }() + } + + wg.Wait() + + v, ok := r.cancelledPVB.Load(pvb.Name) + assert.True(t, ok) + assert.Equal(t, firstTime, v.(time.Time), "The initially recorded timestamp should be preserved") +} diff --git a/pkg/controller/pod_volume_restore_controller.go b/pkg/controller/pod_volume_restore_controller.go index ca25b4f95..b6d4985fa 100644 --- a/pkg/controller/pod_volume_restore_controller.go +++ b/pkg/controller/pod_volume_restore_controller.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "strings" + "sync" "time" "github.com/cockroachdb/errors" @@ -90,7 +91,6 @@ func NewPodVolumeRestoreReconciler( preparingTimeout: preparingTimeout, resourceTimeout: resourceTimeout, exposer: exposer.NewPodVolumeExposer(kubeClient, logger), - cancelledPVR: make(map[string]time.Time), dataMovePriorityClass: dataMovePriorityClass, privileged: privileged, repoConfigMgr: repoConfigMgr, @@ -114,7 +114,7 @@ type PodVolumeRestoreReconciler struct { vgdpCounter *exposer.VgdpCounter preparingTimeout time.Duration resourceTimeout time.Duration - cancelledPVR map[string]time.Time + cancelledPVR sync.Map dataMovePriorityClass string privileged bool repoConfigMgr repository.ConfigManager @@ -188,7 +188,7 @@ func (r *PodVolumeRestoreReconciler) Reconcile(ctx context.Context, req ctrl.Req } } } else { - delete(r.cancelledPVR, pvr.Name) + r.cancelledPVR.Delete(pvr.Name) if controllerutil.ContainsFinalizer(pvr, PodVolumeFinalizer) { if err := UpdatePVRWithRetry(ctx, r.client, req.NamespacedName, log, func(pvr *velerov1api.PodVolumeRestore) bool { @@ -209,9 +209,9 @@ func (r *PodVolumeRestoreReconciler) Reconcile(ctx context.Context, req ctrl.Req } if pvr.Spec.Cancel { - if spotted, found := r.cancelledPVR[pvr.Name]; !found { - r.cancelledPVR[pvr.Name] = r.clock.Now() - } else { + v, loaded := r.cancelledPVR.LoadOrStore(pvr.Name, r.clock.Now()) + if loaded { + spotted := v.(time.Time) delay := cancelDelayOthers if pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseInProgress { delay = cancelDelayInProgress @@ -220,7 +220,7 @@ func (r *PodVolumeRestoreReconciler) Reconcile(ctx context.Context, req ctrl.Req if time.Since(spotted) > delay { log.Infof("PVR %s is canceled in Phase %s but not handled in rasonable time", pvr.GetName(), pvr.Status.Phase) if r.tryCancelPodVolumeRestore(ctx, pvr, "") { - delete(r.cancelledPVR, pvr.Name) + r.cancelledPVR.Delete(pvr.Name) } return ctrl.Result{}, nil @@ -236,7 +236,15 @@ func (r *PodVolumeRestoreReconciler) Reconcile(ctx context.Context, req ctrl.Req return ctrl.Result{}, nil } - shouldProcess, pod, err := shouldProcess(ctx, r.client, log, pvr, r.resourceTimeout) + pod, err := getTargetPod(ctx, r.client, log, pvr) + if err != nil { + return ctrl.Result{}, err + } + if pod == nil { + return ctrl.Result{}, nil + } + + shouldProcess, err := shouldProcess(pod, log) if err != nil { return r.errorOut(ctx, pvr, err, "Pod for this PVR is not ready", log) } @@ -255,12 +263,6 @@ func (r *PodVolumeRestoreReconciler) Reconcile(ctx context.Context, req ctrl.Req return ctrl.Result{}, errors.Wrapf(err, "error accepting PVR %s", pvr.Name) } - initContainerIndex := getInitContainerIndex(pod) - if initContainerIndex > 0 { - log.Warnf(`Init containers before the %s container may cause issues - if they interfere with volumes being restored: %s index %d`, restorehelper.WaitInitContainer, restorehelper.WaitInitContainer, initContainerIndex) - } - log.Info("Exposing PVR") exposeParam := r.setupExposeParam(pvr) @@ -565,71 +567,61 @@ func UpdatePVRStatusToFailed(ctx context.Context, c client.Client, pvr *velerov1 return err } -func shouldProcess(ctx context.Context, client client.Client, log logrus.FieldLogger, pvr *velerov1api.PodVolumeRestore, timeout time.Duration) (bool, *corev1api.Pod, error) { - if !isPVRNew(pvr) { - log.Debug("PVR is not new, skip") - return false, nil, nil - } - +func getTargetPod(ctx context.Context, client client.Client, log logrus.FieldLogger, pvr *velerov1api.PodVolumeRestore) (*corev1api.Pod, error) { // we filter the pods during the initialization of cache, if we can get a pod here, the pod must be in the same node with the controller // so we don't need to compare the node anymore - var targetPod *corev1api.Pod - err := wait.PollUntilContextTimeout(ctx, time.Millisecond*100, timeout, true, func(ctx context.Context) (bool, error) { - updated := &corev1api.Pod{} - if err := client.Get(ctx, types.NamespacedName{Namespace: pvr.Spec.Pod.Namespace, Name: pvr.Spec.Pod.Name}, updated); err != nil { - if apierrors.IsNotFound(err) { - return false, nil - } - - return false, err - } - - targetPod = updated - - return true, nil - }) - - if err != nil { - if errors.Is(err, context.DeadlineExceeded) { - return false, nil, errors.Errorf("timeout to wait for pod %s/%s", pvr.Spec.Pod.Namespace, pvr.Spec.Pod.Name) - } else { - return false, nil, errors.Wrapf(err, "error waiting for pod %s/%s", pvr.Spec.Pod.Namespace, pvr.Spec.Pod.Name) + pod := &corev1api.Pod{} + if err := client.Get(ctx, types.NamespacedName{Namespace: pvr.Spec.Pod.Namespace, Name: pvr.Spec.Pod.Name}, pod); err != nil { + if apierrors.IsNotFound(err) { + log.WithError(err).Debug("Pod not found on this node, skip") + return nil, nil } + log.WithError(err).Error("Unable to get pod") + return nil, err } + return pod, nil +} + +func shouldProcess(targetPod *corev1api.Pod, log logrus.FieldLogger) (bool, error) { if targetPod.Status.Phase == corev1api.PodFailed || targetPod.Status.Phase == corev1api.PodUnknown { - return false, nil, errors.Errorf("unexpected state for pod %s/%s", targetPod.Namespace, targetPod.Name) + return false, errors.Errorf("unexpected state for pod %s/%s", targetPod.Namespace, targetPod.Name) } idx := getInitContainerIndex(targetPod) if idx < 0 { - return false, nil, errors.Errorf("no restore-wait init container in pod %s/%s", targetPod.Namespace, targetPod.Name) + return false, errors.Errorf("no restore-wait init container in pod %s/%s", targetPod.Namespace, targetPod.Name) } if len(targetPod.Status.InitContainerStatuses) <= idx { log.Debug("Pod init container statuses are not fully populated yet, skip") - return false, nil, nil + return false, nil + } + + if idx > 0 { + log.Warnf(`Init containers before the %s container may cause issues + if they interfere with volumes being restored: %s index %d`, restorehelper.WaitInitContainer, restorehelper.WaitInitContainer, idx) } containerStatus := targetPod.Status.InitContainerStatuses[idx] if containerStatus.State.Terminated != nil { - return false, nil, errors.Errorf("restore-wait init container has already completed in pod %s/%s", targetPod.Namespace, targetPod.Name) + return false, errors.Errorf("restore-wait init container has already completed in pod %s/%s", targetPod.Namespace, targetPod.Name) } if containerStatus.State.Waiting != nil { reason := containerStatus.State.Waiting.Reason if reason == "ImagePullBackOff" || reason == "ErrImageNeverPull" || reason == "CreateContainerConfigError" || reason == "CreateContainerError" || reason == "InvalidImageName" || reason == "ErrImagePull" { - return false, nil, errors.Errorf("restore-wait init container in pod %s/%s is in unrecoverable waiting state with reason %s", targetPod.Namespace, targetPod.Name, reason) + return false, errors.Errorf("restore-wait init container in pod %s/%s is in unrecoverable waiting state with reason %s", targetPod.Namespace, targetPod.Name, reason) } } if containerStatus.State.Running == nil { log.Debug("Pod is not running restore-wait init container, skip") - return false, nil, nil + return false, nil } - return true, targetPod, nil + return true, nil } func (r *PodVolumeRestoreReconciler) closeDataPath(ctx context.Context, pvrName string) { @@ -903,7 +895,7 @@ func (r *PodVolumeRestoreReconciler) OnDataPathCancelled(ctx context.Context, na }); err != nil { log.WithError(err).Error("error updating PVR status on cancel") } else { - delete(r.cancelledPVR, pvr.Name) + r.cancelledPVR.Delete(pvr.Name) } } diff --git a/pkg/controller/pod_volume_restore_controller_test.go b/pkg/controller/pod_volume_restore_controller_test.go index abd2df206..73167c76f 100644 --- a/pkg/controller/pod_volume_restore_controller_test.go +++ b/pkg/controller/pod_volume_restore_controller_test.go @@ -19,9 +19,12 @@ package controller import ( "context" "fmt" + "sync" "testing" "time" + clocktesting "k8s.io/utils/clock/testing" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" @@ -117,8 +120,6 @@ func TestShouldProcess(t *testing.T) { }, }, shouldProcessed: false, - expectError: true, - errString: "timeout to wait for pod ns-1/pod-1", }, { name: "Empty phase pvr with pod on node not running init container should not be processed", @@ -470,8 +471,6 @@ func TestShouldProcess(t *testing.T) { for _, ts := range tests { t.Run(ts.name, func(t *testing.T) { - ctx := t.Context() - var objs []runtime.Object if ts.obj != nil { objs = append(objs, ts.obj) @@ -487,7 +486,21 @@ func TestShouldProcess(t *testing.T) { clock: &clocks.RealClock{}, } - shouldProcess, _, err := shouldProcess(ctx, c.client, c.logger, ts.obj, time.Second) + if !isPVRNew(ts.obj) { + require.False(t, ts.shouldProcessed) + return + } + + if ts.pod == nil { + _, err := getTargetPod(context.Background(), c.client, c.logger, ts.obj) + if ts.expectError { + require.Error(t, err) + } + require.False(t, ts.shouldProcessed) + return + } + + shouldProcess, err := shouldProcess(ts.pod, c.logger) require.Equal(t, ts.shouldProcessed, shouldProcess) if ts.expectError { require.Error(t, err) @@ -1077,7 +1090,7 @@ func TestPodVolumeRestoreReconcile(t *testing.T) { } if test.sportTime != nil { - r.cancelledPVR[test.pvr.Name] = test.sportTime.Time + r.cancelledPVR.Store(test.pvr.Name, test.sportTime.Time) } if test.constrained { @@ -1198,9 +1211,15 @@ func TestPodVolumeRestoreReconcile(t *testing.T) { } if test.expectCancelRecord { - assert.Contains(t, r.cancelledPVR, test.pvr.Name) + _, ok := r.cancelledPVR.Load(test.pvr.Name) + assert.True(t, ok) } else { - assert.Empty(t, r.cancelledPVR) + empty := true + r.cancelledPVR.Range(func(key, value any) bool { + empty = false + return false + }) + assert.True(t, empty) } if isPVRInFinalState(&pvr) || pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseInProgress { @@ -1925,3 +1944,47 @@ func TestResumeCancellablePodVolumeRestore(t *testing.T) { }) } } + +type pvrSequenceClock struct { + *clocktesting.FakeClock + mu sync.Mutex +} + +func (c *pvrSequenceClock) Now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + c.FakeClock.Step(time.Second) + return c.FakeClock.Now() +} + +func TestPodVolumeRestoreCancelConcurrency(t *testing.T) { + ctx := t.Context() + pvr := builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, "pvr-1").Cancel(true).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Result() + + r, err := initPodVolumeRestoreReconciler(nil, []client.Object{pvr}) + require.NoError(t, err) + + firstTime := time.Now() + // manually store the initial time + r.cancelledPVR.Store(pvr.Name, firstTime) + + // Custom clock that returns a different time each call + r.clock = &pvrSequenceClock{FakeClock: clocktesting.NewFakeClock(firstTime)} + + var wg sync.WaitGroup + routines := 50 + wg.Add(routines) + + for i := 0; i < routines; i++ { + go func() { + defer wg.Done() + _, _ = r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: pvr.Name, Namespace: pvr.Namespace}}) + }() + } + + wg.Wait() + + v, ok := r.cancelledPVR.Load(pvr.Name) + assert.True(t, ok) + assert.Equal(t, firstTime, v.(time.Time), "The initially recorded timestamp should be preserved") +} diff --git a/pkg/controller/restore_finalizer_controller.go b/pkg/controller/restore_finalizer_controller.go index 4e02bb0ef..d9acd0a09 100644 --- a/pkg/controller/restore_finalizer_controller.go +++ b/pkg/controller/restore_finalizer_controller.go @@ -39,6 +39,7 @@ import ( "github.com/vmware-tanzu/velero/internal/hook" "github.com/vmware-tanzu/velero/internal/volume" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + serverconfig "github.com/vmware-tanzu/velero/pkg/cmd/server/config" "github.com/vmware-tanzu/velero/pkg/constant" "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/metrics" @@ -374,19 +375,20 @@ func (ctx *finalizerContext) patchDynamicPVWithVolumeInfo() (errs results.Result // failures due to the PVC not being bound, which could cause a timeout and result in a failed restore. if pvc.Status.Phase == corev1api.ClaimPending { // check if storage class used has VolumeBindingMode as WaitForFirstConsumer - scName := *pvc.Spec.StorageClassName - sc := &storagev1api.StorageClass{} - err = ctx.crClient.Get(context.Background(), client.ObjectKey{Name: scName}, sc) + if pvc.Spec.StorageClassName != nil && *pvc.Spec.StorageClassName != "" { + scName := *pvc.Spec.StorageClassName + sc := &storagev1api.StorageClass{} + err = ctx.crClient.Get(context.Background(), client.ObjectKey{Name: scName}, sc) - if err != nil { - errs.Add(restoredNamespace, err) - return false, err - } - // skip PV patch step for this scenario - // because pvc would not be bound and the PV patch step would fail due to timeout thus failing the restore - if *sc.VolumeBindingMode == storagev1api.VolumeBindingWaitForFirstConsumer { - log.Warnf("skipping PV patch to restore custom reclaim policy, if any: StorageClass %s used by PVC %s has VolumeBindingMode set to WaitForFirstConsumer, and the PVC is also in a pending state", scName, pvc.Name) - return true, nil + if err != nil { + return false, err + } + // skip PV patch step for this scenario + // because pvc would not be bound and the PV patch step would fail due to timeout thus failing the restore + if sc.VolumeBindingMode != nil && *sc.VolumeBindingMode == storagev1api.VolumeBindingWaitForFirstConsumer { + log.Warnf("skipping PV patch to restore custom reclaim policy, if any: StorageClass %s used by PVC %s has VolumeBindingMode set to WaitForFirstConsumer, and the PVC is also in a pending state", scName, pvc.Name) + return true, nil + } } } @@ -576,8 +578,18 @@ func (ctx *finalizerContext) WaitRestoreExecHook() (errs results.Result) { log := ctx.logger.WithField("restore", ctx.restore.Name) log.Info("Waiting for restore exec hooks starts") - // wait for restore exec hooks to finish - err := wait.PollUntilContextCancel(context.Background(), 1*time.Second, true, func(context.Context) (bool, error) { + // Bound the wait by resourceTimeout (the same budget Velero already + // applies to other finalizer phases). Previously this poll had no + // deadline, so a hook that was registered via Add() but never + // recorded as executed left the restore stuck in Finalizing forever + // and blocked every other restore on the cluster. + timeout := ctx.resourceTimeout + if timeout <= 0 { + timeout = serverconfig.DefaultResourceTimeout + } + pollCtx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + err := wait.PollUntilContextCancel(pollCtx, 1*time.Second, true, func(context.Context) (bool, error) { log.Debug("Checking the progress of hooks execution") if ctx.multiHookTracker.IsComplete(ctx.restore.Name) { return true, nil diff --git a/pkg/controller/restore_finalizer_controller_test.go b/pkg/controller/restore_finalizer_controller_test.go index 8f2618f9d..226a2283c 100644 --- a/pkg/controller/restore_finalizer_controller_test.go +++ b/pkg/controller/restore_finalizer_controller_test.go @@ -482,6 +482,10 @@ func TestWaitRestoreExecHook(t *testing.T) { hookFailed, hookErr := true, fmt.Errorf("hook failed") hookTracker3.Add(restoreName3, podNs, podName, container, source, hookName, hook.PhasePre, 0) + hookTracker4 := hook.NewMultiHookTracker() + restoreName4 := "restore4" + hookTracker4.Add(restoreName4, "ns", "pod", "con1", "s1", "h1", hook.PhasePre, 0) + tests := []struct { name string hookTracker *hook.MultiHookTracker @@ -497,6 +501,8 @@ func TestWaitRestoreExecHook(t *testing.T) { hookName string hookFailed bool hookErr error + resourceTimeout time.Duration + expectTimeoutErr bool }{ { name: "no restore exec hooks", @@ -530,6 +536,16 @@ func TestWaitRestoreExecHook(t *testing.T) { hookFailed: hookFailed, hookErr: hookErr, }, + { + name: "hook never recorded should timeout instead of hanging", + hookTracker: hookTracker4, + restore: builder.ForRestore(velerov1api.DefaultNamespace, restoreName4).Result(), + expectedHooksAttempted: 0, + expectedHooksFailed: 0, + expectedHookErrs: 1, + resourceTimeout: 3 * time.Second, + expectTimeoutErr: true, + }, } for _, tc := range tests { @@ -542,6 +558,7 @@ func TestWaitRestoreExecHook(t *testing.T) { crClient: fakeClient, restore: tc.restore, multiHookTracker: tc.hookTracker, + resourceTimeout: tc.resourceTimeout, } require.NoError(t, ctx.crClient.Create(t.Context(), tc.restore)) @@ -553,6 +570,10 @@ func TestWaitRestoreExecHook(t *testing.T) { } errs := ctx.WaitRestoreExecHook() + if tc.expectTimeoutErr { + assert.NotEmpty(t, errs.Namespaces, "expected timeout error but got none") + continue + } assert.Len(t, errs.Namespaces, tc.expectedHookErrs) updated := &velerov1api.Restore{} @@ -676,11 +697,11 @@ func TestNeedPatch(t *testing.T) { { name: "same label key different values", newPV: builder.ForPersistentVolume("pv1"). - ObjectMeta(builder.WithLabels("topology.kubernetes.io/zone", "us-west-2a")). + ObjectMeta(builder.WithLabels(corev1api.LabelTopologyZone, "us-west-2a")). ReclaimPolicy(corev1api.PersistentVolumeReclaimDelete).Result(), pvInfo: &volume.PVInfo{ ReclaimPolicy: string(corev1api.PersistentVolumeReclaimDelete), - Labels: map[string]string{"topology.kubernetes.io/zone": "us-east-1a"}, + Labels: map[string]string{corev1api.LabelTopologyZone: "us-east-1a"}, }, expected: false, }, diff --git a/pkg/controller/schedule_controller.go b/pkg/controller/schedule_controller.go index d71c86ca4..2e707bdc0 100644 --- a/pkg/controller/schedule_controller.go +++ b/pkg/controller/schedule_controller.go @@ -111,7 +111,11 @@ func (c *scheduleReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c original := schedule.DeepCopy() if schedule.Spec.SkipImmediately == nil { - schedule.Spec.SkipImmediately = &c.skipImmediately + // Copy the value rather than aliasing &c.skipImmediately: c is a long-lived + // singleton reconciler, and the block below can write through this pointer, + // which would otherwise mutate the reconciler's shared default field. + skipImmediately := c.skipImmediately + schedule.Spec.SkipImmediately = &skipImmediately } if schedule.Spec.SkipImmediately != nil && *schedule.Spec.SkipImmediately { *schedule.Spec.SkipImmediately = false diff --git a/pkg/controller/schedule_controller_test.go b/pkg/controller/schedule_controller_test.go index 85b87474a..1c134d2bc 100644 --- a/pkg/controller/schedule_controller_test.go +++ b/pkg/controller/schedule_controller_test.go @@ -246,6 +246,54 @@ func parseTime(timeString string) time.Time { return res } +// TestReconcileDoesNotCorruptReconcilerSkipImmediately guards against a regression where +// aliasing &c.skipImmediately into a Schedule's spec (when SkipImmediately is nil) let a +// subsequent write-through-pointer mutate the reconciler's own shared default field, +// silently corrupting it for every later reconcile in the process. +func TestReconcileDoesNotCorruptReconcilerSkipImmediately(t *testing.T) { + require.NoError(t, velerov1.AddToScheme(scheme.Scheme)) + + client := fake.NewClientBuilder().WithScheme(scheme.Scheme).Build() + logger := velerotest.NewLogger() + + // Server configured with schedule-skip-immediately=true. + reconciler := NewScheduleReconciler("ns", logger, client, metrics.NewServerMetrics(), true) + reconciler.clock = testclocks.NewFakeClock(time.Now()) + + makeSchedule := func(name string) *velerov1.Schedule { + return builder.ForSchedule("ns", name). + Phase(velerov1.SchedulePhaseEnabled). + CronSchedule("@every 5m"). + LastBackupTime("2000-01-01 00:00:00"). // long past due, but should be skipped + Result() // SkipImmediately left nil + } + + sched1 := makeSchedule("sched-1") + require.NoError(t, client.Create(ctx, sched1)) + _, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "ns", Name: "sched-1"}}) + require.NoError(t, err) + + // The reconciler's own default must be unchanged after processing a schedule with a nil + // SkipImmediately -- every later schedule relies on this field still being true. + assert.True(t, reconciler.skipImmediately, "reconciler's shared skipImmediately default was mutated by reconciling sched-1") + + sched2 := makeSchedule("sched-2") + require.NoError(t, client.Create(ctx, sched2)) + _, err = reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "ns", Name: "sched-2"}}) + require.NoError(t, err) + + assert.True(t, reconciler.skipImmediately, "reconciler's shared skipImmediately default was mutated by reconciling sched-2") + + // Functional check: sched-2 should ALSO have been skipped (server default still true), + // proving the bug's user-visible symptom (second+ schedule silently loses the skip + // behavior) is fixed, not just the internal field. + got := &velerov1.Schedule{} + require.NoError(t, client.Get(ctx, types.NamespacedName{Namespace: "ns", Name: "sched-2"}, got)) + require.NotNil(t, got.Status.LastSkipped, "sched-2 should have been skipped due to server-wide skipImmediately default") + require.NotNil(t, got.Status.LastBackup) + assert.Equal(t, parseTime("2000-01-01 00:00:00").Unix(), got.Status.LastBackup.Unix(), "sched-2 should not have triggered a new backup") +} + func TestGetNextRunTime(t *testing.T) { defaultSchedule := func() *velerov1.Schedule { return builder.ForSchedule("velero", "schedule-1").CronSchedule("@every 5m").Result() diff --git a/pkg/datamover/backup_micro_service.go b/pkg/datamover/backup_micro_service.go index 7398d6480..5ac1ce6ee 100644 --- a/pkg/datamover/backup_micro_service.go +++ b/pkg/datamover/backup_micro_service.go @@ -184,7 +184,7 @@ func (r *BackupMicroService) RunCancelableDataPath(ctx context.Context) (string, return "", errors.Wrap(err, "error to create data path") } - log.Debug("Async fs br created") + log.Debug("Async br created") if err := dp.Init(ctx, &datapath.InitParam{ BSLName: du.Spec.BackupStorageLocation, @@ -198,18 +198,23 @@ func (r *BackupMicroService) RunCancelableDataPath(ctx context.Context) (string, return "", errors.Wrap(err, "error to initialize data path") } - log.Info("Async fs br init") + log.Info("Async br init") tags := map[string]string{ velerov1api.AsyncOperationIDLabel: du.Labels[velerov1api.AsyncOperationIDLabel], } - // Modify the ParentSnapshot to "" and ForceFull to true when ParentSnapshot is "none". + // "none" requests a full backup. "auto" requests that the data mover finds the most + // recent backup of the same volume as parent, which is what an empty ParentSnapshot + // already does, so both map to "". parentSnapshot := du.Spec.ParentSnapshot forceFull := false - if du.Spec.ParentSnapshot == veleroshared.DataUploadParentSnapshotNone { + switch du.Spec.ParentSnapshot { + case veleroshared.ParentSnapshotNone: parentSnapshot = "" forceFull = true + case veleroshared.ParentSnapshotAuto: + parentSnapshot = "" } if err := dp.StartBackup(r.sourceTargetPath, du.Spec.DataMoverConfig, &datapath.BackupStartParam{ @@ -231,7 +236,7 @@ func (r *BackupMicroService) RunCancelableDataPath(ctx context.Context) (string, result := "" select { case <-ctx.Done(): - err = errors.New("timed out waiting for fs backup to complete") + err = errors.New("timed out waiting for backup to complete") break case res := <-r.resultSignal: err = res.err @@ -315,9 +320,9 @@ func (r *BackupMicroService) OnDataUploadProgress(ctx context.Context, namespace } func (r *BackupMicroService) closeDataPath(ctx context.Context, duName string) { - fsBackup := r.dataPathMgr.GetAsyncBR(duName) - if fsBackup != nil { - fsBackup.Close(ctx) + asyncBR := r.dataPathMgr.GetAsyncBR(duName) + if asyncBR != nil { + asyncBR.Close(ctx) } r.dataPathMgr.RemoveAsyncBR(duName) @@ -328,11 +333,11 @@ func (r *BackupMicroService) cancelDataUpload(du *velerov2alpha1api.DataUpload) r.eventRecorder.Event(du, false, datapath.EventReasonCancelling, "Canceling for data upload %s", du.Name) - fsBackup := r.dataPathMgr.GetAsyncBR(du.Name) - if fsBackup == nil { + asyncBR := r.dataPathMgr.GetAsyncBR(du.Name) + if asyncBR == nil { r.OnDataUploadCancelled(r.ctx, du.GetNamespace(), du.GetName()) r.eventRecorder.EndingEvent(du, false, datapath.EventReasonStopped, "Data path for %s exited without start", du.Name) } else { - fsBackup.Cancel() + asyncBR.Cancel() } } diff --git a/pkg/datamover/backup_micro_service_test.go b/pkg/datamover/backup_micro_service_test.go index e6291244b..c9accdd77 100644 --- a/pkg/datamover/backup_micro_service_test.go +++ b/pkg/datamover/backup_micro_service_test.go @@ -32,6 +32,7 @@ import ( kbclient "sigs.k8s.io/controller-runtime/pkg/client" clientFake "sigs.k8s.io/controller-runtime/pkg/client/fake" + veleroshared "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" "github.com/vmware-tanzu/velero/pkg/builder" @@ -345,7 +346,7 @@ func TestRunCancelableDataPath(t *testing.T) { kubeClientObj: []runtime.Object{duInProgress}, dataPathStarted: true, expectedEventMsg: fmt.Sprintf("Data path for %s stopped", dataUploadName), - expectedErr: "timed out waiting for fs backup to complete", + expectedErr: "timed out waiting for backup to complete", }, { name: "data path returns error", @@ -445,3 +446,89 @@ func TestRunCancelableDataPath(t *testing.T) { cancel() } + +func TestRunCancelableDataPathParentSnapshot(t *testing.T) { + dataUploadName := "fake-data-upload" + + tests := []struct { + name string + parentSnapshot string + expectedParentSnapshot string + expectedForceFull bool + }{ + { + name: "empty lets the data mover search for a parent", + parentSnapshot: "", + expectedParentSnapshot: "", + expectedForceFull: false, + }, + { + name: "auto lets the data mover search for a parent", + parentSnapshot: veleroshared.ParentSnapshotAuto, + expectedParentSnapshot: "", + expectedForceFull: false, + }, + { + name: "none forces a full backup", + parentSnapshot: veleroshared.ParentSnapshotNone, + expectedParentSnapshot: "", + expectedForceFull: true, + }, + { + name: "a specific snapshot ID is passed through unchanged", + parentSnapshot: "fake-parent-snapshot-id", + expectedParentSnapshot: "fake-parent-snapshot-id", + expectedForceFull: false, + }, + } + + scheme := runtime.NewScheme() + velerov2alpha1api.AddToScheme(scheme) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + duInProgress := builder.ForDataUpload(velerov1api.DefaultNamespace, dataUploadName). + Phase(velerov2alpha1api.DataUploadPhaseInProgress). + CSISnapshot(&velerov2alpha1api.CSISnapshotSpec{VolumeSnapshot: "fake-snapshot"}). + Result() + duInProgress.Spec.ParentSnapshot = test.parentSnapshot + + fakeClient := clientFake.NewClientBuilder().WithScheme(scheme). + WithRuntimeObjects(duInProgress).Build() + + bs := &BackupMicroService{ + namespace: velerov1api.DefaultNamespace, + dataUploadName: dataUploadName, + ctx: t.Context(), + client: fakeClient, + dataPathMgr: datapath.NewManager(1), + eventRecorder: &backupMsTestHelper{}, + resultSignal: make(chan dataPathResult), + logger: velerotest.NewLogger(), + } + + var startParam *datapath.BackupStartParam + datapath.VGDPCreator = func(string, string, kbclient.Client, string, datapath.Callbacks, logrus.FieldLogger) datapath.AsyncBR { + fsBR := datapathmockes.NewAsyncBR(t) + fsBR.On("Init", mock.Anything, mock.Anything).Return(nil) + fsBR.On("StartBackup", mock.Anything, mock.Anything, mock.Anything). + Run(func(args mock.Arguments) { + startParam = args.Get(2).(*datapath.BackupStartParam) + }).Return(nil) + return fsBR + } + + go func() { + time.Sleep(time.Millisecond * 500) + bs.resultSignal <- dataPathResult{result: "fake-succeed-result"} + }() + + _, err := bs.RunCancelableDataPath(t.Context()) + require.NoError(t, err) + + require.NotNil(t, startParam) + assert.Equal(t, test.expectedParentSnapshot, startParam.ParentSnapshot) + assert.Equal(t, test.expectedForceFull, startParam.ForceFull) + }) + } +} diff --git a/pkg/datamover/restore_micro_service.go b/pkg/datamover/restore_micro_service.go index 799fa0add..a158a4216 100644 --- a/pkg/datamover/restore_micro_service.go +++ b/pkg/datamover/restore_micro_service.go @@ -178,7 +178,7 @@ func (r *RestoreMicroService) RunCancelableDataPath(ctx context.Context) (string }); err != nil { return "", errors.Wrap(err, "error to initialize data path") } - log.Info("fs init") + log.Info("Async br init") if err := dp.StartRestore(dd.Spec.SnapshotID, r.sourceTargetPath, dd.Spec.DataMoverConfig, &datapath.RestoreStartParam{}); err != nil { return "", errors.Wrap(err, "error starting data path restore") @@ -190,7 +190,7 @@ func (r *RestoreMicroService) RunCancelableDataPath(ctx context.Context) (string result := "" select { case <-ctx.Done(): - err = errors.New("timed out waiting for fs restore to complete") + err = errors.New("timed out waiting for restore to complete") break case res := <-r.resultSignal: err = res.err @@ -199,7 +199,7 @@ func (r *RestoreMicroService) RunCancelableDataPath(ctx context.Context) (string } if err != nil { - log.WithError(err).Error("Async fs restore was not completed") + log.WithError(err).Error("Async restore was not completed") } r.eventRecorder.EndingEvent(dd, false, datapath.EventReasonStopped, "Data path for %s stopped", dd.Name) @@ -272,9 +272,9 @@ func (r *RestoreMicroService) OnDataDownloadProgress(ctx context.Context, namesp } func (r *RestoreMicroService) closeDataPath(ctx context.Context, ddName string) { - fsRestore := r.dataPathMgr.GetAsyncBR(ddName) - if fsRestore != nil { - fsRestore.Close(ctx) + asyncBR := r.dataPathMgr.GetAsyncBR(ddName) + if asyncBR != nil { + asyncBR.Close(ctx) } r.dataPathMgr.RemoveAsyncBR(ddName) @@ -285,11 +285,11 @@ func (r *RestoreMicroService) cancelDataDownload(dd *velerov2alpha1api.DataDownl r.eventRecorder.Event(dd, false, datapath.EventReasonCancelling, "Canceling for data download %s", dd.Name) - fsBackup := r.dataPathMgr.GetAsyncBR(dd.Name) - if fsBackup == nil { + asyncBR := r.dataPathMgr.GetAsyncBR(dd.Name) + if asyncBR == nil { r.OnDataDownloadCancelled(r.ctx, dd.GetNamespace(), dd.GetName()) r.eventRecorder.EndingEvent(dd, false, datapath.EventReasonStopped, "Data path for %s exited without start", dd.Name) } else { - fsBackup.Cancel() + asyncBR.Cancel() } } diff --git a/pkg/datamover/restore_micro_service_test.go b/pkg/datamover/restore_micro_service_test.go index 39e055572..311c015a7 100644 --- a/pkg/datamover/restore_micro_service_test.go +++ b/pkg/datamover/restore_micro_service_test.go @@ -291,7 +291,7 @@ func TestRunCancelableRestore(t *testing.T) { kubeClientObj: []runtime.Object{ddInProgress}, dataPathStarted: true, expectedEventMsg: fmt.Sprintf("Data path for %s stopped", dataDownloadName), - expectedErr: "timed out waiting for fs restore to complete", + expectedErr: "timed out waiting for restore to complete", }, { name: "data path returns error", diff --git a/pkg/datapath/micro_service_watcher.go b/pkg/datapath/micro_service_watcher.go index 67ec4c29d..a824ea469 100644 --- a/pkg/datapath/micro_service_watcher.go +++ b/pkg/datapath/micro_service_watcher.go @@ -19,9 +19,11 @@ package datapath import ( "context" "encoding/json" + "fmt" "os" "strings" "sync" + "sync/atomic" "time" "github.com/cockroachdb/errors" @@ -54,6 +56,7 @@ const ( EventReasonProgress = "Data-Path-Progress" EventReasonCancelling = "Data-Path-Canceling" EventReasonStopped = "Data-Path-Stopped" + EventReasonEvicted = "Evicted" ) type microServiceBRWatcher struct { @@ -72,14 +75,15 @@ type microServiceBRWatcher struct { associatedObject string eventCh chan *corev1api.Event podCh chan *corev1api.Pod - startedFromEvent bool - terminatedFromEvent bool + startedFromEvent atomic.Bool + terminatedFromEvent atomic.Bool wgWatcher sync.WaitGroup eventInformer ctrlcache.Informer podInformer ctrlcache.Informer eventHandler cache.ResourceEventHandlerRegistration podHandler cache.ResourceEventHandlerRegistration watcherLock sync.Mutex + eventMessages sync.Map } func newMicroServiceBRWatcher(client client.Client, kubeClient kubernetes.Interface, mgr manager.Manager, taskType string, taskName string, namespace string, @@ -285,7 +289,7 @@ func (ms *microServiceBRWatcher) startWatch() { } epilogLoop: - for !ms.startedFromEvent || !ms.terminatedFromEvent { + for !ms.startedFromEvent.Load() || !ms.terminatedFromEvent.Load() { select { case <-ms.ctx.Done(): ms.log.Warn("Watch loop is canceled on waiting final event") @@ -303,11 +307,11 @@ func (ms *microServiceBRWatcher) startWatch() { logger.Infof("Finish waiting data path pod, phase %s, message %s", lastPod.Status.Phase, terminateMessage) - if !ms.startedFromEvent { + if !ms.startedFromEvent.Load() { logger.Warn("VGDP seems not started") } - if ms.startedFromEvent && !ms.terminatedFromEvent { + if ms.startedFromEvent.Load() && !ms.terminatedFromEvent.Load() { logger.Warn("VGDP started but termination event is not received") } @@ -326,8 +330,12 @@ func (ms *microServiceBRWatcher) startWatch() { } else { if strings.HasSuffix(terminateMessage, ErrCancelled) { ms.callbacks.OnCancelled(ms.ctx, ms.namespace, ms.taskName) - } else { + } else if terminateMessage != "" { ms.callbacks.OnFailed(ms.ctx, ms.namespace, ms.taskName, errors.New(terminateMessage)) + } else if msg, evicted := ms.eventMessages.Load(EventReasonEvicted); evicted { + ms.callbacks.OnFailed(ms.ctx, ms.namespace, ms.taskName, errors.New(msg.(string))) + } else { + ms.callbacks.OnFailed(ms.ctx, ms.namespace, ms.taskName, errors.New(lastPod.Status.Message)) } } @@ -338,7 +346,7 @@ func (ms *microServiceBRWatcher) startWatch() { func (ms *microServiceBRWatcher) onEvent(evt *corev1api.Event) { switch evt.Reason { case EventReasonStarted: - ms.startedFromEvent = true + ms.startedFromEvent.Store(true) ms.log.Infof("Received data path start message: %s", evt.Message) case EventReasonProgress: ms.callbacks.OnProgress(ms.ctx, ms.namespace, ms.taskName, funcGetProgressFromMessage(evt.Message, ms.log)) @@ -351,8 +359,11 @@ func (ms *microServiceBRWatcher) onEvent(evt *corev1api.Event) { case EventReasonCancelling: ms.log.Infof("Received data path canceling message: %s", evt.Message) case EventReasonStopped: - ms.terminatedFromEvent = true + ms.terminatedFromEvent.Store(true) ms.log.Infof("Received data path stop message: %s", evt.Message) + case EventReasonEvicted: + ms.eventMessages.Store(EventReasonEvicted, fmt.Sprintf("data path pod was evicted, message: %s", evt.Message)) + ms.log.Infof("Pod was evicted for data path %s, message: %s", ms.taskName, evt.Message) default: ms.log.Infof("Received event for data path %s, reason: %s, message: %s", ms.taskName, evt.Reason, evt.Message) } diff --git a/pkg/datapath/micro_service_watcher_test.go b/pkg/datapath/micro_service_watcher_test.go index 6724c290c..dee9560ae 100644 --- a/pkg/datapath/micro_service_watcher_test.go +++ b/pkg/datapath/micro_service_watcher_test.go @@ -120,6 +120,7 @@ type startWatchFake struct { redirectErr error complete bool failed bool + failedErr error canceled bool progress int } @@ -142,6 +143,7 @@ func (sw *startWatchFake) OnCompleted(ctx context.Context, namespace string, tas func (sw *startWatchFake) OnFailed(ctx context.Context, namespace string, task string, err error) { sw.failed = true + sw.failedErr = err } func (sw *startWatchFake) OnCancelled(ctx context.Context, namespace string, task string) { @@ -175,6 +177,7 @@ func TestStartWatch(t *testing.T) { expectComplete bool expectCancel bool expectFail bool + expectFailMsg string expectProgress int }{ { @@ -370,6 +373,27 @@ func TestStartWatch(t *testing.T) { expectTerminateEvent: true, expectCancel: true, }, + { + name: "evicted", + thisPod: "fak-pod-1", + thisContainer: "fake-container-1", + insertPod: builder.ForPod("velero", "fake-pod-1").Phase(corev1api.PodFailed).Result(), + insertEventsBefore: []insertEvent{ + { + event: &corev1api.Event{Reason: EventReasonStarted}, + }, + { + event: &corev1api.Event{Reason: EventReasonEvicted, Message: "fake-evicted-message"}, + }, + { + event: &corev1api.Event{Reason: EventReasonStopped}, + }, + }, + expectStartEvent: true, + expectTerminateEvent: true, + expectFail: true, + expectFailMsg: "data path pod was evicted, message: fake-evicted-message", + }, } for _, test := range tests { @@ -437,11 +461,14 @@ func TestStartWatch(t *testing.T) { ms.wgWatcher.Wait() - assert.Equal(t, test.expectStartEvent, ms.startedFromEvent) - assert.Equal(t, test.expectTerminateEvent, ms.terminatedFromEvent) + assert.Equal(t, test.expectStartEvent, ms.startedFromEvent.Load()) + assert.Equal(t, test.expectTerminateEvent, ms.terminatedFromEvent.Load()) assert.Equal(t, test.expectComplete, sw.complete) assert.Equal(t, test.expectCancel, sw.canceled) assert.Equal(t, test.expectFail, sw.failed) + if test.expectFailMsg != "" { + require.EqualError(t, sw.failedErr, test.expectFailMsg) + } assert.Equal(t, test.expectProgress, sw.progress) cancel() diff --git a/pkg/exposer/csi_snapshot.go b/pkg/exposer/csi_snapshot.go index 2e8e08889..30e299380 100644 --- a/pkg/exposer/csi_snapshot.go +++ b/pkg/exposer/csi_snapshot.go @@ -44,6 +44,11 @@ import ( "github.com/vmware-tanzu/velero/pkg/util/kube" ) +// BackupPVCSecretLabel is the label applied to secrets and configmaps copied to the +// Velero namespace for backup PVC provisioning. The value is the owning DataUpload/DataDownload +// UID, which is a stable, valid label value (the owner name may exceed the label-value limit). +const BackupPVCSecretLabel = "velero.io/backup-pvc-secret" //nolint:gosec // not a credential + // CSISnapshotExposeParam define the input param for Expose of CSI snapshots type CSISnapshotExposeParam struct { // SnapshotName is the original volume snapshot name @@ -157,7 +162,29 @@ func (e *csiSnapshotExposer) Expose(ctx context.Context, ownerObject corev1api.O curLog.Info("Volumesnapshot is ready") - vsc, err := csi.GetVolumeSnapshotContentForVolumeSnapshot(volumeSnapshot, e.csiSnapshotClient) + // Copy secrets and configmaps from source namespace to Velero namespace if configured. + // Done before creating any intermediate objects so failure doesn't require cleanup. + // These are needed by CSI drivers that require namespace-scoped resources for volume + // provisioning (e.g., encrypted volumes with KMS tokens and tenant Vault configs). + if value, exists := csiExposeParam.BackupPVCConfig[csiExposeParam.StorageClass]; exists { + copyLabels := map[string]string{BackupPVCSecretLabel: string(ownerObject.UID)} + for _, secretName := range value.SecretNames { + if copyErr := kube.CopySecret(ctx, e.kubeClient.CoreV1(), secretName, + csiExposeParam.SourceNamespace, ownerObject.Namespace, copyLabels, curLog); copyErr != nil { + return errors.Wrapf(copyErr, "error copying secret %s from %s to %s", + secretName, csiExposeParam.SourceNamespace, ownerObject.Namespace) + } + } + for _, cmName := range value.ConfigMapNames { + if copyErr := kube.CopyConfigMap(ctx, e.kubeClient.CoreV1(), cmName, + csiExposeParam.SourceNamespace, ownerObject.Namespace, copyLabels, curLog); copyErr != nil { + return errors.Wrapf(copyErr, "error copying configmap %s from %s to %s", + cmName, csiExposeParam.SourceNamespace, ownerObject.Namespace) + } + } + } + + vsc, err := csi.GetVolumeSnapshotContentForVolumeSnapshot(ctx, volumeSnapshot, e.csiSnapshotClient) if err != nil { return errors.Wrap(err, "error to get volume snapshot content") } @@ -219,6 +246,7 @@ func (e *csiSnapshotExposer) Expose(ctx context.Context, ownerObject corev1api.O backupPVCStorageClass := csiExposeParam.StorageClass backupPVCReadOnly := false spcNoRelabeling := false + backupPVCReadWriteOncePod := false backupPVCAnnotations := map[string]string{} intoleratableNodes := []string{} if value, exists := csiExposeParam.BackupPVCConfig[csiExposeParam.StorageClass]; exists { @@ -235,6 +263,14 @@ func (e *csiSnapshotExposer) Expose(ctx context.Context, ownerObject corev1api.O } } + if value.ReadWriteOncePod { + if backupPVCReadOnly { + curLog.WithField("vs name", volumeSnapshot.Name).Warn("Ignoring readWriteOncePod for read-only volume") + } else { + backupPVCReadWriteOncePod = true + } + } + if len(value.Annotations) > 0 { backupPVCAnnotations = value.Annotations } @@ -249,7 +285,7 @@ func (e *csiSnapshotExposer) Expose(ctx context.Context, ownerObject corev1api.O } } - backupPVC, err := e.createBackupPVC(ctx, ownerObject, backupVS.Name, backupPVCStorageClass, csiExposeParam.AccessMode, volumeSize, backupPVCReadOnly, backupPVCAnnotations, csiExposeParam.DataMover) + backupPVC, err := e.createBackupPVC(ctx, ownerObject, backupVS.Name, backupPVCStorageClass, csiExposeParam.AccessMode, volumeSize, backupPVCReadOnly, backupPVCReadWriteOncePod, backupPVCAnnotations, csiExposeParam.DataMover) if err != nil { return errors.Wrap(err, "error to create backup pvc") } @@ -514,6 +550,11 @@ func (e *csiSnapshotExposer) CleanUp(ctx context.Context, ownerObject corev1api. kube.DeletePodIfAny(ctx, e.kubeClient.CoreV1(), backupPodName, ownerObject.Namespace, e.log) kube.DeletePVAndPVCIfAny(ctx, e.kubeClient.CoreV1(), backupPVCName, ownerObject.Namespace, cleanUpTimeout, e.log) + kube.DeleteSecretsWithLabel(ctx, e.kubeClient.CoreV1(), ownerObject.Namespace, + BackupPVCSecretLabel, string(ownerObject.UID), e.log) + kube.DeleteConfigMapsWithLabel(ctx, e.kubeClient.CoreV1(), ownerObject.Namespace, + BackupPVCSecretLabel, string(ownerObject.UID), e.log) + csi.DeleteVolumeSnapshotIfAny(ctx, e.csiSnapshotClient, backupVSName, ownerObject.Namespace, e.log) csi.DeleteVolumeSnapshotIfAny(ctx, e.csiSnapshotClient, vsName, sourceNamespace, e.log) } @@ -600,7 +641,7 @@ func (e *csiSnapshotExposer) createBackupVSC(ctx context.Context, ownerObject co return e.csiSnapshotClient.VolumeSnapshotContents().Create(ctx, vsc, metav1.CreateOptions{}) } -func (e *csiSnapshotExposer) createBackupPVC(ctx context.Context, ownerObject corev1api.ObjectReference, backupVS, storageClass, accessMode string, resource resource.Quantity, readOnly bool, annotations map[string]string, dataMover string) (*corev1api.PersistentVolumeClaim, error) { +func (e *csiSnapshotExposer) createBackupPVC(ctx context.Context, ownerObject corev1api.ObjectReference, backupVS, storageClass, accessMode string, resource resource.Quantity, readOnly bool, readWriteOncePod bool, annotations map[string]string, dataMover string) (*corev1api.PersistentVolumeClaim, error) { backupPVCName := ownerObject.Name volumeMode, err := getVolumeModeByAccessMode(accessMode, dataMover) @@ -612,6 +653,8 @@ func (e *csiSnapshotExposer) createBackupPVC(ctx context.Context, ownerObject co if readOnly { pvcAccessMode = corev1api.ReadOnlyMany + } else if readWriteOncePod { + pvcAccessMode = corev1api.ReadWriteOncePod } dataSource := &corev1api.TypedLocalObjectReference{ @@ -811,7 +854,7 @@ func (e *csiSnapshotExposer) createBackupPod( } affinity.NodeSelector.MatchExpressions = append(affinity.NodeSelector.MatchExpressions, metav1.LabelSelectorRequirement{ - Key: "kubernetes.io/hostname", + Key: corev1api.LabelHostname, Values: intoleratableNodes, Operator: metav1.LabelSelectorOpNotIn, }) @@ -839,7 +882,7 @@ func (e *csiSnapshotExposer) createBackupPod( TopologySpreadConstraints: []corev1api.TopologySpreadConstraint{ { MaxSkew: 1, - TopologyKey: "kubernetes.io/hostname", + TopologyKey: corev1api.LabelHostname, WhenUnsatisfiable: corev1api.ScheduleAnyway, LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ diff --git a/pkg/exposer/csi_snapshot_test.go b/pkg/exposer/csi_snapshot_test.go index e5a7aa9a7..7b5c2eb3e 100644 --- a/pkg/exposer/csi_snapshot_test.go +++ b/pkg/exposer/csi_snapshot_test.go @@ -219,6 +219,7 @@ func TestExpose(t *testing.T) { err string expectedVolumeSize *resource.Quantity expectedReadOnlyPVC bool + expectedRWOPPVC bool expectedBackupPVCStorageClass string expectedAffinity *corev1api.Affinity expectedPVCAnnotation map[string]string @@ -492,7 +493,7 @@ func TestExpose(t *testing.T) { { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"windows"}, }, @@ -530,7 +531,7 @@ func TestExpose(t *testing.T) { { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"windows"}, }, @@ -570,7 +571,7 @@ func TestExpose(t *testing.T) { { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"windows"}, }, @@ -615,7 +616,7 @@ func TestExpose(t *testing.T) { { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"windows"}, }, @@ -661,7 +662,96 @@ func TestExpose(t *testing.T) { { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, + Operator: corev1api.NodeSelectorOpNotIn, + Values: []string{"windows"}, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "backupPVC uses ReadWriteOncePod access mode", + ownerBackup: backup, + exposeParam: CSISnapshotExposeParam{ + SnapshotName: "fake-vs", + SourceNamespace: "fake-ns", + StorageClass: "fake-sc", + SourcePVName: "fake-pv", + AccessMode: AccessModeFileSystem, + OperationTimeout: time.Millisecond, + ExposeTimeout: time.Millisecond, + BackupPVCConfig: map[string]velerotypes.BackupPVC{ + "fake-sc": { + ReadWriteOncePod: true, + }, + }, + }, + snapshotClientObj: []runtime.Object{ + vsObject, + vscObj, + }, + kubeClientObj: []runtime.Object{ + daemonSet, + scObj, + }, + expectedRWOPPVC: true, + expectedAffinity: &corev1api.Affinity{ + NodeAffinity: &corev1api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: corev1api.LabelOSStable, + Operator: corev1api.NodeSelectorOpNotIn, + Values: []string{"windows"}, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "readOnly takes precedence over readWriteOncePod", + ownerBackup: backup, + exposeParam: CSISnapshotExposeParam{ + SnapshotName: "fake-vs", + SourceNamespace: "fake-ns", + StorageClass: "fake-sc", + SourcePVName: "fake-pv", + AccessMode: AccessModeFileSystem, + OperationTimeout: time.Millisecond, + ExposeTimeout: time.Millisecond, + BackupPVCConfig: map[string]velerotypes.BackupPVC{ + "fake-sc": { + ReadOnly: true, + ReadWriteOncePod: true, + }, + }, + }, + snapshotClientObj: []runtime.Object{ + vsObject, + vscObj, + }, + kubeClientObj: []runtime.Object{ + daemonSet, + scObj, + }, + expectedReadOnlyPVC: true, + expectedAffinity: &corev1api.Affinity{ + NodeAffinity: &corev1api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: corev1api.LabelOSStable, Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"windows"}, }, @@ -705,7 +795,7 @@ func TestExpose(t *testing.T) { { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"windows"}, }, @@ -732,7 +822,7 @@ func TestExpose(t *testing.T) { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: metav1.LabelSelectorOpIn, Values: []string{"Linux"}, }, @@ -757,12 +847,12 @@ func TestExpose(t *testing.T) { { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: corev1api.NodeSelectorOpIn, Values: []string{"Linux"}, }, { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"windows"}, }, @@ -794,7 +884,7 @@ func TestExpose(t *testing.T) { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { - Key: "kubernetes.io/arch", + Key: corev1api.LabelArchStable, Operator: metav1.LabelSelectorOpIn, Values: []string{"amd64"}, }, @@ -820,12 +910,12 @@ func TestExpose(t *testing.T) { { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/arch", + Key: corev1api.LabelArchStable, Operator: corev1api.NodeSelectorOpIn, Values: []string{"amd64"}, }, { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"windows"}, }, @@ -870,7 +960,7 @@ func TestExpose(t *testing.T) { { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"windows"}, }, @@ -923,7 +1013,7 @@ func TestExpose(t *testing.T) { { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"windows"}, }, @@ -968,7 +1058,7 @@ func TestExpose(t *testing.T) { { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"windows"}, }, @@ -1015,12 +1105,12 @@ func TestExpose(t *testing.T) { { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"windows"}, }, { - Key: "kubernetes.io/hostname", + Key: corev1api.LabelHostname, Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"node-1", "node-2"}, }, @@ -1061,7 +1151,7 @@ func TestExpose(t *testing.T) { { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"windows"}, }, @@ -1150,6 +1240,12 @@ func TestExpose(t *testing.T) { assert.Equal(t, test.expectedReadOnlyPVC, gotReadOnlyAccessMode) } + if test.expectedRWOPPVC { + assert.Equal(t, []corev1api.PersistentVolumeAccessMode{corev1api.ReadWriteOncePod}, backupPVC.Spec.AccessModes) + } else { + assert.NotContains(t, backupPVC.Spec.AccessModes, corev1api.ReadWriteOncePod) + } + if test.expectedBackupPVCStorageClass != "" { assert.Equal(t, test.expectedBackupPVCStorageClass, *backupPVC.Spec.StorageClassName) } @@ -1521,6 +1617,37 @@ func Test_csiSnapshotExposer_createBackupPVC(t *testing.T) { }, } + backupPVCReadWriteOncePod := corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: velerov1.DefaultNamespace, + Name: "fake-backup", + Annotations: map[string]string{}, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: backup.APIVersion, + Kind: backup.Kind, + Name: backup.Name, + UID: backup.UID, + Controller: ptr.To(true), + }, + }, + }, + Spec: corev1api.PersistentVolumeClaimSpec{ + AccessModes: []corev1api.PersistentVolumeAccessMode{ + corev1api.ReadWriteOncePod, + }, + VolumeMode: &volumeMode, + DataSource: dataSource, + DataSourceRef: nil, + StorageClassName: ptr.To("fake-storage-class"), + Resources: corev1api.VolumeResourceRequirements{ + Requests: corev1api.ResourceList{ + corev1api.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + }, + } + tests := []struct { name string ownerBackup *velerov1.Backup @@ -1529,6 +1656,7 @@ func Test_csiSnapshotExposer_createBackupPVC(t *testing.T) { accessMode string resource resource.Quantity readOnly bool + readWriteOncePod bool kubeClientObj []runtime.Object snapshotClientObj []runtime.Object want *corev1api.PersistentVolumeClaim @@ -1556,6 +1684,30 @@ func Test_csiSnapshotExposer_createBackupPVC(t *testing.T) { want: &backupPVCReadOnly, wantErr: assert.NoError, }, + { + name: "backupPVC gets created with ReadWriteOncePod access mode when readWriteOncePod is set", + ownerBackup: backup, + backupVS: "fake-snapshot", + storageClass: "fake-storage-class", + accessMode: AccessModeFileSystem, + resource: resource.MustParse("1Gi"), + readOnly: false, + readWriteOncePod: true, + want: &backupPVCReadWriteOncePod, + wantErr: assert.NoError, + }, + { + name: "readOnly takes precedence over readWriteOncePod", + ownerBackup: backup, + backupVS: "fake-snapshot", + storageClass: "fake-storage-class", + accessMode: AccessModeFileSystem, + resource: resource.MustParse("1Gi"), + readOnly: true, + readWriteOncePod: true, + want: &backupPVCReadOnly, + wantErr: assert.NoError, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -1576,7 +1728,7 @@ func Test_csiSnapshotExposer_createBackupPVC(t *testing.T) { APIVersion: tt.ownerBackup.APIVersion, } } - got, err := e.createBackupPVC(t.Context(), ownerObject, tt.backupVS, tt.storageClass, tt.accessMode, tt.resource, tt.readOnly, map[string]string{}, "") + got, err := e.createBackupPVC(t.Context(), ownerObject, tt.backupVS, tt.storageClass, tt.accessMode, tt.resource, tt.readOnly, tt.readWriteOncePod, map[string]string{}, "") if !tt.wantErr(t, err, fmt.Sprintf("createBackupPVC(%v, %v, %v, %v, %v, %v)", ownerObject, tt.backupVS, tt.storageClass, tt.accessMode, tt.resource, tt.readOnly)) { return } @@ -2198,3 +2350,186 @@ func TestGetCBTInfo(t *testing.T) { }) } } + +func TestExpose_SecretCopy(t *testing.T) { + backup := &velerov1.Backup{ + TypeMeta: metav1.TypeMeta{ + APIVersion: velerov1.SchemeGroupVersion.String(), + Kind: "Backup", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: velerov1.DefaultNamespace, + Name: "fake-backup", + UID: "fake-uid", + }, + } + + ownerObject := corev1api.ObjectReference{ + Kind: backup.Kind, + Namespace: backup.Namespace, + Name: backup.Name, + UID: backup.UID, + APIVersion: backup.APIVersion, + } + + // The secret/configmap copy runs after GetVolumeTopology and WaitVolumeSnapshotReady, + // so a StorageClass and a ready VolumeSnapshot are needed to reach the copy block. + scObj := &storagev1api.StorageClass{ + ObjectMeta: metav1.ObjectMeta{Name: "encrypted-sc"}, + } + readyVS := func() *snapshotv1api.VolumeSnapshot { + vscName := "fake-vsc" + return &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{Name: "fake-vs", Namespace: "app-ns"}, + Spec: snapshotv1api.VolumeSnapshotSpec{ + Source: snapshotv1api.VolumeSnapshotSource{VolumeSnapshotContentName: &vscName}, + }, + Status: &snapshotv1api.VolumeSnapshotStatus{ + BoundVolumeSnapshotContentName: &vscName, + ReadyToUse: boolptr.True(), + RestoreSize: resource.NewQuantity(1234, ""), + }, + } + } + + param := func() *CSISnapshotExposeParam { + return &CSISnapshotExposeParam{ + SourceNamespace: "app-ns", + SourcePVName: "fake-pv", + SnapshotName: "fake-vs", + StorageClass: "encrypted-sc", + OperationTimeout: time.Millisecond, + ExposeTimeout: time.Second, + } + } + + t.Run("copies secret from source namespace", func(t *testing.T) { + srcSecret := &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "kms-token", Namespace: "app-ns"}, + Data: map[string][]byte{"token": []byte("vault-token")}, + Type: corev1api.SecretTypeOpaque, + } + fakeKubeClient := fake.NewSimpleClientset(srcSecret, scObj) + fakeSnapshotClient := snapshotFake.NewSimpleClientset(readyVS()) + + exposer := csiSnapshotExposer{ + kubeClient: fakeKubeClient, + csiSnapshotClient: fakeSnapshotClient.SnapshotV1(), + log: velerotest.NewLogger(), + } + + p := param() + p.BackupPVCConfig = map[string]velerotypes.BackupPVC{ + "encrypted-sc": {SecretNames: []string{"kms-token"}}, + } + + // Expose will fail later (no VSC exists), but the secret copy should succeed + _ = exposer.Expose(t.Context(), ownerObject, p) + + copied, err := fakeKubeClient.CoreV1().Secrets(ownerObject.Namespace).Get( + t.Context(), "kms-token", metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, []byte("vault-token"), copied.Data["token"]) + assert.Equal(t, string(ownerObject.UID), copied.Labels[BackupPVCSecretLabel]) + }) + + t.Run("copies configmap from source namespace", func(t *testing.T) { + srcCM := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "kms-config", Namespace: "app-ns"}, + Data: map[string]string{"vaultAddress": "https://vault.example.com"}, + } + fakeKubeClient := fake.NewSimpleClientset(srcCM, scObj) + fakeSnapshotClient := snapshotFake.NewSimpleClientset(readyVS()) + + exposer := csiSnapshotExposer{ + kubeClient: fakeKubeClient, + csiSnapshotClient: fakeSnapshotClient.SnapshotV1(), + log: velerotest.NewLogger(), + } + + p := param() + p.BackupPVCConfig = map[string]velerotypes.BackupPVC{ + "encrypted-sc": {ConfigMapNames: []string{"kms-config"}}, + } + + _ = exposer.Expose(t.Context(), ownerObject, p) + + copied, err := fakeKubeClient.CoreV1().ConfigMaps(ownerObject.Namespace).Get( + t.Context(), "kms-config", metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, "https://vault.example.com", copied.Data["vaultAddress"]) + assert.Equal(t, string(ownerObject.UID), copied.Labels[BackupPVCSecretLabel]) + }) + + t.Run("returns error when source secret missing", func(t *testing.T) { + fakeKubeClient := fake.NewSimpleClientset(scObj) + fakeSnapshotClient := snapshotFake.NewSimpleClientset(readyVS()) + + exposer := csiSnapshotExposer{ + kubeClient: fakeKubeClient, + csiSnapshotClient: fakeSnapshotClient.SnapshotV1(), + log: velerotest.NewLogger(), + } + + p := param() + p.BackupPVCConfig = map[string]velerotypes.BackupPVC{ + "encrypted-sc": {SecretNames: []string{"missing-secret"}}, + } + + err := exposer.Expose(t.Context(), ownerObject, p) + require.Error(t, err) + assert.Contains(t, err.Error(), "error copying secret") + }) +} + +func TestCleanUp_SecretsAndConfigMaps(t *testing.T) { + ownerObject := corev1api.ObjectReference{ + Kind: "Backup", + Namespace: "velero", + Name: "du-123", + UID: "fake-uid", + APIVersion: "v1", + } + + secret := &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kms-token", Namespace: "velero", + Labels: map[string]string{BackupPVCSecretLabel: string(ownerObject.UID)}, + UID: "secret-uid", + }, + } + cm := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kms-config", Namespace: "velero", + Labels: map[string]string{BackupPVCSecretLabel: string(ownerObject.UID)}, + UID: "cm-uid", + }, + } + unrelatedSecret := &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "other-secret", Namespace: "velero", + Labels: map[string]string{BackupPVCSecretLabel: "other-owner-uid"}, + UID: "other-uid", + }, + } + + fakeKubeClient := fake.NewSimpleClientset(secret, cm, unrelatedSecret) + fakeSnapshotClient := snapshotFake.NewSimpleClientset() + + exposer := csiSnapshotExposer{ + kubeClient: fakeKubeClient, + csiSnapshotClient: fakeSnapshotClient.SnapshotV1(), + log: velerotest.NewLogger(), + } + + exposer.CleanUp(t.Context(), ownerObject, "", "app-ns") + + _, err := fakeKubeClient.CoreV1().Secrets("velero").Get(t.Context(), "kms-token", metav1.GetOptions{}) + require.Error(t, err, "owned secret should be deleted") + + _, err = fakeKubeClient.CoreV1().ConfigMaps("velero").Get(t.Context(), "kms-config", metav1.GetOptions{}) + require.Error(t, err, "owned configmap should be deleted") + + _, err = fakeKubeClient.CoreV1().Secrets("velero").Get(t.Context(), "other-secret", metav1.GetOptions{}) + assert.NoError(t, err, "unrelated secret should not be deleted") +} diff --git a/pkg/exposer/generic_restore.go b/pkg/exposer/generic_restore.go index 16a114e64..fe8e571d4 100644 --- a/pkg/exposer/generic_restore.go +++ b/pkg/exposer/generic_restore.go @@ -196,6 +196,36 @@ func (e *genericRestoreExposer) Expose(ctx context.Context, ownerObject corev1ap } } + // Copy secrets and configmaps from the target namespace to the Velero namespace if configured. + // These are needed by CSI drivers that require namespace-scoped resources for volume + // provisioning of the restorePVC (e.g., encrypted volumes with KMS tokens and tenant Vault configs). + copyLabels := map[string]string{BackupPVCSecretLabel: string(ownerObject.UID)} + for _, secretName := range param.RestorePVCConfig.SecretNames { + if copyErr := kube.CopySecret(ctx, e.kubeClient.CoreV1(), secretName, + param.TargetNamespace, ownerObject.Namespace, copyLabels, curLog); copyErr != nil { + err = errors.Wrapf(copyErr, "error copying secret %s from %s to %s", + secretName, param.TargetNamespace, ownerObject.Namespace) + return err + } + } + for _, cmName := range param.RestorePVCConfig.ConfigMapNames { + if copyErr := kube.CopyConfigMap(ctx, e.kubeClient.CoreV1(), cmName, + param.TargetNamespace, ownerObject.Namespace, copyLabels, curLog); copyErr != nil { + err = errors.Wrapf(copyErr, "error copying configmap %s from %s to %s", + cmName, param.TargetNamespace, ownerObject.Namespace) + return err + } + } + + defer func() { + if err != nil { + kube.DeleteSecretsWithLabel(ctx, e.kubeClient.CoreV1(), ownerObject.Namespace, + BackupPVCSecretLabel, string(ownerObject.UID), curLog) + kube.DeleteConfigMapsWithLabel(ctx, e.kubeClient.CoreV1(), ownerObject.Namespace, + BackupPVCSecretLabel, string(ownerObject.UID), curLog) + } + }() + restorePVC, err := e.createRestorePVC(ctx, ownerObject, targetPVC, selectedNode, param.DataMover) if err != nil { return errors.Wrap(err, "error to create restore pvc") @@ -397,6 +427,11 @@ func (e *genericRestoreExposer) CleanUp(ctx context.Context, ownerObject corev1a kube.DeletePodIfAny(ctx, e.kubeClient.CoreV1(), restorePodName, ownerObject.Namespace, e.log) kube.DeletePVAndPVCIfAny(ctx, e.kubeClient.CoreV1(), restorePVCName, ownerObject.Namespace, 0, e.log) kube.DeletePVAndPVCIfAny(ctx, e.kubeClient.CoreV1(), cachePVCName, ownerObject.Namespace, 0, e.log) + + kube.DeleteSecretsWithLabel(ctx, e.kubeClient.CoreV1(), ownerObject.Namespace, + BackupPVCSecretLabel, string(ownerObject.UID), e.log) + kube.DeleteConfigMapsWithLabel(ctx, e.kubeClient.CoreV1(), ownerObject.Namespace, + BackupPVCSecretLabel, string(ownerObject.UID), e.log) } func (e *genericRestoreExposer) RebindVolume(ctx context.Context, ownerObject corev1api.ObjectReference, param GenericRestoreRebindVolumeParam) error { @@ -620,7 +655,7 @@ func (e *genericRestoreExposer) createRestorePod( nodeSelector := map[string]string{} if selectedNode != "" { affinity = nil - nodeSelector["kubernetes.io/hostname"] = selectedNode + nodeSelector[corev1api.LabelHostname] = selectedNode e.log.Infof("Selected node for restore pod. Ignore affinity from the node-agent config.") } @@ -762,7 +797,7 @@ func (e *genericRestoreExposer) createRestorePod( TopologySpreadConstraints: []corev1api.TopologySpreadConstraint{ { MaxSkew: 1, - TopologyKey: "kubernetes.io/hostname", + TopologyKey: corev1api.LabelHostname, WhenUnsatisfiable: corev1api.ScheduleAnyway, LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ diff --git a/pkg/exposer/generic_restore_test.go b/pkg/exposer/generic_restore_test.go index b65863318..6087d0f71 100644 --- a/pkg/exposer/generic_restore_test.go +++ b/pkg/exposer/generic_restore_test.go @@ -34,6 +34,7 @@ import ( velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerotest "github.com/vmware-tanzu/velero/pkg/test" + velerotypes "github.com/vmware-tanzu/velero/pkg/types" "github.com/vmware-tanzu/velero/pkg/util/datamover" "github.com/vmware-tanzu/velero/pkg/util/kube" ) @@ -353,6 +354,88 @@ func TestRestoreExpose(t *testing.T) { } } +func TestRestoreExpose_SecretCopy(t *testing.T) { + scName := "fake-sc" + restore := &velerov1.Restore{ + TypeMeta: metav1.TypeMeta{APIVersion: velerov1.SchemeGroupVersion.String(), Kind: "Restore"}, + ObjectMeta: metav1.ObjectMeta{Namespace: velerov1.DefaultNamespace, Name: "fake-restore", UID: "fake-uid"}, + } + ownerObject := corev1api.ObjectReference{ + Kind: restore.Kind, + Namespace: restore.Namespace, + Name: restore.Name, + UID: restore.UID, + APIVersion: restore.APIVersion, + } + targetPVCObj := &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Namespace: "fake-ns", Name: "fake-target-pvc"}, + Spec: corev1api.PersistentVolumeClaimSpec{StorageClassName: &scName}, + } + storageClass := &storagev1api.StorageClass{ObjectMeta: metav1.ObjectMeta{Name: "fake-sc"}} + daemonSet := &appsv1api.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Namespace: "velero", Name: "node-agent"}, + TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: appsv1api.SchemeGroupVersion.String()}, + Spec: appsv1api.DaemonSetSpec{ + Template: corev1api.PodTemplateSpec{ + Spec: corev1api.PodSpec{Containers: []corev1api.Container{{Image: "fake-image"}}}, + }, + }, + } + + t.Run("copies secret and configmap from target namespace", func(t *testing.T) { + srcSecret := &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "kms-token", Namespace: "fake-ns"}, + Data: map[string][]byte{"token": []byte("vault-token")}, + Type: corev1api.SecretTypeOpaque, + } + srcCM := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "kms-config", Namespace: "fake-ns"}, + Data: map[string]string{"vaultAddress": "https://vault.example.com"}, + } + fakeKubeClient := fake.NewSimpleClientset(targetPVCObj, storageClass, daemonSet, srcSecret, srcCM) + exposer := genericRestoreExposer{kubeClient: fakeKubeClient, log: velerotest.NewLogger()} + + err := exposer.Expose(t.Context(), ownerObject, GenericRestoreExposeParam{ + TargetPVCName: "fake-target-pvc", + TargetNamespace: "fake-ns", + HostingPodLabels: map[string]string{}, + Resources: corev1api.ResourceRequirements{}, + ExposeTimeout: time.Millisecond, + RestorePVCConfig: velerotypes.RestorePVC{ + SecretNames: []string{"kms-token"}, + ConfigMapNames: []string{"kms-config"}, + }, + }) + require.NoError(t, err) + + copiedSecret, err := fakeKubeClient.CoreV1().Secrets(ownerObject.Namespace).Get(t.Context(), "kms-token", metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, []byte("vault-token"), copiedSecret.Data["token"]) + assert.Equal(t, string(ownerObject.UID), copiedSecret.Labels[BackupPVCSecretLabel]) + + copiedCM, err := fakeKubeClient.CoreV1().ConfigMaps(ownerObject.Namespace).Get(t.Context(), "kms-config", metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, "https://vault.example.com", copiedCM.Data["vaultAddress"]) + assert.Equal(t, string(ownerObject.UID), copiedCM.Labels[BackupPVCSecretLabel]) + }) + + t.Run("returns error when source secret missing", func(t *testing.T) { + fakeKubeClient := fake.NewSimpleClientset(targetPVCObj, storageClass, daemonSet) + exposer := genericRestoreExposer{kubeClient: fakeKubeClient, log: velerotest.NewLogger()} + + err := exposer.Expose(t.Context(), ownerObject, GenericRestoreExposeParam{ + TargetPVCName: "fake-target-pvc", + TargetNamespace: "fake-ns", + HostingPodLabels: map[string]string{}, + Resources: corev1api.ResourceRequirements{}, + ExposeTimeout: time.Millisecond, + RestorePVCConfig: velerotypes.RestorePVC{SecretNames: []string{"missing-secret"}}, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "error copying secret") + }) +} + func TestRebindVolume(t *testing.T) { restore := &velerov1.Restore{ TypeMeta: metav1.TypeMeta{ @@ -1330,12 +1413,13 @@ func TestCreateRestorePod(t *testing.T) { } tests := []struct { - name string - kubeClientObj []runtime.Object - selectedNode string - affinity *kube.LoadAffinity - nodeOS string - expectedPod *corev1api.Pod + name string + kubeClientObj []runtime.Object + selectedNode string + affinity *kube.LoadAffinity + nodeOS string + expectedPod *corev1api.Pod + expectedNodeSelector map[string]string }{ { name: "linux", @@ -1345,7 +1429,7 @@ func TestCreateRestorePod(t *testing.T) { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: metav1.LabelSelectorOpIn, Values: []string{"linux"}, }, @@ -1363,7 +1447,7 @@ func TestCreateRestorePod(t *testing.T) { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: metav1.LabelSelectorOpIn, Values: []string{"windows"}, }, @@ -1373,6 +1457,29 @@ func TestCreateRestorePod(t *testing.T) { }, nodeOS: "windows", }, + { + // A selected node is pinned through the node selector, and the + // affinity from the node-agent config is ignored. + name: "selected node", + kubeClientObj: []runtime.Object{daemonSet, daemonSetWin, targetPVCObj}, + selectedNode: "fake-selected-node", + affinity: &kube.LoadAffinity{ + NodeSelector: metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: corev1api.LabelOSStable, + Operator: metav1.LabelSelectorOpIn, + Values: []string{"linux"}, + }, + }, + }, + StorageClass: scName, + }, + nodeOS: "linux", + expectedNodeSelector: map[string]string{ + corev1api.LabelHostname: "fake-selected-node", + }, + }, } for _, test := range tests { @@ -1407,6 +1514,9 @@ func TestCreateRestorePod(t *testing.T) { if test.expectedPod != nil { assert.Equal(t, test.expectedPod, pod) } + if test.expectedNodeSelector != nil { + assert.Equal(t, test.expectedNodeSelector, pod.Spec.NodeSelector) + } }) } } diff --git a/pkg/install/daemonset.go b/pkg/install/daemonset.go index 190e785d8..6ef1139d2 100644 --- a/pkg/install/daemonset.go +++ b/pkg/install/daemonset.go @@ -247,7 +247,7 @@ func DaemonSet(namespace string, opts ...podTemplateOption) *appsv1api.DaemonSet { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Values: []string{"windows"}, Operator: corev1api.NodeSelectorOpIn, }, @@ -280,7 +280,7 @@ func DaemonSet(namespace string, opts ...podTemplateOption) *appsv1api.DaemonSet { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Values: []string{"windows"}, Operator: corev1api.NodeSelectorOpNotIn, }, diff --git a/pkg/install/daemonset_test.go b/pkg/install/daemonset_test.go index 6cab7f063..2c7e201e4 100644 --- a/pkg/install/daemonset_test.go +++ b/pkg/install/daemonset_test.go @@ -41,7 +41,7 @@ func TestDaemonSet(t *testing.T) { { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Values: []string{"windows"}, Operator: corev1api.NodeSelectorOpNotIn, }, @@ -107,7 +107,7 @@ func TestDaemonSet(t *testing.T) { { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Values: []string{"windows"}, Operator: corev1api.NodeSelectorOpIn, }, diff --git a/pkg/install/deployment.go b/pkg/install/deployment.go index 6bea8b0be..642a51321 100644 --- a/pkg/install/deployment.go +++ b/pkg/install/deployment.go @@ -395,7 +395,7 @@ func Deployment(namespace string, opts ...podTemplateOption) *appsv1api.Deployme { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Values: []string{"windows"}, Operator: corev1api.NodeSelectorOpNotIn, }, @@ -444,6 +444,15 @@ func Deployment(namespace string, opts ...podTemplateOption) *appsv1api.Deployme }, }, Resources: c.resources, + SecurityContext: &corev1api.SecurityContext{ + Capabilities: &corev1api.Capabilities{ + Drop: []corev1api.Capability{"ALL"}, + }, + AllowPrivilegeEscalation: ptr.To(false), + SeccompProfile: &corev1api.SeccompProfile{ + Type: corev1api.SeccompProfileTypeRuntimeDefault, + }, + }, }, }, Volumes: []corev1api.Volume{ @@ -523,7 +532,7 @@ func Deployment(namespace string, opts ...podTemplateOption) *appsv1api.Deployme if len(c.plugins) > 0 { for _, image := range c.plugins { - container := *builder.ForPluginContainer(image, pullPolicy).Result() + container := *builder.ForPluginContainer(image, pullPolicy, deployment.Spec.Template.Spec.InitContainers).Result() deployment.Spec.Template.Spec.InitContainers = append(deployment.Spec.Template.Spec.InitContainers, container) } } diff --git a/pkg/install/deployment_test.go b/pkg/install/deployment_test.go index 6e9ff6ec5..c2d582b8e 100644 --- a/pkg/install/deployment_test.go +++ b/pkg/install/deployment_test.go @@ -120,7 +120,7 @@ func TestDeployment(t *testing.T) { { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Values: []string{"windows"}, Operator: corev1api.NodeSelectorOpNotIn, }, diff --git a/pkg/install/install_test.go b/pkg/install/install_test.go index 47a9aa273..c09250ef2 100644 --- a/pkg/install/install_test.go +++ b/pkg/install/install_test.go @@ -21,7 +21,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/controller-runtime/pkg/client/fake" - v1crds "github.com/vmware-tanzu/velero/config/crd/v1/crds" + v1crds "github.com/vmware-tanzu/velero/config/crd/v1" "github.com/vmware-tanzu/velero/pkg/test" ) diff --git a/pkg/install/resources.go b/pkg/install/resources.go index 9f9543300..3869c1969 100644 --- a/pkg/install/resources.go +++ b/pkg/install/resources.go @@ -27,8 +27,8 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - v1crds "github.com/vmware-tanzu/velero/config/crd/v1/crds" - v2alpha1crds "github.com/vmware-tanzu/velero/config/crd/v2alpha1/crds" + v1crds "github.com/vmware-tanzu/velero/config/crd/v1" + v2alpha1crds "github.com/vmware-tanzu/velero/config/crd/v2alpha1" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/util/kube" ) diff --git a/pkg/persistence/object_store.go b/pkg/persistence/object_store.go index 440ca8756..338ee016e 100644 --- a/pkg/persistence/object_store.go +++ b/pkg/persistence/object_store.go @@ -94,6 +94,7 @@ type BackupStore interface { // DownloadURLTTL is how long a download URL is valid for. const DownloadURLTTL = 10 * time.Minute +const maxDecompressedSize = 1024 * 1024 * 1024 // 1 GB type objectBackupStore struct { objectStore velero.ObjectStore @@ -164,6 +165,9 @@ func (b *objectBackupStoreGetter) Get(location *velerov1api.BackupStorageLocatio } } + // Delete any user-provided credentialsFile to prevent path traversal vulnerabilities + delete(objectStoreConfig, "credentialsFile") + // add the bucket name and prefix to the config map so that object stores // can use them when initializing. The AWS object store uses the bucket // name to determine the bucket's region when setting up its client. @@ -320,7 +324,8 @@ func (s *objectBackupStore) GetBackupMetadata(name string) (*velerov1api.Backup, } defer res.Close() - data, err := io.ReadAll(res) + limitReader := io.LimitReader(res, maxDecompressedSize) + data, err := io.ReadAll(limitReader) if err != nil { return nil, errors.WithStack(err) } @@ -431,7 +436,9 @@ func decode(jsongzReader io.Reader, into any) error { } defer gzr.Close() - if err := json.NewDecoder(gzr).Decode(into); err != nil { + limitReader := io.LimitReader(gzr, maxDecompressedSize) + + if err := json.NewDecoder(limitReader).Decode(into); err != nil { return errors.Wrap(err, "error decoding object data") } diff --git a/pkg/podvolume/backup_micro_service.go b/pkg/podvolume/backup_micro_service.go index 246221d25..1f71ba0b2 100644 --- a/pkg/podvolume/backup_micro_service.go +++ b/pkg/podvolume/backup_micro_service.go @@ -32,6 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/cache" "github.com/vmware-tanzu/velero/internal/credentials" + veleroshared "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/datapath" "github.com/vmware-tanzu/velero/pkg/repository" @@ -192,10 +193,23 @@ func (r *BackupMicroService) RunCancelableDataPath(ctx context.Context) (string, tags := map[string]string{} + // "none" requests a full backup. "auto" requests that the data mover finds the most + // recent backup of the same volume as parent, which is what an empty ParentSnapshot + // already does, so both map to "". + parentSnapshot := pvb.Spec.ParentSnapshot + forceFull := false + switch pvb.Spec.ParentSnapshot { + case veleroshared.ParentSnapshotNone: + parentSnapshot = "" + forceFull = true + case veleroshared.ParentSnapshotAuto: + parentSnapshot = "" + } + if err := fsBackup.StartBackup(r.sourceTargetPath, pvb.Spec.UploaderSettings, &datapath.BackupStartParam{ RealSource: GetRealSource(pvb), - ParentSnapshot: "", - ForceFull: false, + ParentSnapshot: parentSnapshot, + ForceFull: forceFull, Tags: tags, }); err != nil { return "", errors.Wrap(err, "error starting data path backup") diff --git a/pkg/podvolume/backup_micro_service_test.go b/pkg/podvolume/backup_micro_service_test.go index eac17e4de..2de4705af 100644 --- a/pkg/podvolume/backup_micro_service_test.go +++ b/pkg/podvolume/backup_micro_service_test.go @@ -34,6 +34,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/datapath" "github.com/vmware-tanzu/velero/pkg/uploader" + veleroshared "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" clientFake "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -446,3 +447,88 @@ func TestRunCancelableDataPath(t *testing.T) { cancel() } + +func TestRunCancelableDataPathParentSnapshot(t *testing.T) { + pvbName := "fake-pvb" + + tests := []struct { + name string + parentSnapshot string + expectedParentSnapshot string + expectedForceFull bool + }{ + { + name: "empty lets the data mover search for a parent", + parentSnapshot: "", + expectedParentSnapshot: "", + expectedForceFull: false, + }, + { + name: "auto lets the data mover search for a parent", + parentSnapshot: veleroshared.ParentSnapshotAuto, + expectedParentSnapshot: "", + expectedForceFull: false, + }, + { + name: "none forces a full backup", + parentSnapshot: veleroshared.ParentSnapshotNone, + expectedParentSnapshot: "", + expectedForceFull: true, + }, + { + name: "a specific snapshot ID is passed through unchanged", + parentSnapshot: "fake-parent-snapshot-id", + expectedParentSnapshot: "fake-parent-snapshot-id", + expectedForceFull: false, + }, + } + + scheme := runtime.NewScheme() + velerov1api.AddToScheme(scheme) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + pvbInProgress := builder.ForPodVolumeBackup(velerov1api.DefaultNamespace, pvbName). + Phase(velerov1api.PodVolumeBackupPhaseInProgress). + Result() + pvbInProgress.Spec.ParentSnapshot = test.parentSnapshot + + fakeClient := clientFake.NewClientBuilder().WithScheme(scheme). + WithRuntimeObjects(pvbInProgress).Build() + + bs := &BackupMicroService{ + namespace: velerov1api.DefaultNamespace, + pvbName: pvbName, + ctx: t.Context(), + client: fakeClient, + dataPathMgr: datapath.NewManager(1), + eventRecorder: &backupMsTestHelper{}, + resultSignal: make(chan dataPathResult), + logger: velerotest.NewLogger(), + } + + var startParam *datapath.BackupStartParam + datapath.VGDPCreator = func(string, string, kbclient.Client, string, datapath.Callbacks, logrus.FieldLogger) datapath.AsyncBR { + fsBR := datapathmockes.NewAsyncBR(t) + fsBR.On("Init", mock.Anything, mock.Anything).Return(nil) + fsBR.On("StartBackup", mock.Anything, mock.Anything, mock.Anything). + Run(func(args mock.Arguments) { + startParam = args.Get(2).(*datapath.BackupStartParam) + }).Return(nil) + return fsBR + } + + go func() { + time.Sleep(time.Millisecond * 500) + bs.resultSignal <- dataPathResult{result: "fake-succeed-result"} + }() + + _, err := bs.RunCancelableDataPath(t.Context()) + require.NoError(t, err) + + require.NotNil(t, startParam) + assert.Equal(t, test.expectedParentSnapshot, startParam.ParentSnapshot) + assert.Equal(t, test.expectedForceFull, startParam.ForceFull) + }) + } +} diff --git a/pkg/podvolume/backupper.go b/pkg/podvolume/backupper.go index 261f227f8..46ed4defd 100644 --- a/pkg/podvolume/backupper.go +++ b/pkg/podvolume/backupper.go @@ -34,6 +34,7 @@ import ( ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/internal/resourcepolicies" + veleroshared "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" veleroclient "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/label" @@ -598,5 +599,9 @@ func newPodVolumeBackup(backup *velerov1api.Backup, pod *corev1api.Pod, volume c pvb.Spec.UploaderSettings = uploaderutil.StoreBackupConfig(backup.Spec.UploaderConfig) } + if backup.Spec.BackupType == velerov1api.BackupTypeFull { + pvb.Spec.ParentSnapshot = veleroshared.ParentSnapshotNone + } + return pvb } diff --git a/pkg/podvolume/backupper_test.go b/pkg/podvolume/backupper_test.go index 1ef4297af..92fab63ed 100644 --- a/pkg/podvolume/backupper_test.go +++ b/pkg/podvolume/backupper_test.go @@ -357,7 +357,7 @@ func createPVBObj(fail bool, withSnapshot bool, index int, uploaderType string) } func createNodeObj() *corev1api.Node { - return builder.ForNode("fake-node-name").Labels(map[string]string{"kubernetes.io/os": "linux"}).Result() + return builder.ForNode("fake-node-name").Labels(map[string]string{corev1api.LabelOSStable: "linux"}).Result() } func TestBackupPodVolumes(t *testing.T) { diff --git a/pkg/repository/provider/unified_repo.go b/pkg/repository/provider/unified_repo.go index b30e4618b..24c744fe4 100644 --- a/pkg/repository/provider/unified_repo.go +++ b/pkg/repository/provider/unified_repo.go @@ -501,11 +501,16 @@ func getStorageCredentials(backupLocation *velerov1api.BackupStorageLocation, cr return map[string]string{}, errors.New("invalid storage provider") } - config := backupLocation.Spec.Config - if config == nil { - config = map[string]string{} + config := make(map[string]string) + if backupLocation.Spec.Config != nil { + for k, v := range backupLocation.Spec.Config { + config[k] = v + } } + // Delete any user-provided credentialsFile to prevent path traversal vulnerabilities + delete(config, repoconfig.CredentialsFileKey) + if backupLocation.Spec.Credential != nil { config[repoconfig.CredentialsFileKey], err = credentialsFileStore.Path(backupLocation.Spec.Credential) if err != nil { @@ -549,11 +554,16 @@ func getStorageVariables(backupLocation *velerov1api.BackupStorageLocation, repo return map[string]string{}, errors.New("invalid storage provider") } - config := backupLocation.Spec.Config - if config == nil { - config = map[string]string{} + config := make(map[string]string) + if backupLocation.Spec.Config != nil { + for k, v := range backupLocation.Spec.Config { + config[k] = v + } } + // Delete any user-provided credentialsFile to prevent path traversal vulnerabilities + delete(config, repoconfig.CredentialsFileKey) + bucket := strings.Trim(config["bucket"], "/") prefix := strings.Trim(config["prefix"], "/") if backupLocation.Spec.ObjectStorage != nil { diff --git a/pkg/repository/provider/unified_repo_test.go b/pkg/repository/provider/unified_repo_test.go index 2cd9bf576..d2131d6bc 100644 --- a/pkg/repository/provider/unified_repo_test.go +++ b/pkg/repository/provider/unified_repo_test.go @@ -85,7 +85,7 @@ func TestGetStorageCredentials(t *testing.T) { Spec: velerov1api.BackupStorageLocationSpec{ Provider: "velero.io/aws", Config: map[string]string{ - "credentialsFile": "credentials-from-config-map", + "credentialsFile": "credentials-from-config-map", // This should be ignored }, }, }, @@ -96,7 +96,7 @@ func TestGetStorageCredentials(t *testing.T) { }, credFileStore: new(credmock.FileStore), expected: map[string]string{ - "accessKeyID": "from: credentials-from-config-map", + "accessKeyID": "from: ", "providerName": "", "secretAccessKey": "", "sessionToken": "", @@ -108,7 +108,7 @@ func TestGetStorageCredentials(t *testing.T) { Spec: velerov1api.BackupStorageLocationSpec{ Provider: "velero.io/aws", Config: map[string]string{ - "credentialsFile": "credentials-from-config-map", + "credentialsFile": "credentials-from-config-map", // This should be ignored }, Credential: &corev1api.SecretKeySelector{}, }, @@ -134,7 +134,7 @@ func TestGetStorageCredentials(t *testing.T) { Spec: velerov1api.BackupStorageLocationSpec{ Provider: "velero.io/aws", Config: map[string]string{ - "credentialsFile": "credentials-from-config-map", + "credentialsFile": "credentials-from-config-map", // This should be ignored }, }, }, @@ -176,16 +176,16 @@ func TestGetStorageCredentials(t *testing.T) { Spec: velerov1api.BackupStorageLocationSpec{ Provider: "velero.io/gcp", Config: map[string]string{ - "credentialsFile": "credentials-from-config-map", + "credentialsFile": "credentials-from-config-map", // This should be ignored }, }, }, getGCPCredentials: func(config map[string]string) string { - return "credentials-from-config-map" + return config["credentialsFile"] }, credFileStore: new(credmock.FileStore), expected: map[string]string{ - "credFile": "credentials-from-config-map", + "credFile": "", }, }, } diff --git a/pkg/repository/udmrepo/kopialib/repo_init.go b/pkg/repository/udmrepo/kopialib/repo_init.go index 4e62c9087..5c272298c 100644 --- a/pkg/repository/udmrepo/kopialib/repo_init.go +++ b/pkg/repository/udmrepo/kopialib/repo_init.go @@ -43,12 +43,18 @@ type kopiaBackendStore struct { store backend.Store } +type kopiaBackendStoreFactory struct { + name string + description string + newStore func() backend.Store +} + // backendStores lists the supported backend storages at present -var backendStores = []kopiaBackendStore{ - {udmrepo.StorageTypeAzure, "an Azure blob storage", &backend.AzureBackend{}}, - {udmrepo.StorageTypeFs, "a filesystem", &backend.FsBackend{}}, - {udmrepo.StorageTypeGcs, "a Google Cloud Storage bucket", &backend.GCSBackend{}}, - {udmrepo.StorageTypeS3, "an S3 bucket", &backend.S3Backend{}}, +var backendStores = []kopiaBackendStoreFactory{ + {udmrepo.StorageTypeAzure, "an Azure blob storage", func() backend.Store { return &backend.AzureBackend{} }}, + {udmrepo.StorageTypeFs, "a filesystem", func() backend.Store { return &backend.FsBackend{} }}, + {udmrepo.StorageTypeGcs, "a Google Cloud Storage bucket", func() backend.Store { return &backend.GCSBackend{} }}, + {udmrepo.StorageTypeS3, "an S3 bucket", func() backend.Store { return &backend.S3Backend{} }}, } const udmRepoBlobID = "udmrepo.Repository" @@ -226,7 +232,11 @@ func connectStore(ctx context.Context, repoOption udmrepo.RepoOptions, logger lo func findBackendStore(storage string) *kopiaBackendStore { for _, options := range backendStores { if strings.EqualFold(options.name, storage) { - return &options + return &kopiaBackendStore{ + name: options.name, + description: options.description, + store: options.newStore(), + } } } diff --git a/pkg/repository/udmrepo/kopialib/repo_init_test.go b/pkg/repository/udmrepo/kopialib/repo_init_test.go index c8b8e6aa1..130bb7b4d 100644 --- a/pkg/repository/udmrepo/kopialib/repo_init_test.go +++ b/pkg/repository/udmrepo/kopialib/repo_init_test.go @@ -41,6 +41,29 @@ import ( "github.com/cockroachdb/errors" ) +func TestFindBackendStore(t *testing.T) { + // findBackendStore should return a unique instance on each call + // so that concurrently executing controllers do not overwrite each other's credentials/options. + t.Run("returns distinct instances", func(t *testing.T) { + store1 := findBackendStore(udmrepo.StorageTypeS3) + require.NotNil(t, store1) + + store2 := findBackendStore(udmrepo.StorageTypeS3) + require.NotNil(t, store2) + + // The pointers to the wrapper struct must be different + assert.NotSame(t, store1, store2, "findBackendStore should return different kopiaBackendStore instances") + + // The pointers to the actual underlying store must be different + assert.NotSame(t, store1.store, store2.store, "findBackendStore should return different backend.Store instances") + }) + + t.Run("returns nil for unknown storage type", func(t *testing.T) { + store := findBackendStore("unknown-type") + assert.Nil(t, store) + }) +} + type comparableError struct { message string } @@ -133,11 +156,11 @@ func TestCreateBackupRepo(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { logger := velerotest.NewLogger() - backendStores = []kopiaBackendStore{ - {udmrepo.StorageTypeAzure, "fake store", tc.backendStore}, - {udmrepo.StorageTypeFs, "fake store", tc.backendStore}, - {udmrepo.StorageTypeGcs, "fake store", tc.backendStore}, - {udmrepo.StorageTypeS3, "fake store", tc.backendStore}, + backendStores = []kopiaBackendStoreFactory{ + {udmrepo.StorageTypeAzure, "fake store", func() backend.Store { return tc.backendStore }}, + {udmrepo.StorageTypeFs, "fake store", func() backend.Store { return tc.backendStore }}, + {udmrepo.StorageTypeGcs, "fake store", func() backend.Store { return tc.backendStore }}, + {udmrepo.StorageTypeS3, "fake store", func() backend.Store { return tc.backendStore }}, } if tc.backendStore != nil { @@ -219,11 +242,11 @@ func TestConnectBackupRepo(t *testing.T) { logger := velerotest.NewLogger() for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - backendStores = []kopiaBackendStore{ - {udmrepo.StorageTypeAzure, "fake store", tc.backendStore}, - {udmrepo.StorageTypeFs, "fake store", tc.backendStore}, - {udmrepo.StorageTypeGcs, "fake store", tc.backendStore}, - {udmrepo.StorageTypeS3, "fake store", tc.backendStore}, + backendStores = []kopiaBackendStoreFactory{ + {udmrepo.StorageTypeAzure, "fake store", func() backend.Store { return tc.backendStore }}, + {udmrepo.StorageTypeFs, "fake store", func() backend.Store { return tc.backendStore }}, + {udmrepo.StorageTypeGcs, "fake store", func() backend.Store { return tc.backendStore }}, + {udmrepo.StorageTypeS3, "fake store", func() backend.Store { return tc.backendStore }}, } if tc.backendStore != nil { @@ -441,11 +464,11 @@ func TestGetRepositoryStatus(t *testing.T) { logger := velerotest.NewLogger() for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - backendStores = []kopiaBackendStore{ - {udmrepo.StorageTypeAzure, "fake store", tc.backendStore}, - {udmrepo.StorageTypeFs, "fake store", tc.backendStore}, - {udmrepo.StorageTypeGcs, "fake store", tc.backendStore}, - {udmrepo.StorageTypeS3, "fake store", tc.backendStore}, + backendStores = []kopiaBackendStoreFactory{ + {udmrepo.StorageTypeAzure, "fake store", func() backend.Store { return tc.backendStore }}, + {udmrepo.StorageTypeFs, "fake store", func() backend.Store { return tc.backendStore }}, + {udmrepo.StorageTypeGcs, "fake store", func() backend.Store { return tc.backendStore }}, + {udmrepo.StorageTypeS3, "fake store", func() backend.Store { return tc.backendStore }}, } if tc.backendStore != nil { diff --git a/pkg/restore/restore.go b/pkg/restore/restore.go index a6d97d1ec..a4c29d067 100644 --- a/pkg/restore/restore.go +++ b/pkg/restore/restore.go @@ -1004,6 +1004,19 @@ func (ctx *restoreContext) processSelectedResource( targetNS = namespace } } + + // Make sure the resource in the "resourceMustHave" set will always be created in the namespace where velero is installed. + if ctx.resourceMustHave.Has(groupResource.String()) && targetNS != "" && targetNS != ctx.restore.Namespace { + err := fmt.Errorf("resource %s/%s is must-have per velero internal setting, and is namespace-scoped, but its target namespace %q is not Velero's namespace %q", groupResource.String(), selectedItem.name, targetNS, ctx.restore.Namespace) + ctx.log.WithFields(logrus.Fields{ + "resource": groupResource.String(), + "name": selectedItem.name, + "targetNamespace": targetNS, + "veleroNamespace": ctx.restore.Namespace, + }).Error(err.Error()) + errs.Add(targetNS, err) + continue + } // If we don't know whether this namespace exists yet, attempt to create // it in order to ensure it exists. Try to get it from the backup tarball // (in order to get any backed-up metadata), but if we don't find it there, @@ -2072,10 +2085,7 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso return warnings, errs, itemExists } - // Do not create podvolumerestore when current restore excludes pv/pvc - if ctx.resourceIncludesExcludes.ShouldInclude(kuberesource.PersistentVolumeClaims.String()) && - ctx.resourceIncludesExcludes.ShouldInclude(kuberesource.PersistentVolumes.String()) && - len(podvolume.GetVolumeBackupsForPod(ctx.podVolumeBackups, pod, originalNamespace)) > 0 { + if len(podvolume.GetVolumeBackupsForPod(ctx.podVolumeBackups, pod, originalNamespace)) > 0 { restorePodVolumeBackups(ctx, createdObj, originalNamespace) } } @@ -2833,7 +2843,7 @@ func (ctx *restoreContext) getSelectedRestoreableItems(resource string, original } if skipItem { - ctx.log.Infof("restore orSelector labels did not match, skipping restore of item: %s", skipItem, item) + ctx.log.Infof("restore orSelector labels did not match, skipping restore of item: %s", item) continue } } diff --git a/pkg/restore/restore_test.go b/pkg/restore/restore_test.go index 5b013b2be..fdb6f20c4 100644 --- a/pkg/restore/restore_test.go +++ b/pkg/restore/restore_test.go @@ -813,6 +813,87 @@ func TestRestoreResourceFiltering(t *testing.T) { } } +func TestRestoreMustHaveResourceNamespaceEnforcement(t *testing.T) { + tests := []struct { + name string + restore *velerov1api.Restore + backup *velerov1api.Backup + apiResources []*test.APIResource + tarball io.Reader + want map[*test.APIResource][]string + expectError bool + }{ + { + name: "resourceMustHave item in velero namespace is restored", + restore: defaultRestore().IncludedNamespaces("velero").Result(), + backup: defaultBackup().Result(), + tarball: test.NewTarWriter(t). + AddItems("datauploads.velero.io", + builder.ForDataUpload("velero", "du-1").Result(), + ). + Done(), + apiResources: []*test.APIResource{ + test.DataUploads(), + }, + want: map[*test.APIResource][]string{ + test.DataUploads(): {"velero/du-1"}, + }, + expectError: false, + }, + { + name: "resourceMustHave item outside velero namespace is rejected and produces error", + restore: defaultRestore().IncludedNamespaces("app-foo").Result(), + backup: defaultBackup().Result(), + tarball: test.NewTarWriter(t). + AddItems("datauploads.velero.io", + builder.ForDataUpload("attacker-ns", "du-2").Result(), + ). + Done(), + apiResources: []*test.APIResource{ + test.DataUploads(), + }, + want: map[*test.APIResource][]string{ + test.DataUploads(): {}, + }, + expectError: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + h := newHarness(t) + + for _, r := range tc.apiResources { + h.DiscoveryClient.WithAPIResource(r) + } + require.NoError(t, h.restorer.discoveryHelper.Refresh()) + + resPolicies, err := resourcepolicies.GetResourcePoliciesFromRestore(t.Context(), tc.restore, h.restorer.kbClient, h.log) + require.NoError(t, err) + + data := &Request{ + Log: h.log, + Restore: tc.restore, + Backup: tc.backup, + BackupReader: tc.tarball, + ResPolicies: resPolicies, + } + _, errs := h.restorer.Restore( + data, + nil, + nil, + ) + + if tc.expectError { + assert.False(t, errs.IsEmpty(), "expected errors but got empty") + } else { + assert.True(t, errs.IsEmpty(), "expected no errors but got %v", errs) + } + assertAPIContents(t, h, tc.want) + }) + } +} + // TestRestoreNamespaceMapping runs restores with namespace mappings specified, // and verifies that the set of items created in the API are in the correct // namespaces. Validation is done by looking at the namespaces/names of the items diff --git a/pkg/types/node_agent.go b/pkg/types/node_agent.go index 42fe06f58..16e50b283 100644 --- a/pkg/types/node_agent.go +++ b/pkg/types/node_agent.go @@ -57,13 +57,44 @@ type BackupPVC struct { // ignored if ReadOnly is false SPCNoRelabeling bool `json:"spcNoRelabeling,omitempty"` + // ReadWriteOncePod sets the backupPVC's access mode to ReadWriteOncePod so the kubelet can use + // mount-level SELinux labeling (-o context) instead of per-file relabeling, when the CSI driver + // advertises SELinux mount support. + // ignored if ReadOnly is true + ReadWriteOncePod bool `json:"readWriteOncePod,omitempty"` + // Annotations permits setting annotations for the backupPVC Annotations map[string]string `json:"annotations,omitempty"` + + // SecretNames is a list of secret names to copy from the source PVC namespace + // to the Velero namespace before creating the backupPVC. The secrets are deleted + // after the DataUpload completes. This is needed for CSI drivers that require + // namespace-scoped secrets for volume provisioning (e.g., encrypted volumes). + SecretNames []string `json:"secretNames,omitempty"` + + // ConfigMapNames is a list of configmap names to copy from the source PVC namespace + // to the Velero namespace before creating the backupPVC. The configmaps are deleted + // after the DataUpload completes. This is needed for CSI drivers that require + // namespace-scoped configmaps for volume provisioning (e.g., tenant-specific + // Vault connection overrides for encrypted volumes). + ConfigMapNames []string `json:"configMapNames,omitempty"` } type RestorePVC struct { // IgnoreDelayBinding indicates to ignore delay binding the restorePVC when it is in WaitForFirstConsumer mode IgnoreDelayBinding bool `json:"ignoreDelayBinding,omitempty"` + + // SecretNames is a list of secret names to copy from the target namespace to the + // Velero namespace before creating the restorePVC. The secrets are deleted after the + // DataDownload completes. This is needed for CSI drivers that require namespace-scoped + // secrets for volume provisioning (e.g., encrypted volumes). + SecretNames []string `json:"secretNames,omitempty"` + + // ConfigMapNames is a list of configmap names to copy from the target namespace to the + // Velero namespace before creating the restorePVC. The configmaps are deleted after the + // DataDownload completes. This is needed for CSI drivers that require namespace-scoped + // configmaps for volume provisioning (e.g., tenant-specific Vault connection overrides). + ConfigMapNames []string `json:"configMapNames,omitempty"` } type CachePVC struct { diff --git a/pkg/uploader/block/snapshot.go b/pkg/uploader/block/snapshot.go index adec352ef..1ecfedbbe 100644 --- a/pkg/uploader/block/snapshot.go +++ b/pkg/uploader/block/snapshot.go @@ -149,6 +149,12 @@ func snapshotSource( func getParentBackupInfo(ctx context.Context, rep udmrepo.BackupRepo, forceFull bool, parentSnapshot string, volumeID string, realSource string, snapshotTags map[string]string, log logrus.FieldLogger) parentBackupInfo { var previous *udmrepo.Snapshot + + // parentID names whichever snapshot ended up being the parent. On the discovery + // branch the parentSnapshot parameter is empty by definition, so logging it there + // produces messages that describe a decision without naming the object it was about. + parentID := parentSnapshot + if !forceFull { if parentSnapshot != "" { snap, err := rep.GetSnapshot(ctx, udmrepo.ID(parentSnapshot)) @@ -166,6 +172,7 @@ func getParentBackupInfo(ctx context.Context, rep udmrepo.BackupRepo, forceFull log.WithError(err).Warn("Failed to search previous snapshot, fallback to full backup") } else { previous = &snap + parentID = string(snap.RootObject.ID) log.Infof("Using previous snapshot %s", snap.RootObject.ID) } } @@ -176,21 +183,21 @@ func getParentBackupInfo(ctx context.Context, rep udmrepo.BackupRepo, forceFull parentInfo := parentBackupInfo{} if previous != nil { if previous.Tags == nil { - log.Warnf("No tag from parent snapshot %s, fallback to full backup", parentSnapshot) + log.Warnf("No tag from parent snapshot %s, fallback to full backup", parentID) } else if previous.Tags[uploader.CBTChangeIDTag] == "" { - log.Warnf("No ChangeID tag from parent snapshot %s, fallback to full backup", parentSnapshot) + log.Warnf("No ChangeID tag from parent snapshot %s, fallback to full backup", parentID) } else if previous.Tags[uploader.CBTVolumeIDTag] == "" { - log.Warnf("No VolumeID tag from parent snapshot %s, fallback to full backup", parentSnapshot) + log.Warnf("No VolumeID tag from parent snapshot %s, fallback to full backup", parentID) } else if previous.Tags[uploader.CBTVolumeIDTag] != volumeID { - log.Warnf("VolumeID %s from parent snapshot %s is not expected as %s, fallback to full backup", previous.Tags[uploader.CBTVolumeIDTag], parentSnapshot, volumeID) + log.Warnf("VolumeID %s from parent snapshot %s is not expected as %s, fallback to full backup", previous.Tags[uploader.CBTVolumeIDTag], parentID, volumeID) } else if obj, err := loadObjectFromSnapshot(ctx, rep, previous); err != nil { - log.WithError(err).Warnf("Failed to load object from parent snapshot %s, fallback to full backup", parentSnapshot) + log.WithError(err).Warnf("Failed to load object from parent snapshot %s, fallback to full backup", parentID) } else { parentInfo.parentObject = obj parentInfo.changeID = previous.Tags[uploader.CBTChangeIDTag] parentInfo.volumeID = previous.Tags[uploader.CBTVolumeIDTag] - log.Infof("Using parent snapshot %s, start time %v, end time %v, description %s", parentSnapshot, previous.StartTime, previous.EndTime, previous.Description) + log.Infof("Using parent snapshot %s, start time %v, end time %v, description %s", parentID, previous.StartTime, previous.EndTime, previous.Description) } } diff --git a/pkg/uploader/block/snapshot_test.go b/pkg/uploader/block/snapshot_test.go index fa77b2d10..3cebd10bc 100644 --- a/pkg/uploader/block/snapshot_test.go +++ b/pkg/uploader/block/snapshot_test.go @@ -21,11 +21,13 @@ package block import ( "context" "os" + "strings" "testing" "time" "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" + logrustest "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -275,6 +277,58 @@ func TestSnapshotSource(t *testing.T) { } } +// TestGetParentBackupInfoLogsDiscoveredParentID pins that the parent-selection messages +// name the snapshot they are about. On the discovery branch the parentSnapshot parameter +// is empty by definition, so logging it there emits "Using parent snapshot , start time ..." +// - a decision logged without the identifier needed to act on it. +func TestGetParentBackupInfoLogsDiscoveredParentID(t *testing.T) { + const volumeID = "vol-123" + const realSource = "/test/source" + const rootObj = "root-obj-42" + + snapshotTags := map[string]string{ + uploader.SnapshotRequesterTag: "test-requester", + uploader.SnapshotUploaderTag: uploader.BlockType, + } + + logger, hook := logrustest.NewNullLogger() + logger.SetLevel(logrus.DebugLevel) + + repo := udmrepomocks.NewBackupRepo(t) + repo.On("ListSnapshot", mock.Anything, realSource). + Return([]udmrepo.Snapshot{{ + RootObject: udmrepo.ObjectMetadata{ID: rootObj}, + Tags: map[string]string{ + uploader.CBTChangeIDTag: "cid-abc", + uploader.CBTVolumeIDTag: volumeID, + uploader.SnapshotRequesterTag: "test-requester", + uploader.SnapshotUploaderTag: uploader.BlockType, + }, + }}, nil) + repo.On("ReadMetadata", mock.Anything, udmrepo.ID(rootObj)). + Return(&udmrepo.Metadata{ + SubObjects: []udmrepo.ObjectMetadata{{ID: udmrepo.ID("parent-obj")}}, + }, nil) + + info := getParentBackupInfo( + context.Background(), repo, + false, "", // no explicit parent -> discovery branch + volumeID, realSource, snapshotTags, logger, + ) + + require.Equal(t, udmrepo.ID("parent-obj"), info.parentObject) + + var found bool + for _, entry := range hook.AllEntries() { + if strings.HasPrefix(entry.Message, "Using parent snapshot ") { + found = true + assert.Contains(t, entry.Message, rootObj, + "parent-selection message must name the discovered snapshot, got %q", entry.Message) + } + } + require.True(t, found, "expected a \"Using parent snapshot\" message") +} + func TestGetParentBackupInfo(t *testing.T) { const volumeID = "vol-123" const realSource = "/test/source" diff --git a/pkg/uploader/provider/block.go b/pkg/uploader/provider/block.go index 6a7ae2802..e37a0c16f 100644 --- a/pkg/uploader/provider/block.go +++ b/pkg/uploader/provider/block.go @@ -134,7 +134,11 @@ func (bp *blockProvider) RunBackup( snapshotInfo, _, err := blockBackupFunc(ctx, blkUploader, bp.bkRepo, path, realSource, cbtParam.Source, forceFull, parentSnapshot, cbtParam.Service, uploaderCfg, tags, log) - if err == block.ErrCanceled { + // errors.Is, not ==: the sentinel is wrapped twice on its way here, by + // block/uploader.go ("error backing up bdev %s") and again by + // block/snapshot.go ("Failed to run uploader backup for si %v"), so an + // equality check never matches and cancellation gets reported as a failure. + if errors.Is(err, block.ErrCanceled) { log.Warn("Block backup is canceled") return snapshotInfo.ID, false, snapshotInfo.Size, snapshotInfo.IncrementalSize, ErrorCanceled } @@ -176,7 +180,8 @@ func (bp *blockProvider) RunRestore( size, err := blockRestoreFunc(ctx, blkUploader, bp.bkRepo, snapshotID, volumePath, uploaderCfg, log) - if err == block.ErrCanceled { + // errors.Is, not ==: see the equivalent comment on the backup path above. + if errors.Is(err, block.ErrCanceled) { log.Warn("Block restore is canceled") return 0, ErrorCanceled } diff --git a/pkg/uploader/provider/block_test.go b/pkg/uploader/provider/block_test.go index ad8f68b52..42375be20 100644 --- a/pkg/uploader/provider/block_test.go +++ b/pkg/uploader/provider/block_test.go @@ -372,6 +372,63 @@ func TestBlockProviderRunBackup(t *testing.T) { } } +// TestBlockProviderCancelThroughWrappedError pins that cancellation is recognized +// after the sentinel has been wrapped, which is the only way it ever arrives in +// production: block/uploader.go wraps it with "error backing up bdev %s" and +// block/snapshot.go wraps that with "Failed to run uploader backup for si %v". +// +// Asserting on the message is useless here — provider.ErrorCanceled and +// block.ErrCanceled carry the *same* text ("uploader is canceled"), so a substring +// check passes whether or not the sentinel was actually recognized. The assertion +// has to be on identity. +func TestBlockProviderCancelThroughWrappedError(t *testing.T) { + t.Run("backup", func(t *testing.T) { + orig := blockBackupFunc + defer func() { blockBackupFunc = orig }() + blockBackupFunc = func(_ context.Context, _ block.Uploader, _ udmrepo.BackupRepo, _ string, _ string, _ cbtservice.SourceInfo, _ bool, _ string, _ cbtservice.Service, _ map[string]string, _ map[string]string, _ logrus.FieldLogger) (uploader.SnapshotInfo, bool, error) { + return uploader.SnapshotInfo{ID: "snap-cancel", Size: 2048, IncrementalSize: 1024}, false, + errors.Wrapf( + errors.Wrapf(block.ErrCanceled, "error backing up bdev %s", "ns/pvc"), + "Failed to run uploader backup for si %v", "si") + } + + bp := &blockProvider{ + requestorType: "test", + bkRepo: udmrepomocks.NewBackupRepo(t), + log: logrus.New(), + } + + _, _, _, _, err := bp.RunBackup( + t.Context(), "/dev/sda", "ns/pvc", map[string]string{}, false, "", + CBTParam{}, uploader.PersistentVolumeBlock, map[string]string{}, + &FakeBackupProgressUpdater{}, + ) + + require.ErrorIs(t, err, ErrorCanceled, + "a wrapped block.ErrCanceled must surface as provider.ErrorCanceled; otherwise the "+ + "DataUpload is marked Failed and the Backup PartiallyFailed for a user-requested cancel") + }) + + t.Run("restore", func(t *testing.T) { + orig := blockRestoreFunc + defer func() { blockRestoreFunc = orig }() + blockRestoreFunc = func(_ context.Context, _ block.Uploader, _ udmrepo.BackupRepo, _ string, _ string, _ map[string]string, _ logrus.FieldLogger) (int64, error) { + return 0, errors.Wrap(block.ErrCanceled, "error restoring bdev") + } + + bp := &blockProvider{ + requestorType: "test", + bkRepo: udmrepomocks.NewBackupRepo(t), + log: logrus.New(), + } + + _, err := bp.RunRestore(t.Context(), "snap-1", "/dev/sda", + uploader.PersistentVolumeBlock, map[string]string{}, &blockMockProgressUpdater{}) + + require.ErrorIs(t, err, ErrorCanceled) + }) +} + func TestBlockProviderRunRestore(t *testing.T) { testCases := []struct { name string diff --git a/pkg/util/csi/volume_snapshot.go b/pkg/util/csi/volume_snapshot.go index e8fe9bead..330f0a73a 100644 --- a/pkg/util/csi/volume_snapshot.go +++ b/pkg/util/csi/volume_snapshot.go @@ -112,6 +112,7 @@ func WaitVolumeSnapshotReady( // GetVolumeSnapshotContentForVolumeSnapshot returns the VolumeSnapshotContent // object associated with the VolumeSnapshot. func GetVolumeSnapshotContentForVolumeSnapshot( + ctx context.Context, volSnap *snapshotv1api.VolumeSnapshot, snapshotClient snapshotter.SnapshotV1Interface, ) (*snapshotv1api.VolumeSnapshotContent, error) { @@ -120,7 +121,7 @@ func GetVolumeSnapshotContentForVolumeSnapshot( } vsc, err := snapshotClient.VolumeSnapshotContents().Get( - context.TODO(), + ctx, *volSnap.Status.BoundVolumeSnapshotContentName, metav1.GetOptions{}, ) @@ -186,6 +187,12 @@ func EnsureDeleteVS(ctx context.Context, snapshotClient snapshotter.SnapshotV1In if err != nil { if errors.Is(err, context.DeadlineExceeded) { + // updated is only set once the VS has been retrieved successfully, so it + // is still nil when the deadline is exceeded before that happens, e.g. + // when the first Get times out. No finalizers are available to report. + if updated == nil { + return errors.Errorf("timeout to assure VolumeSnapshot %s is deleted", vsName) + } return errors.Errorf("timeout to assure VolumeSnapshot %s is deleted, finalizers in VS %v", vsName, updated.Finalizers) } else { return errors.Wrapf(err, "error to assure VolumeSnapshot is deleted, %s", vsName) @@ -245,6 +252,12 @@ func EnsureDeleteVSC(ctx context.Context, snapshotClient snapshotter.SnapshotV1I if err != nil { if errors.Is(err, context.DeadlineExceeded) { + // updated is only set once the VSC has been retrieved successfully, so it + // is still nil when the deadline is exceeded before that happens, e.g. + // when the first Get times out. No finalizers are available to report. + if updated == nil { + return errors.Errorf("timeout to assure VolumeSnapshotContent %s is deleted", vscName) + } return errors.Errorf("timeout to assure VolumeSnapshotContent %s is deleted, finalizers in VSC %v", vscName, updated.Finalizers) } else { return errors.Wrapf(err, "error to assure VolumeSnapshotContent is deleted, %s", vscName) @@ -309,6 +322,7 @@ func patchVSC( } func GetVolumeSnapshotClass( + ctx context.Context, provisioner string, backup *velerov1api.Backup, pvc *corev1api.PersistentVolumeClaim, @@ -317,7 +331,7 @@ func GetVolumeSnapshotClass( policySnapshotClass string, ) (*snapshotv1api.VolumeSnapshotClass, error) { snapshotClasses := new(snapshotv1api.VolumeSnapshotClassList) - err := crClient.List(context.TODO(), snapshotClasses) + err := crClient.List(ctx, snapshotClasses) if err != nil { return nil, errors.Wrap(err, "error listing VolumeSnapshotClass") } @@ -517,13 +531,14 @@ func IsVolumeSnapshotContentHasDeleteSecret(vsc *snapshotv1api.VolumeSnapshotCon // IsVolumeSnapshotExists returns whether a specific volumesnapshot object exists. func IsVolumeSnapshotExists( + ctx context.Context, ns, name string, crClient crclient.Client, ) bool { vs := new(snapshotv1api.VolumeSnapshot) err := crClient.Get( - context.TODO(), + ctx, crclient.ObjectKey{Namespace: ns, Name: name}, vs, ) @@ -532,24 +547,26 @@ func IsVolumeSnapshotExists( } func SetVolumeSnapshotContentDeletionPolicy( + ctx context.Context, vscName string, crClient crclient.Client, policy snapshotv1api.DeletionPolicy, ) (*snapshotv1api.VolumeSnapshotContent, error) { vsc := new(snapshotv1api.VolumeSnapshotContent) - if err := crClient.Get(context.TODO(), crclient.ObjectKey{Name: vscName}, vsc); err != nil { + if err := crClient.Get(ctx, crclient.ObjectKey{Name: vscName}, vsc); err != nil { return nil, err } originVSC := vsc.DeepCopy() vsc.Spec.DeletionPolicy = policy - return vsc, crClient.Patch(context.TODO(), vsc, crclient.MergeFrom(originVSC)) + return vsc, crClient.Patch(ctx, vsc, crclient.MergeFrom(originVSC)) } // CleanupVolumeSnapshot deletes the VolumeSnapshot and the associated VolumeSnapshotContent. It will make sure the // physical snapshot is also deleted. func CleanupVolumeSnapshot( + ctx context.Context, volSnap *snapshotv1api.VolumeSnapshot, crClient crclient.Client, log logrus.FieldLogger, @@ -557,7 +574,7 @@ func CleanupVolumeSnapshot( log.Infof("Deleting Volumesnapshot %s/%s", volSnap.Namespace, volSnap.Name) vs := new(snapshotv1api.VolumeSnapshot) err := crClient.Get( - context.TODO(), + ctx, crclient.ObjectKey{Name: volSnap.Name, Namespace: volSnap.Namespace}, vs, ) @@ -570,6 +587,7 @@ func CleanupVolumeSnapshot( // we patch the DeletionPolicy of the VolumeSnapshotContent to set it to Delete. // This ensures that the volume snapshot in the storage provider is also deleted. _, err := SetVolumeSnapshotContentDeletionPolicy( + ctx, *vs.Status.BoundVolumeSnapshotContentName, crClient, snapshotv1api.VolumeSnapshotContentDelete, @@ -579,7 +597,7 @@ func CleanupVolumeSnapshot( vs.Namespace, vs.Name) } } - err = crClient.Delete(context.TODO(), vs) + err = crClient.Delete(ctx, vs) if err != nil { log.Debugf("Failed to delete volumesnapshot %s/%s: %v", vs.Namespace, vs.Name, err) } else { @@ -589,6 +607,7 @@ func CleanupVolumeSnapshot( } func DeleteReadyVolumeSnapshot( + ctx context.Context, vs snapshotv1api.VolumeSnapshot, client crclient.Client, logger logrus.FieldLogger, @@ -610,6 +629,7 @@ func DeleteReadyVolumeSnapshot( // Patch the DeletionPolicy of the VolumeSnapshotContent to set it to Retain. // This ensures that the volume snapshot in the storage provider is kept. if vsc, err = SetVolumeSnapshotContentDeletionPolicy( + ctx, *vs.Status.BoundVolumeSnapshotContentName, client, snapshotv1api.VolumeSnapshotContentRetain, @@ -619,11 +639,11 @@ func DeleteReadyVolumeSnapshot( return } - if err := client.Delete(context.TODO(), vsc); err != nil { + if err := client.Delete(ctx, vsc); err != nil { logger.WithError(err).Warnf("Failed to delete the VolumeSnapshotContent %s", vsc.Name) } } - if err := client.Delete(context.TODO(), &vs); err != nil { + if err := client.Delete(ctx, &vs); err != nil { logger.WithError(err).Warnf("Failed to delete VolumeSnapshot %s", vs.Namespace+"/"+vs.Name) } else { logger.Infof("Deleted VolumeSnapshot %s and VolumeSnapshotContent %s", @@ -693,7 +713,7 @@ func WaitUntilVSCHandleIsReady( if vsc.Status != nil && vsc.Status.Error != nil { log.Warnf("VolumeSnapshotContent %s has error: %v", - vsc.Name, *vsc.Status.Error.Message) + vsc.Name, stringptr.GetString(vsc.Status.Error.Message)) } return false, nil } @@ -744,10 +764,10 @@ func WaitUntilVSCHandleIsReady( vsc.Status.Error != nil { log.Errorf( "Timed out awaiting reconciliation of VolumeSnapshot, VolumeSnapshotContent %s has error: %v", - vsc.Name, *vsc.Status.Error.Message) + vsc.Name, stringptr.GetString(vsc.Status.Error.Message)) return nil, errors.Errorf("CSI got timed out with error: %v", - *vsc.Status.Error.Message) + stringptr.GetString(vsc.Status.Error.Message)) } else { log.Errorf( "Timed out awaiting reconciliation of volumesnapshot %s/%s", diff --git a/pkg/util/csi/volume_snapshot_test.go b/pkg/util/csi/volume_snapshot_test.go index 335cff6ee..895935b4b 100644 --- a/pkg/util/csi/volume_snapshot_test.go +++ b/pkg/util/csi/volume_snapshot_test.go @@ -17,6 +17,7 @@ limitations under the License. package csi import ( + "context" "errors" "testing" "time" @@ -201,7 +202,7 @@ func TestWaitVolumeSnapshotReady(t *testing.T) { fakeClient := snapshotFake.NewSimpleClientset(test.clientObj...) vs, err := WaitVolumeSnapshotReady(t.Context(), fakeClient.SnapshotV1(), test.vsName, test.namespace, time.Millisecond, velerotest.NewLogger()) - if err != nil { + if test.err != "" { require.EqualError(t, err, test.err) } else { require.NoError(t, err) @@ -286,8 +287,8 @@ func TestGetVolumeSnapshotContentForVolumeSnapshot(t *testing.T) { t.Run(test.name, func(t *testing.T) { fakeClient := snapshotFake.NewSimpleClientset(test.clientObj...) - vs, err := GetVolumeSnapshotContentForVolumeSnapshot(test.snapshotObj, fakeClient.SnapshotV1()) - if err != nil { + vs, err := GetVolumeSnapshotContentForVolumeSnapshot(context.TODO(), test.snapshotObj, fakeClient.SnapshotV1()) + if test.err != "" { require.EqualError(t, err, test.err) } else { require.NoError(t, err) @@ -376,6 +377,29 @@ func TestEnsureDeleteVS(t *testing.T) { }, err: "timeout to assure VolumeSnapshot fake-vs is deleted, finalizers in VS []", }, + { + name: "wait timeout before the VS is ever retrieved", + vsName: "fake-vs", + namespace: "fake-ns", + clientObj: []runtime.Object{vsObjWithFinalizer}, + reactors: []reactor{ + { + verb: "delete", + resource: "volumesnapshots", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, nil + }, + }, + { + verb: "get", + resource: "volumesnapshots", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, context.DeadlineExceeded + }, + }, + }, + err: "timeout to assure VolumeSnapshot fake-vs is deleted", + }, { name: "success", vsName: "fake-vs", @@ -393,7 +417,7 @@ func TestEnsureDeleteVS(t *testing.T) { } err := EnsureDeleteVS(t.Context(), fakeSnapshotClient.SnapshotV1(), test.vsName, test.namespace, time.Millisecond) - if err != nil { + if test.err != "" { assert.EqualError(t, err, test.err) } else { assert.NoError(t, err) @@ -487,6 +511,28 @@ func TestEnsureDeleteVSC(t *testing.T) { }, err: "timeout to assure VolumeSnapshotContent fake-vsc is deleted, finalizers in VSC []", }, + { + name: "wait timeout before the VSC is ever retrieved", + vscName: "fake-vsc", + clientObj: []runtime.Object{vscObjWithFinalizer}, + reactors: []reactor{ + { + verb: "delete", + resource: "volumesnapshotcontents", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, nil + }, + }, + { + verb: "get", + resource: "volumesnapshotcontents", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, context.DeadlineExceeded + }, + }, + }, + err: "timeout to assure VolumeSnapshotContent fake-vsc is deleted", + }, { name: "success", vscName: "fake-vsc", @@ -1032,6 +1078,7 @@ func TestGetVolumeSnapshotClass(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { actualSnapshotClass, actualError := GetVolumeSnapshotClass( + context.TODO(), tc.driverName, tc.backup, tc.pvc, logrus.New(), fakeClient, "") if tc.expectError { require.Error(t, actualError) @@ -1458,7 +1505,7 @@ func TestIsVolumeSnapshotExists(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - actual := IsVolumeSnapshotExists(tc.vs.Namespace, tc.vs.Name, fakeClient) + actual := IsVolumeSnapshotExists(context.TODO(), tc.vs.Namespace, tc.vs.Name, fakeClient) assert.Equal(t, tc.expected, actual) }) } @@ -1529,7 +1576,7 @@ func TestSetVolumeSnapshotContentDeletionPolicy(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { fakeClient := velerotest.NewFakeControllerRuntimeClient(t, tc.objs...) - _, err := SetVolumeSnapshotContentDeletionPolicy(tc.inputVSCName, fakeClient, tc.policy) + _, err := SetVolumeSnapshotContentDeletionPolicy(context.TODO(), tc.inputVSCName, fakeClient, tc.policy) if tc.expectError { assert.Error(t, err) } else { @@ -1586,7 +1633,7 @@ func TestDeleteVolumeSnapshots(t *testing.T) { ) logger := logging.DefaultLogger(logrus.DebugLevel, logging.FormatText) - DeleteReadyVolumeSnapshot(tc.vs, client, logger) + DeleteReadyVolumeSnapshot(context.TODO(), tc.vs, client, logger) vsList := new(snapshotv1api.VolumeSnapshotList) err := client.List( @@ -1719,6 +1766,34 @@ func TestWaitUntilVSCHandleIsReady(t *testing.T) { }, } + errNoMessageVsc := "err-no-message-vsc" + vscWithErrorNoMessage := &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: errNoMessageVsc, + }, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + VolumeSnapshotRef: corev1api.ObjectReference{ + Name: "vol-snap-1", + APIVersion: snapshotv1api.SchemeGroupVersion.String(), + }, + }, + Status: &snapshotv1api.VolumeSnapshotContentStatus{ + SnapshotHandle: nil, + // Error is set while Message is left nil. Both are optional in the + // CSI API, so the error-reporting paths must not dereference Message. + Error: &snapshotv1api.VolumeSnapshotError{Message: nil}, + }, + } + vsForErrorNoMessageVsc := &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs-for-err-no-message", + Namespace: "default", + }, + Status: &snapshotv1api.VolumeSnapshotStatus{ + BoundVolumeSnapshotContentName: &errNoMessageVsc, + }, + } + objs := []runtime.Object{ vscObj, validVS, @@ -1729,6 +1804,8 @@ func TestWaitUntilVSCHandleIsReady(t *testing.T) { vsForNilStatusVsc, vscWithNilStatusField, vsForNilStatusFieldVsc, + vscWithErrorNoMessage, + vsForErrorNoMessageVsc, } fakeClient := velerotest.NewFakeControllerRuntimeClient(t, objs...) testCases := []struct { @@ -1763,6 +1840,12 @@ func TestWaitUntilVSCHandleIsReady(t *testing.T) { }, }, }, + { + name: "waitEnabled should return an error rather than panic when the volumesnapshotcontent has an error without a message", + volSnap: vsForErrorNoMessageVsc, + exepctedVSC: nil, + expectError: true, + }, } for _, tc := range testCases { diff --git a/pkg/util/kube/node.go b/pkg/util/kube/node.go index 3426e508f..6fc2974c9 100644 --- a/pkg/util/kube/node.go +++ b/pkg/util/kube/node.go @@ -30,7 +30,7 @@ import ( const ( NodeOSLinux = "linux" NodeOSWindows = "windows" - NodeOSLabel = "kubernetes.io/os" + NodeOSLabel = corev1api.LabelOSStable ) var realNodeOSMap = map[string]string{ diff --git a/pkg/util/kube/node_test.go b/pkg/util/kube/node_test.go index 612b8f977..41e7b7806 100644 --- a/pkg/util/kube/node_test.go +++ b/pkg/util/kube/node_test.go @@ -35,8 +35,8 @@ import ( func TestIsLinuxNode(t *testing.T) { nodeNoOSLabel := builder.ForNode("fake-node").Result() - nodeWindows := builder.ForNode("fake-node").Labels(map[string]string{"kubernetes.io/os": "windows"}).Result() - nodeLinux := builder.ForNode("fake-node").Labels(map[string]string{"kubernetes.io/os": "linux"}).Result() + nodeWindows := builder.ForNode("fake-node").Labels(map[string]string{corev1api.LabelOSStable: "windows"}).Result() + nodeLinux := builder.ForNode("fake-node").Labels(map[string]string{corev1api.LabelOSStable: "linux"}).Result() scheme := runtime.NewScheme() corev1api.AddToScheme(scheme) @@ -90,8 +90,8 @@ func TestIsLinuxNode(t *testing.T) { } func TestWithLinuxNode(t *testing.T) { - nodeWindows := builder.ForNode("fake-node-1").Labels(map[string]string{"kubernetes.io/os": "windows"}).Result() - nodeLinux := builder.ForNode("fake-node-2").Labels(map[string]string{"kubernetes.io/os": "linux"}).Result() + nodeWindows := builder.ForNode("fake-node-1").Labels(map[string]string{corev1api.LabelOSStable: "windows"}).Result() + nodeLinux := builder.ForNode("fake-node-2").Labels(map[string]string{corev1api.LabelOSStable: "linux"}).Result() scheme := runtime.NewScheme() corev1api.AddToScheme(scheme) @@ -135,8 +135,8 @@ func TestWithLinuxNode(t *testing.T) { func TestGetNodeOSType(t *testing.T) { nodeNoOSLabel := builder.ForNode("fake-node").Result() - nodeWindows := builder.ForNode("fake-node").Labels(map[string]string{"kubernetes.io/os": "windows"}).Result() - nodeLinux := builder.ForNode("fake-node").Labels(map[string]string{"kubernetes.io/os": "linux"}).Result() + nodeWindows := builder.ForNode("fake-node").Labels(map[string]string{corev1api.LabelOSStable: "windows"}).Result() + nodeLinux := builder.ForNode("fake-node").Labels(map[string]string{corev1api.LabelOSStable: "linux"}).Result() scheme := runtime.NewScheme() corev1api.AddToScheme(scheme) tests := []struct { @@ -185,8 +185,8 @@ func TestGetNodeOSType(t *testing.T) { func TestHasNodeWithOS(t *testing.T) { nodeNoOSLabel := builder.ForNode("fake-node-1").Result() - nodeWindows := builder.ForNode("fake-node-2").Labels(map[string]string{"kubernetes.io/os": "windows"}).Result() - nodeLinux := builder.ForNode("fake-node-3").Labels(map[string]string{"kubernetes.io/os": "linux"}).Result() + nodeWindows := builder.ForNode("fake-node-2").Labels(map[string]string{corev1api.LabelOSStable: "windows"}).Result() + nodeLinux := builder.ForNode("fake-node-3").Labels(map[string]string{corev1api.LabelOSStable: "linux"}).Result() scheme := runtime.NewScheme() corev1api.AddToScheme(scheme) diff --git a/pkg/util/kube/pod.go b/pkg/util/kube/pod.go index 3ced95feb..25857cd02 100644 --- a/pkg/util/kube/pod.go +++ b/pkg/util/kube/pod.go @@ -129,6 +129,12 @@ func EnsureDeletePod(ctx context.Context, podGetter corev1client.CoreV1Interface if err != nil { if errors.Is(err, context.DeadlineExceeded) { + // updated is only set once the pod has been retrieved successfully, so it + // is still nil when the deadline is exceeded before that happens, e.g. + // when the first Get times out. No finalizers are available to report. + if updated == nil { + return errors.Errorf("timeout to assure pod %s is deleted", pod) + } return errors.Errorf("timeout to assure pod %s is deleted, finalizers in pod %v", pod, updated.Finalizers) } else { return errors.Wrapf(err, "error to assure pod is deleted, %s", pod) @@ -318,9 +324,26 @@ func ExitPodWithMessage(logger logrus.FieldLogger, succeed bool, message string, funcExit(exitCode) } +// deepCopy returns a deep copy of the LoadAffinity, so that the returned value +// can be safely modified without affecting the source. +func (a *LoadAffinity) deepCopy() *LoadAffinity { + if a == nil { + return nil + } + + result := &LoadAffinity{ + StorageClass: a.StorageClass, + } + a.NodeSelector.DeepCopyInto(&result.NodeSelector) + + return result +} + // GetLoadAffinityByStorageClass retrieves the LoadAffinity from the parameter affinityList. // The function first try to find by the scName. If there is no such LoadAffinity, // it will try to get the LoadAffinity whose StorageClass has no value. +// The returned LoadAffinity is a deep copy of the matched element, so that the +// callers can modify it without corrupting the shared node-agent configuration. func GetLoadAffinityByStorageClass( affinityList []*LoadAffinity, scName string, @@ -331,7 +354,7 @@ func GetLoadAffinityByStorageClass( for _, affinity := range affinityList { if affinity.StorageClass == scName { logger.WithField("StorageClass", scName).Info("Found pod's affinity setting per StorageClass.") - return affinity + return affinity.deepCopy() } if affinity.StorageClass == "" && globalAffinity == nil { @@ -345,5 +368,5 @@ func GetLoadAffinityByStorageClass( logger.Info("No Affinity is found for pod.") } - return globalAffinity + return globalAffinity.deepCopy() } diff --git a/pkg/util/kube/pod_test.go b/pkg/util/kube/pod_test.go index 1d54071c3..ec82fd1f8 100644 --- a/pkg/util/kube/pod_test.go +++ b/pkg/util/kube/pod_test.go @@ -106,6 +106,29 @@ func TestEnsureDeletePod(t *testing.T) { }, err: "timeout to assure pod fake-pod is deleted, finalizers in pod []", }, + { + name: "wait timeout before the pod is ever retrieved", + podName: "fake-pod", + namespace: "fake-ns", + clientObj: []runtime.Object{podObjectWithFinalizer}, + reactors: []reactor{ + { + verb: "delete", + resource: "pods", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, nil + }, + }, + { + verb: "get", + resource: "pods", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, context.DeadlineExceeded + }, + }, + }, + err: "timeout to assure pod fake-pod is deleted", + }, { name: "wait fail", podName: "fake-pod", @@ -1374,7 +1397,7 @@ func TestGetLoadAffinityByStorageClass(t *testing.T) { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { - Key: "kubernetes.io/arch", + Key: corev1api.LabelArchStable, Operator: metav1.LabelSelectorOpIn, Values: []string{"amd64"}, }, @@ -1386,7 +1409,7 @@ func TestGetLoadAffinityByStorageClass(t *testing.T) { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: metav1.LabelSelectorOpIn, Values: []string{"Linux"}, }, @@ -1399,7 +1422,7 @@ func TestGetLoadAffinityByStorageClass(t *testing.T) { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: metav1.LabelSelectorOpIn, Values: []string{"Linux"}, }, @@ -1414,7 +1437,7 @@ func TestGetLoadAffinityByStorageClass(t *testing.T) { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: metav1.LabelSelectorOpIn, Values: []string{"Linux"}, }, @@ -1425,7 +1448,7 @@ func TestGetLoadAffinityByStorageClass(t *testing.T) { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { - Key: "kubernetes.io/arch", + Key: corev1api.LabelArchStable, Operator: metav1.LabelSelectorOpIn, Values: []string{"amd64"}, }, @@ -1436,7 +1459,7 @@ func TestGetLoadAffinityByStorageClass(t *testing.T) { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: metav1.LabelSelectorOpIn, Values: []string{"Windows"}, }, @@ -1449,7 +1472,7 @@ func TestGetLoadAffinityByStorageClass(t *testing.T) { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: metav1.LabelSelectorOpIn, Values: []string{"Linux"}, }, @@ -1475,7 +1498,7 @@ func TestGetLoadAffinityByStorageClass(t *testing.T) { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: metav1.LabelSelectorOpIn, Values: []string{"Linux"}, }, @@ -1487,7 +1510,7 @@ func TestGetLoadAffinityByStorageClass(t *testing.T) { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { - Key: "kubernetes.io/arch", + Key: corev1api.LabelArchStable, Operator: metav1.LabelSelectorOpIn, Values: []string{"amd64"}, }, @@ -1501,7 +1524,7 @@ func TestGetLoadAffinityByStorageClass(t *testing.T) { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: metav1.LabelSelectorOpIn, Values: []string{"Linux"}, }, @@ -1517,7 +1540,7 @@ func TestGetLoadAffinityByStorageClass(t *testing.T) { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { - Key: "kubernetes.io/arch", + Key: corev1api.LabelArchStable, Operator: metav1.LabelSelectorOpIn, Values: []string{"amd64"}, }, @@ -1545,3 +1568,82 @@ func TestGetLoadAffinityByStorageClass(t *testing.T) { }) } } + +func TestGetLoadAffinityByStorageClassReturnsCopy(t *testing.T) { + newAffinityList := func() []*LoadAffinity { + return []*LoadAffinity{ + { + NodeSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{"pool": "backup"}, + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: corev1api.LabelArchStable, + Operator: metav1.LabelSelectorOpIn, + Values: []string{"amd64"}, + }, + }, + }, + }, + { + NodeSelector: metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: corev1api.LabelArchStable, + Operator: metav1.LabelSelectorOpIn, + Values: []string{"arm64"}, + }, + }, + }, + StorageClass: "storage-class-01", + }, + } + } + + tests := []struct { + name string + scName string + }{ + { + name: "global affinity", + scName: "no-such-storage-class", + }, + { + name: "affinity matched by StorageClass", + scName: "storage-class-01", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + affinityList := newAffinityList() + + // Simulate the exposers, which append an OS related term to the returned + // affinity on every expose call. The source list must not be affected. + for range 3 { + result := GetLoadAffinityByStorageClass(affinityList, test.scName, velerotest.NewLogger()) + require.NotNil(t, result) + + result.NodeSelector.MatchExpressions = append(result.NodeSelector.MatchExpressions, metav1.LabelSelectorRequirement{ + Key: NodeOSLabel, + Operator: metav1.LabelSelectorOpNotIn, + Values: []string{NodeOSWindows}, + }) + + assert.Len(t, result.NodeSelector.MatchExpressions, 2) + } + + assert.Equal(t, newAffinityList(), affinityList) + + // The other fields must be copied as well. + result := GetLoadAffinityByStorageClass(affinityList, test.scName, velerotest.NewLogger()) + require.NotNil(t, result) + result.StorageClass = "modified" + result.NodeSelector.MatchExpressions[0].Values[0] = "modified" + if result.NodeSelector.MatchLabels != nil { + result.NodeSelector.MatchLabels["pool"] = "modified" + } + + assert.Equal(t, newAffinityList(), affinityList) + }) + } +} diff --git a/pkg/util/kube/pvc_pv.go b/pkg/util/kube/pvc_pv.go index 7db9df3e4..b375ce0ba 100644 --- a/pkg/util/kube/pvc_pv.go +++ b/pkg/util/kube/pvc_pv.go @@ -153,6 +153,12 @@ func EnsureDeletePVC(ctx context.Context, pvcGetter corev1client.CoreV1Interface if err != nil { if errors.Is(err, context.DeadlineExceeded) { + // updated is only set once the PVC has been retrieved successfully, so it + // is still nil when the deadline is exceeded before that happens, e.g. + // when the first Get times out. No finalizers are available to report. + if updated == nil { + return errors.Errorf("timeout to assure pvc %s is deleted", pvcName) + } return errors.Errorf("timeout to assure pvc %s is deleted, finalizers in pvc %v", pvcName, updated.Finalizers) } else { return errors.Wrapf(err, "error to ensure pvc deleted for %s", pvcName) @@ -189,6 +195,12 @@ func EnsureDeletePV(ctx context.Context, pvGetter corev1client.CoreV1Interface, if err != nil { if errors.Is(err, context.DeadlineExceeded) { + // updated is only set once the PV has been retrieved successfully, so it + // is still nil when the deadline is exceeded before that happens, e.g. + // when the first Get times out. No finalizers are available to report. + if updated == nil { + return errors.Errorf("timeout to assure pv %s is deleted", pvName) + } return errors.Errorf("timeout to assure pv %s is deleted, finalizers in pv %v", pvName, updated.Finalizers) } else { return errors.Wrapf(err, "error to ensure pv deleted for %s", pvName) diff --git a/pkg/util/kube/pvc_pv_test.go b/pkg/util/kube/pvc_pv_test.go index 9b93f2971..c805929d7 100644 --- a/pkg/util/kube/pvc_pv_test.go +++ b/pkg/util/kube/pvc_pv_test.go @@ -17,6 +17,7 @@ limitations under the License. package kube import ( + "context" "testing" "time" @@ -687,6 +688,30 @@ func TestEnsureDeletePVC(t *testing.T) { }, err: "timeout to assure pvc fake-pvc is deleted, finalizers in pvc []", }, + { + name: "wait timeout before the pvc is ever retrieved", + pvcName: "fake-pvc", + namespace: "fake-ns", + clientObj: []runtime.Object{pvcObjectWithFinalizer}, + timeout: time.Millisecond, + reactors: []reactor{ + { + verb: "delete", + resource: "persistentvolumeclaims", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + return true, pvcObject, nil + }, + }, + { + verb: "get", + resource: "persistentvolumeclaims", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, context.DeadlineExceeded + }, + }, + }, + err: "timeout to assure pvc fake-pvc is deleted", + }, } for _, test := range tests { @@ -1729,7 +1754,7 @@ func TestDiagnosePV(t *testing.T) { func TestGetPVCAttachingNodeOS(t *testing.T) { storageClass := "fake-storage-class" nodeNoOSLabel := builder.ForNode("fake-node").Result() - nodeWindows := builder.ForNode("fake-node").Labels(map[string]string{"kubernetes.io/os": "windows"}).Result() + nodeWindows := builder.ForNode("fake-node").Labels(map[string]string{corev1api.LabelOSStable: "windows"}).Result() pvcObj := &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ @@ -2059,6 +2084,13 @@ func TestEnsureDeletePV(t *testing.T) { }, } + pvObjWithFinalizer := &corev1api.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-pv", + Finalizers: []string{"fake-finalizer-1", "fake-finalizer-2"}, + }, + } + tests := []struct { name string pvName string @@ -2134,6 +2166,29 @@ func TestEnsureDeletePV(t *testing.T) { }, expectedErr: "timeout to assure pv fake-pv is deleted, finalizers in pv []", }, + { + name: "wait timeout before the pv is ever retrieved", + pvName: "fake-pv", + timeout: time.Millisecond, + kubeClientObj: []runtime.Object{pvObjWithFinalizer}, + kubeReactors: []reactor{ + { + verb: "delete", + resource: "persistentvolumes", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, nil + }, + }, + { + verb: "get", + resource: "persistentvolumes", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, context.DeadlineExceeded + }, + }, + }, + expectedErr: "timeout to assure pv fake-pv is deleted", + }, } for _, test := range tests { diff --git a/pkg/util/kube/secrets.go b/pkg/util/kube/secrets.go index f1d19b84e..e949b0e97 100644 --- a/pkg/util/kube/secrets.go +++ b/pkg/util/kube/secrets.go @@ -18,9 +18,14 @@ package kube import ( "context" + "reflect" "github.com/cockroachdb/errors" + "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" kbclient "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -49,3 +54,151 @@ func GetSecretKey(client kbclient.Client, namespace string, selector *corev1api. return key, nil } + +// ErrSecretCollision is returned when a secret or configmap with the same name but different +// data already exists in the target namespace, indicating another owner is using it. +var ErrSecretCollision = errors.New("secret collision: same name exists with different data") + +// labelsMatch reports whether all entries in want are present in have with matching values. +func labelsMatch(have, want map[string]string) bool { + for k, v := range want { + if have[k] != v { + return false + } + } + return true +} + +// CopySecret copies a secret from sourceNamespace to targetNamespace, applying the given labels. +// If a secret with the same name already exists in the target with identical data and matching +// labels, it is a no-op. If the data matches but the labels differ, or the data differs, it +// returns ErrSecretCollision. +func CopySecret(ctx context.Context, client corev1client.CoreV1Interface, secretName, sourceNamespace, targetNamespace string, labels map[string]string, log logrus.FieldLogger) error { + srcSecret, err := client.Secrets(sourceNamespace).Get(ctx, secretName, metav1.GetOptions{}) + if err != nil { + return errors.Wrapf(err, "error getting secret %s/%s", sourceNamespace, secretName) + } + + newSecret := &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: targetNamespace, + Labels: labels, + }, + Type: srcSecret.Type, + Data: srcSecret.Data, + } + + _, err = client.Secrets(targetNamespace).Create(ctx, newSecret, metav1.CreateOptions{}) + if err == nil { + log.Infof("Copied secret %s from %s to %s", secretName, sourceNamespace, targetNamespace) + return nil + } + + if !apierrors.IsAlreadyExists(err) { + return errors.Wrapf(err, "error creating secret %s in %s", secretName, targetNamespace) + } + + existing, err := client.Secrets(targetNamespace).Get(ctx, secretName, metav1.GetOptions{}) + if err != nil { + return errors.Wrapf(err, "error getting existing secret %s/%s", targetNamespace, secretName) + } + + if reflect.DeepEqual(existing.Data, srcSecret.Data) && labelsMatch(existing.Labels, labels) { + log.Infof("Secret %s already exists in %s with same data and labels, skipping copy", secretName, targetNamespace) + return nil + } + + log.Infof("Secret %s already exists in %s owned by a different owner, collision detected", secretName, targetNamespace) + return ErrSecretCollision +} + +// DeleteSecretsWithLabel deletes all secrets in a namespace matching a label key=value pair. +// Uses UID preconditions to avoid deleting a recreated object with the same name. +func DeleteSecretsWithLabel(ctx context.Context, client corev1client.CoreV1Interface, namespace, labelKey, labelValue string, log logrus.FieldLogger) { + secrets, err := client.Secrets(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: labelKey + "=" + labelValue, + }) + if err != nil { + log.WithError(err).Errorf("Failed to list secrets with label %s=%s in %s", labelKey, labelValue, namespace) + return + } + + for i := range secrets.Items { + uid := secrets.Items[i].UID + err := client.Secrets(namespace).Delete(ctx, secrets.Items[i].Name, metav1.DeleteOptions{ + Preconditions: &metav1.Preconditions{UID: &uid}, + }) + if err != nil && !apierrors.IsNotFound(err) { + log.WithError(err).Errorf("Failed to delete secret %s/%s", namespace, secrets.Items[i].Name) + } + } +} + +// CopyConfigMap copies a configmap from sourceNamespace to targetNamespace, applying the given +// labels. If a configmap with the same name already exists in the target with identical data and +// matching labels, it is a no-op. If the data matches but the labels differ, or the data differs, +// it returns ErrSecretCollision. +func CopyConfigMap(ctx context.Context, client corev1client.CoreV1Interface, cmName, sourceNamespace, targetNamespace string, labels map[string]string, log logrus.FieldLogger) error { + srcCM, err := client.ConfigMaps(sourceNamespace).Get(ctx, cmName, metav1.GetOptions{}) + if err != nil { + return errors.Wrapf(err, "error getting configmap %s/%s", sourceNamespace, cmName) + } + + newCM := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: cmName, + Namespace: targetNamespace, + Labels: labels, + }, + Data: srcCM.Data, + BinaryData: srcCM.BinaryData, + } + + _, err = client.ConfigMaps(targetNamespace).Create(ctx, newCM, metav1.CreateOptions{}) + if err == nil { + log.Infof("Copied configmap %s from %s to %s", cmName, sourceNamespace, targetNamespace) + return nil + } + + if !apierrors.IsAlreadyExists(err) { + return errors.Wrapf(err, "error creating configmap %s in %s", cmName, targetNamespace) + } + + existing, err := client.ConfigMaps(targetNamespace).Get(ctx, cmName, metav1.GetOptions{}) + if err != nil { + return errors.Wrapf(err, "error getting existing configmap %s/%s", targetNamespace, cmName) + } + + if reflect.DeepEqual(existing.Data, srcCM.Data) && + reflect.DeepEqual(existing.BinaryData, srcCM.BinaryData) && + labelsMatch(existing.Labels, labels) { + log.Infof("ConfigMap %s already exists in %s with same data and labels, skipping copy", cmName, targetNamespace) + return nil + } + + log.Infof("ConfigMap %s already exists in %s owned by a different owner, collision detected", cmName, targetNamespace) + return ErrSecretCollision +} + +// DeleteConfigMapsWithLabel deletes all configmaps in a namespace matching a label key=value pair. +// Uses UID preconditions to avoid deleting a recreated object with the same name. +func DeleteConfigMapsWithLabel(ctx context.Context, client corev1client.CoreV1Interface, namespace, labelKey, labelValue string, log logrus.FieldLogger) { + cms, err := client.ConfigMaps(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: labelKey + "=" + labelValue, + }) + if err != nil { + log.WithError(err).Errorf("Failed to list configmaps with label %s=%s in %s", labelKey, labelValue, namespace) + return + } + + for i := range cms.Items { + uid := cms.Items[i].UID + err := client.ConfigMaps(namespace).Delete(ctx, cms.Items[i].Name, metav1.DeleteOptions{ + Preconditions: &metav1.Preconditions{UID: &uid}, + }) + if err != nil && !apierrors.IsNotFound(err) { + log.WithError(err).Errorf("Failed to delete configmap %s/%s", namespace, cms.Items[i].Name) + } + } +} diff --git a/pkg/util/kube/secrets_copy_test.go b/pkg/util/kube/secrets_copy_test.go new file mode 100644 index 000000000..ea294eb4e --- /dev/null +++ b/pkg/util/kube/secrets_copy_test.go @@ -0,0 +1,362 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kube + +import ( + "context" + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1api "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" +) + +const testCopyLabel = "velero.io/backup-pvc-secret" + +func TestCopySecret(t *testing.T) { + log := logrus.New() + + tests := []struct { + name string + secretName string + sourceNS string + targetNS string + ownerName string + objects []k8sruntime.Object + expectErr bool + errContains string + }{ + { + name: "successfully copies secret to target namespace", + secretName: "ceph-csi-kms-token", + sourceNS: "app-ns", + targetNS: "velero", + ownerName: "du-123", + objects: []k8sruntime.Object{ + &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "ceph-csi-kms-token", Namespace: "app-ns"}, + Data: map[string][]byte{"token": []byte("vault-token-a")}, + Type: corev1api.SecretTypeOpaque, + }, + }, + }, + { + name: "returns error when source secret does not exist", + secretName: "missing-secret", + sourceNS: "app-ns", + targetNS: "velero", + ownerName: "du-123", + objects: []k8sruntime.Object{}, + expectErr: true, + errContains: "error getting secret", + }, + { + name: "no-op when target already has secret with same data and same owner", + secretName: "ceph-csi-kms-token", + sourceNS: "app-ns", + targetNS: "velero", + ownerName: "du-123", + objects: []k8sruntime.Object{ + &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "ceph-csi-kms-token", Namespace: "app-ns"}, + Data: map[string][]byte{"token": []byte("same-token")}, + Type: corev1api.SecretTypeOpaque, + }, + &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ceph-csi-kms-token", Namespace: "velero", + Labels: map[string]string{testCopyLabel: "du-123"}, + }, + Data: map[string][]byte{"token": []byte("same-token")}, + Type: corev1api.SecretTypeOpaque, + }, + }, + }, + { + name: "returns collision when same data but different owner", + secretName: "ceph-csi-kms-token", + sourceNS: "app-ns", + targetNS: "velero", + ownerName: "du-456", + objects: []k8sruntime.Object{ + &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "ceph-csi-kms-token", Namespace: "app-ns"}, + Data: map[string][]byte{"token": []byte("same-token")}, + Type: corev1api.SecretTypeOpaque, + }, + &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ceph-csi-kms-token", Namespace: "velero", + Labels: map[string]string{testCopyLabel: "du-123"}, + }, + Data: map[string][]byte{"token": []byte("same-token")}, + Type: corev1api.SecretTypeOpaque, + }, + }, + expectErr: true, + errContains: "collision", + }, + { + name: "returns collision error when target has secret with different data", + secretName: "ceph-csi-kms-token", + sourceNS: "app-ns", + targetNS: "velero", + ownerName: "du-123", + objects: []k8sruntime.Object{ + &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "ceph-csi-kms-token", Namespace: "app-ns"}, + Data: map[string][]byte{"token": []byte("token-a")}, + Type: corev1api.SecretTypeOpaque, + }, + &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "ceph-csi-kms-token", Namespace: "velero"}, + Data: map[string][]byte{"token": []byte("token-b")}, + Type: corev1api.SecretTypeOpaque, + }, + }, + expectErr: true, + errContains: "secret collision", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fakeClient := fake.NewSimpleClientset(tt.objects...) + + err := CopySecret(context.Background(), fakeClient.CoreV1(), + tt.secretName, tt.sourceNS, tt.targetNS, + map[string]string{testCopyLabel: tt.ownerName}, log) + + if tt.expectErr { + require.Error(t, err) + if tt.errContains != "" { + assert.Contains(t, err.Error(), tt.errContains) + } + return + } + + require.NoError(t, err) + + copied, getErr := fakeClient.CoreV1().Secrets(tt.targetNS).Get( + context.Background(), tt.secretName, metav1.GetOptions{}) + require.NoError(t, getErr) + assert.NotNil(t, copied) + }) + } +} + +func TestDeleteSecretsWithLabel(t *testing.T) { + log := logrus.New() + + fakeClient := fake.NewSimpleClientset( + &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "secret-1", Namespace: "velero", + Labels: map[string]string{testCopyLabel: "du-123"}, + }, + }, + &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "secret-2", Namespace: "velero", + Labels: map[string]string{testCopyLabel: "du-456"}, + }, + }, + ) + + DeleteSecretsWithLabel(context.Background(), fakeClient.CoreV1(), "velero", + testCopyLabel, "du-123", log) + + _, err := fakeClient.CoreV1().Secrets("velero").Get( + context.Background(), "secret-1", metav1.GetOptions{}) + require.Error(t, err, "secret-1 should be deleted") + + _, err = fakeClient.CoreV1().Secrets("velero").Get( + context.Background(), "secret-2", metav1.GetOptions{}) + assert.NoError(t, err, "secret-2 should still exist") +} + +func TestCopyConfigMap(t *testing.T) { + log := logrus.New() + + tests := []struct { + name string + cmName string + sourceNS string + targetNS string + ownerName string + objects []k8sruntime.Object + expectErr bool + errContains string + }{ + { + name: "successfully copies configmap to target namespace", + cmName: "ceph-csi-kms-config", + sourceNS: "app-ns", + targetNS: "velero", + ownerName: "du-123", + objects: []k8sruntime.Object{ + &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "ceph-csi-kms-config", Namespace: "app-ns"}, + Data: map[string]string{"vaultAddress": "https://vault.example.com"}, + }, + }, + }, + { + name: "returns error when source configmap does not exist", + cmName: "missing-cm", + sourceNS: "app-ns", + targetNS: "velero", + ownerName: "du-123", + objects: []k8sruntime.Object{}, + expectErr: true, + errContains: "error getting configmap", + }, + { + name: "no-op when target already has configmap with same data and same owner", + cmName: "ceph-csi-kms-config", + sourceNS: "app-ns", + targetNS: "velero", + ownerName: "du-123", + objects: []k8sruntime.Object{ + &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "ceph-csi-kms-config", Namespace: "app-ns"}, + Data: map[string]string{"vaultAddress": "https://vault.example.com"}, + }, + &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ceph-csi-kms-config", Namespace: "velero", + Labels: map[string]string{testCopyLabel: "du-123"}, + }, + Data: map[string]string{"vaultAddress": "https://vault.example.com"}, + }, + }, + }, + { + name: "returns collision when same data but different owner", + cmName: "ceph-csi-kms-config", + sourceNS: "app-ns", + targetNS: "velero", + ownerName: "du-456", + objects: []k8sruntime.Object{ + &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "ceph-csi-kms-config", Namespace: "app-ns"}, + Data: map[string]string{"vaultAddress": "https://vault.example.com"}, + }, + &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ceph-csi-kms-config", Namespace: "velero", + Labels: map[string]string{testCopyLabel: "du-123"}, + }, + Data: map[string]string{"vaultAddress": "https://vault.example.com"}, + }, + }, + expectErr: true, + errContains: "collision", + }, + { + name: "copies configmap with BinaryData", + cmName: "ceph-csi-kms-config", + sourceNS: "app-ns", + targetNS: "velero", + ownerName: "du-123", + objects: []k8sruntime.Object{ + &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "ceph-csi-kms-config", Namespace: "app-ns"}, + BinaryData: map[string][]byte{"ca.crt": []byte("binary-ca-bundle")}, + }, + }, + }, + { + name: "returns collision error when target has configmap with different data", + cmName: "ceph-csi-kms-config", + sourceNS: "app-ns", + targetNS: "velero", + ownerName: "du-123", + objects: []k8sruntime.Object{ + &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "ceph-csi-kms-config", Namespace: "app-ns"}, + Data: map[string]string{"vaultAddress": "https://vault-a.example.com"}, + }, + &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "ceph-csi-kms-config", Namespace: "velero"}, + Data: map[string]string{"vaultAddress": "https://vault-b.example.com"}, + }, + }, + expectErr: true, + errContains: "secret collision", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fakeClient := fake.NewSimpleClientset(tt.objects...) + + err := CopyConfigMap(context.Background(), fakeClient.CoreV1(), + tt.cmName, tt.sourceNS, tt.targetNS, + map[string]string{testCopyLabel: tt.ownerName}, log) + + if tt.expectErr { + require.Error(t, err) + if tt.errContains != "" { + assert.Contains(t, err.Error(), tt.errContains) + } + return + } + + require.NoError(t, err) + + copied, getErr := fakeClient.CoreV1().ConfigMaps(tt.targetNS).Get( + context.Background(), tt.cmName, metav1.GetOptions{}) + require.NoError(t, getErr) + assert.NotNil(t, copied) + }) + } +} + +func TestDeleteConfigMapsWithLabel(t *testing.T) { + log := logrus.New() + + fakeClient := fake.NewSimpleClientset( + &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cm-1", Namespace: "velero", + Labels: map[string]string{testCopyLabel: "du-123"}, + }, + }, + &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cm-2", Namespace: "velero", + Labels: map[string]string{testCopyLabel: "du-456"}, + }, + }, + ) + + DeleteConfigMapsWithLabel(context.Background(), fakeClient.CoreV1(), "velero", + testCopyLabel, "du-123", log) + + _, err := fakeClient.CoreV1().ConfigMaps("velero").Get( + context.Background(), "cm-1", metav1.GetOptions{}) + require.Error(t, err, "cm-1 should be deleted") + + _, err = fakeClient.CoreV1().ConfigMaps("velero").Get( + context.Background(), "cm-2", metav1.GetOptions{}) + assert.NoError(t, err, "cm-2 should still exist") +} diff --git a/site/content/docs/main/customize-installation.md b/site/content/docs/main/customize-installation.md index 194d947eb..4b4a11f30 100644 --- a/site/content/docs/main/customize-installation.md +++ b/site/content/docs/main/customize-installation.md @@ -348,6 +348,21 @@ By default, only one backup is processed in the `InProgress` phase at a time. Th Enabling parallel backups can provide a significant performance benefit for backups which contain a large number of Kubernetes resources or ones which contain a large number of smaller volumes. Backups dominated by large volumes will not see as much benefit, since the majority of time for those backups is spent waiting for the async phase to complete. A larger `concurrent-backups` configuration may require additional memory and CPU resources for the velero container. +## Limiting Resource Backup Data Cache Size +For Kubernetes resource data (non volume data), for some operations like Restores or Backup Deletions, etc., Velero uses local cache (in the root file system of the cluster node) to download and extract the data from the backup storage location, Velero sets a limit for the cache size. If the cache size exceeds the limit, the specific operation would fail. +By default Velero sets the limit as 16GB, if your backup data is large, you can change the Velero server parameter `max-backup-extraction-size`. Here is an example to set the limit to 32GB: + +```yaml +containers: + - name: velero + image: velero/velero:latest + command: + - /velero + args: + - server + - --max-backup-extraction-size=32768 +``` + ## Additional options Run `velero install --help` or see the [Helm chart documentation](https://vmware-tanzu.github.io/helm-charts/) for the full set of installation options. diff --git a/site/content/docs/main/data-movement-backup-pvc-configuration.md b/site/content/docs/main/data-movement-backup-pvc-configuration.md index 9d48b0a5f..afcd7e50d 100644 --- a/site/content/docs/main/data-movement-backup-pvc-configuration.md +++ b/site/content/docs/main/data-movement-backup-pvc-configuration.md @@ -34,12 +34,28 @@ default the source PVC's storage class will be used. the SELinux point of view, this will be considered a "Super Privileged Container" which means that selinux enforcement will be disabled and volume relabeling will not occur. This field is ignored if `readOnly` is `false`. +- `readWriteOncePod`: This is a boolean value. If set to `true`, then `ReadWriteOncePod` will be the only value set to the backupPVC's access modes. On + SELinux-enabled clusters the kubelet applies the SELinux label to a `ReadWriteOncePod` volume at mount time (`-o context=`) instead of recursively + relabeling every file on the volume, which can take hours on volumes with a high file count. It requires a CSI driver that advertises SELinux mount + support (`CSIDriver.spec.seLinuxMount: true`) and a storage class that supports creating `ReadWriteOncePod` PVCs from a snapshot. This field is ignored + if `readOnly` is `true`. + The users can specify the ConfigMap name during velero installation by CLI: `velero install --node-agent-configmap=` - `annotations`: permits to set annotations on the backupPVC itself. typically useful for some CSI provider which cannot mount a VolumeSnapshot without a custom annotation. +- `secretNames`: a list of secret names to copy from the source PVC's namespace to the Velero namespace before the backupPVC is + created, and delete after the DataUpload completes. This is needed for CSI drivers that require namespace-scoped secrets to + provision the volume, for example ODF/ceph-csi encrypted volumes that fetch a KMS token secret (`ceph-csi-kms-token`) from the + PVC's namespace. Without this, the backupPVC created in the Velero namespace fails to provision because the secret only exists + in the source namespace. + +- `configMapNames`: a list of configmap names to copy from the source PVC's namespace to the Velero namespace before the backupPVC + is created, and delete after the DataUpload completes. This is needed for CSI drivers that require namespace-scoped configmaps to + provision the volume, for example a tenant-specific ceph-csi KMS connection override configmap (`ceph-csi-kms-config`). + A sample of `backupPVC` config as part of the ConfigMap would look like: ```json { @@ -60,11 +76,24 @@ A sample of `backupPVC` config as part of the ConfigMap would look like: "storage-class-4": { "readOnly": true, "spcNoRelabeling": true + }, + "ocs-storagecluster-ceph-rbd-encrypted": { + "secretNames": ["ceph-csi-kms-token"], + "configMapNames": ["ceph-csi-kms-config"] + }, + "storage-class-5": { + "readWriteOncePod": true } } } ``` +**Note on encrypted volumes:** the copied secrets/configmaps are labeled `velero.io/backup-pvc-secret=` and +deleted when the DataUpload completes (or on failure). If concurrent DataUploads from different namespaces need a secret with the same +name but different content in the Velero namespace, they conflict. For ceph-csi, +this can be avoided by configuring a unique +[`tenantTokenName` per tenant](https://github.com/ceph/ceph-csi/blob/devel/docs/design/proposals/encryption-with-vault-tokens.md#example-of-the-kms-configuration-file-for-vault-tokens). + **Note:** - Users should make sure that the storage class specified in `backupPVC` config should exist in the cluster and can be used by the `backupPVC`, otherwise the corresponding DataUpload CR will stay in `Accepted` phase until timeout (data movement prepare timeout value is 30m by default). @@ -73,6 +102,10 @@ A sample of `backupPVC` config as part of the ConfigMap would look like: timeout (data movement prepare timeout value is 30m by default). - In an SELinux-enabled cluster, any time users set `readOnly=true` they must also set `spcNoRelabeling=true`. There is no need to set `spcNoRelabeling=true` if the volume is not readOnly. +- `readWriteOncePod` and `readOnly` are mutually exclusive. If both are set to `true`, `readOnly` wins, `readWriteOncePod` is ignored and a warning is logged. +- `readWriteOncePod` is an alternative to `readOnly`+`spcNoRelabeling` for SELinux-enabled clusters whose storage does not support `ReadOnlyMany` +(for example Ceph RBD in Filesystem mode or LVM). Users must make sure the storage class used for `backupPVC` supports creating a `ReadWriteOncePod` PVC from +a snapshot, otherwise the corresponding DataUpload CR will stay in `Accepted` phase until timeout. - If any of the above problems occur, then the DataUpload CR is `canceled` after timeout, and the backupPod and backupPVC will be deleted, and the backup will be marked as `PartiallyFailed`. diff --git a/site/content/docs/main/data-movement-restore-pvc-configuration.md b/site/content/docs/main/data-movement-restore-pvc-configuration.md index 1cb8fa14d..1728d346e 100644 --- a/site/content/docs/main/data-movement-restore-pvc-configuration.md +++ b/site/content/docs/main/data-movement-restore-pvc-configuration.md @@ -12,6 +12,10 @@ Velero introduces a new section in the node agent configuration ConfigMap (the n - `ignoreDelayBinding`: If this flag is set, the data movement restore will ignore the delay binding requirements from `WaitForFirstConsumer` mode, create the restore pod and provision the volume associated to an arbitrary node. When multiple volume restores happen in parallel, the restore pods will be spread evenly to all the nodes. +- `secretNames`: a list of secret names to copy from the target (restore) namespace to the Velero namespace before the restorePVC is created, and delete after the DataDownload completes. This is needed for CSI drivers that require namespace-scoped secrets to provision the volume, for example ODF/ceph-csi encrypted volumes that fetch a KMS token secret (`ceph-csi-kms-token`) from the PVC's namespace. Without this, the restorePVC created in the Velero namespace fails to provision because the secret only exists in the target namespace. + +- `configMapNames`: a list of configmap names to copy from the target (restore) namespace to the Velero namespace before the restorePVC is created, and delete after the DataDownload completes. This is needed for CSI drivers that require namespace-scoped configmaps to provision the volume, for example a tenant-specific ceph-csi KMS connection override configmap (`ceph-csi-kms-config`). + The users can specify the ConfigMap name during velero installation by CLI: `velero install --node-agent-configmap=` @@ -20,11 +24,15 @@ A sample of `restorePVC` config as part of the ConfigMap would look like: ```json { "restorePVC": { - "ignoreDelayBinding": true + "ignoreDelayBinding": true, + "secretNames": ["ceph-csi-kms-token"], + "configMapNames": ["ceph-csi-kms-config"] } } ``` +**Note on encrypted volumes:** unlike `backupPVC` (which is keyed per source storage class), `restorePVC` is a single config that applies to all restore PVCs. The copied secrets/configmaps are labeled `velero.io/backup-pvc-secret=` and deleted when the DataDownload completes (or on failure). + **Note:** - If `ignoreDelayBinding` is set, the restored volume is provisioned in the storage areas associated to an arbitrary node, if the restored pod cannot be scheduled to that node, e.g., because of topology constraints, the data mover restore still completes, but the workload is not usable since the restored pod cannot mount the restored volume - At present, node selection is not supported for data mover restore, so the restored volume may be attached to any node in the cluster; once node selection is supported and enabled, the restored volume will be attached to one of the selected nodes only. In this way, node selection and `ignoreDelayBinding` can work together even though the environment is with topology constraints diff --git a/site/content/docs/main/supported-configmaps/node-agent-configmap.md b/site/content/docs/main/supported-configmaps/node-agent-configmap.md index fc90d4436..f8f8f2c5c 100644 --- a/site/content/docs/main/supported-configmaps/node-agent-configmap.md +++ b/site/content/docs/main/supported-configmaps/node-agent-configmap.md @@ -297,6 +297,9 @@ For detailed information, see [BackupPVC Configuration for Data Movement Backup] - **`storageClass`**: Alternative storage class for backup PVCs (defaults to source PVC's storage class) - **`readOnly`**: This is a boolean value. If set to `true` then `ReadOnlyMany` will be the only value set to the backupPVC's access modes. Otherwise `ReadWriteOnce` value will be used. - **`spcNoRelabeling`**: This is a boolean value. If set to true, then `pod.Spec.SecurityContext.SELinuxOptions.Type` will be set to `spc_t`. From the SELinux point of view, this will be considered a `Super Privileged Container` which means that selinux enforcement will be disabled and volume relabeling will not occur. This field is ignored if `readOnly` is `false`. +- **`secretNames`**: List of secret names to copy from the source PVC's namespace to the Velero namespace before creating the backupPVC (deleted after the DataUpload completes). Needed for CSI drivers that require namespace-scoped secrets to provision the volume, e.g. ODF/ceph-csi encrypted volumes (`ceph-csi-kms-token`). +- **`configMapNames`**: List of configmap names to copy from the source PVC's namespace to the Velero namespace before creating the backupPVC (deleted after the DataUpload completes). Needed for CSI drivers that require namespace-scoped configmaps to provision the volume, e.g. a tenant ceph-csi KMS config (`ceph-csi-kms-config`). +- **`readWriteOncePod`**: This is a boolean value. If set to `true`, then `ReadWriteOncePod` will be the only value set to the backupPVC's access modes, so the kubelet labels the volume at mount time instead of relabeling every file. Requires a CSI driver with `seLinuxMount: true` and a storage class that supports `ReadWriteOncePod` PVCs from a snapshot. This field is ignored if `readOnly` is `true`. **Use Cases:** - Use read-only volumes for faster snapshot-to-volume conversion @@ -307,6 +310,7 @@ For detailed information, see [BackupPVC Configuration for Data Movement Backup] **Important Notes:** - Ensure specified storage classes exist and support required access modes - In SELinux environments, always set `spcNoRelabeling: true` when using `readOnly: true` +- In SELinux environments where the storage does not support `ReadOnlyMany`, use `readWriteOncePod: true` instead; it is ignored when `readOnly: true` is also set - Failures result in DataUpload CR staying in `Accepted` phase until timeout (30m default) #### Storage Class Mapping @@ -360,6 +364,8 @@ For detailed information, see [RestorePVC Configuration for Data Movement Restor #### Configuration Options - **`ignoreDelayBinding`**: Ignore `WaitForFirstConsumer` binding mode constraints +- **`secretNames`**: List of secret names to copy from the target (restore) namespace to the Velero namespace before creating the restorePVC (deleted after the DataDownload completes). Needed for CSI drivers that require namespace-scoped secrets to provision the volume, e.g. ODF/ceph-csi encrypted volumes (`ceph-csi-kms-token`). +- **`configMapNames`**: List of configmap names to copy from the target (restore) namespace to the Velero namespace before creating the restorePVC (deleted after the DataDownload completes). Needed for CSI drivers that require namespace-scoped configmaps to provision the volume, e.g. a tenant ceph-csi KMS config (`ceph-csi-kms-config`). **Use Cases:** - Improve restore parallelism by not waiting for pod scheduling diff --git a/site/content/docs/v1.18/data-movement-backup-pvc-configuration.md b/site/content/docs/v1.18/data-movement-backup-pvc-configuration.md index 9d48b0a5f..956d03997 100644 --- a/site/content/docs/v1.18/data-movement-backup-pvc-configuration.md +++ b/site/content/docs/v1.18/data-movement-backup-pvc-configuration.md @@ -40,6 +40,16 @@ The users can specify the ConfigMap name during velero installation by CLI: - `annotations`: permits to set annotations on the backupPVC itself. typically useful for some CSI provider which cannot mount a VolumeSnapshot without a custom annotation. +- `secretNames`: a list of secret names to copy from the source PVC's namespace to the Velero namespace before the backupPVC is + created, and delete after the DataUpload completes. This is needed for CSI drivers that require namespace-scoped secrets to + provision the volume, for example ODF/ceph-csi encrypted volumes that fetch a KMS token secret (`ceph-csi-kms-token`) from the + PVC's namespace. Without this, the backupPVC created in the Velero namespace fails to provision because the secret only exists + in the source namespace. + +- `configMapNames`: a list of configmap names to copy from the source PVC's namespace to the Velero namespace before the backupPVC + is created, and delete after the DataUpload completes. This is needed for CSI drivers that require namespace-scoped configmaps to + provision the volume, for example a tenant-specific ceph-csi KMS connection override configmap (`ceph-csi-kms-config`). + A sample of `backupPVC` config as part of the ConfigMap would look like: ```json { @@ -60,11 +70,21 @@ A sample of `backupPVC` config as part of the ConfigMap would look like: "storage-class-4": { "readOnly": true, "spcNoRelabeling": true + }, + "ocs-storagecluster-ceph-rbd-encrypted": { + "secretNames": ["ceph-csi-kms-token"], + "configMapNames": ["ceph-csi-kms-config"] } } } ``` +**Note on encrypted volumes:** the copied secrets/configmaps are labeled `velero.io/backup-pvc-secret=` and +deleted when the DataUpload completes (or on failure). If concurrent DataUploads from different namespaces need a secret with the same +name but different content in the Velero namespace, they conflict. For ceph-csi, +this can be avoided by configuring a unique +[`tenantTokenName` per tenant](https://github.com/ceph/ceph-csi/blob/devel/docs/design/proposals/encryption-with-vault-tokens.md#example-of-the-kms-configuration-file-for-vault-tokens). + **Note:** - Users should make sure that the storage class specified in `backupPVC` config should exist in the cluster and can be used by the `backupPVC`, otherwise the corresponding DataUpload CR will stay in `Accepted` phase until timeout (data movement prepare timeout value is 30m by default). diff --git a/site/content/docs/v1.18/data-movement-restore-pvc-configuration.md b/site/content/docs/v1.18/data-movement-restore-pvc-configuration.md index 1cb8fa14d..1728d346e 100644 --- a/site/content/docs/v1.18/data-movement-restore-pvc-configuration.md +++ b/site/content/docs/v1.18/data-movement-restore-pvc-configuration.md @@ -12,6 +12,10 @@ Velero introduces a new section in the node agent configuration ConfigMap (the n - `ignoreDelayBinding`: If this flag is set, the data movement restore will ignore the delay binding requirements from `WaitForFirstConsumer` mode, create the restore pod and provision the volume associated to an arbitrary node. When multiple volume restores happen in parallel, the restore pods will be spread evenly to all the nodes. +- `secretNames`: a list of secret names to copy from the target (restore) namespace to the Velero namespace before the restorePVC is created, and delete after the DataDownload completes. This is needed for CSI drivers that require namespace-scoped secrets to provision the volume, for example ODF/ceph-csi encrypted volumes that fetch a KMS token secret (`ceph-csi-kms-token`) from the PVC's namespace. Without this, the restorePVC created in the Velero namespace fails to provision because the secret only exists in the target namespace. + +- `configMapNames`: a list of configmap names to copy from the target (restore) namespace to the Velero namespace before the restorePVC is created, and delete after the DataDownload completes. This is needed for CSI drivers that require namespace-scoped configmaps to provision the volume, for example a tenant-specific ceph-csi KMS connection override configmap (`ceph-csi-kms-config`). + The users can specify the ConfigMap name during velero installation by CLI: `velero install --node-agent-configmap=` @@ -20,11 +24,15 @@ A sample of `restorePVC` config as part of the ConfigMap would look like: ```json { "restorePVC": { - "ignoreDelayBinding": true + "ignoreDelayBinding": true, + "secretNames": ["ceph-csi-kms-token"], + "configMapNames": ["ceph-csi-kms-config"] } } ``` +**Note on encrypted volumes:** unlike `backupPVC` (which is keyed per source storage class), `restorePVC` is a single config that applies to all restore PVCs. The copied secrets/configmaps are labeled `velero.io/backup-pvc-secret=` and deleted when the DataDownload completes (or on failure). + **Note:** - If `ignoreDelayBinding` is set, the restored volume is provisioned in the storage areas associated to an arbitrary node, if the restored pod cannot be scheduled to that node, e.g., because of topology constraints, the data mover restore still completes, but the workload is not usable since the restored pod cannot mount the restored volume - At present, node selection is not supported for data mover restore, so the restored volume may be attached to any node in the cluster; once node selection is supported and enabled, the restored volume will be attached to one of the selected nodes only. In this way, node selection and `ignoreDelayBinding` can work together even though the environment is with topology constraints diff --git a/site/content/docs/v1.18/supported-configmaps/node-agent-configmap.md b/site/content/docs/v1.18/supported-configmaps/node-agent-configmap.md index 0062b0391..4ffea44cb 100644 --- a/site/content/docs/v1.18/supported-configmaps/node-agent-configmap.md +++ b/site/content/docs/v1.18/supported-configmaps/node-agent-configmap.md @@ -295,6 +295,8 @@ For detailed information, see [BackupPVC Configuration for Data Movement Backup] - **`storageClass`**: Alternative storage class for backup PVCs (defaults to source PVC's storage class) - **`readOnly`**: This is a boolean value. If set to `true` then `ReadOnlyMany` will be the only value set to the backupPVC's access modes. Otherwise `ReadWriteOnce` value will be used. - **`spcNoRelabeling`**: This is a boolean value. If set to true, then `pod.Spec.SecurityContext.SELinuxOptions.Type` will be set to `spc_t`. From the SELinux point of view, this will be considered a `Super Privileged Container` which means that selinux enforcement will be disabled and volume relabeling will not occur. This field is ignored if `readOnly` is `false`. +- **`secretNames`**: List of secret names to copy from the source PVC's namespace to the Velero namespace before creating the backupPVC (deleted after the DataUpload completes). Needed for CSI drivers that require namespace-scoped secrets to provision the volume, e.g. ODF/ceph-csi encrypted volumes (`ceph-csi-kms-token`). +- **`configMapNames`**: List of configmap names to copy from the source PVC's namespace to the Velero namespace before creating the backupPVC (deleted after the DataUpload completes). Needed for CSI drivers that require namespace-scoped configmaps to provision the volume, e.g. a tenant ceph-csi KMS config (`ceph-csi-kms-config`). **Use Cases:** - Use read-only volumes for faster snapshot-to-volume conversion @@ -358,6 +360,8 @@ For detailed information, see [RestorePVC Configuration for Data Movement Restor #### Configuration Options - **`ignoreDelayBinding`**: Ignore `WaitForFirstConsumer` binding mode constraints +- **`secretNames`**: List of secret names to copy from the target (restore) namespace to the Velero namespace before creating the restorePVC (deleted after the DataDownload completes). Needed for CSI drivers that require namespace-scoped secrets to provision the volume, e.g. ODF/ceph-csi encrypted volumes (`ceph-csi-kms-token`). +- **`configMapNames`**: List of configmap names to copy from the target (restore) namespace to the Velero namespace before creating the restorePVC (deleted after the DataDownload completes). Needed for CSI drivers that require namespace-scoped configmaps to provision the volume, e.g. a tenant ceph-csi KMS config (`ceph-csi-kms-config`). **Use Cases:** - Improve restore parallelism by not waiting for pod scheduling diff --git a/test/e2e/README.md b/test/e2e/README.md index a621ee70c..2120e8b55 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -28,7 +28,7 @@ These are the current set of limitations with the E2E tests. 1. Flag `-install-velero` is for purpose of having tests on an existed Velero instance, but by default `-install-velero` is set to true, because it's mandatory for some of cases to testing on specific version of Velero, such as upgrade and migration tests. In upgrade tests, we must install a specific old version and then upgrade it to the target version, multiple installations is involved here, also migration tests have the same situation with upgrade tests, therefore if you're going to test against an existed Velero instance, make sure to skip upgrade and migration tests from a single E2E test execution. 1. To improve E2E test execution efficiency, E2E tests will skip re-installation between test cases except for those which need a fresh Velero installation like upgrade , migration and some other test cases. When starting a E2E test execution which setting flag `-install-velero` with the default value(true), there will be a Velero installation at the beginning, then test cases will be run in random order, and test cases behavior is as below: 1. If the scheduled test case is upgrade (or other cases needs a fresh Velero installation), then upgrade test will uninstall the current Velero instance at the beginning and uninstall the tested Velero instance in the end to avoid unexpected installation parameters for the following test cases. - 1. If the scheduled test case is the normal one, it will check the existence of Velero instance, if no one there then start a new standard instaillation, otherwise proceeding test steps. + 1. If the scheduled test case is the normal one, it will check the existence of Velero instance, if no one there then start a new standard installation, otherwise proceeding test steps. ## 3. Configuration for E2E tests @@ -113,10 +113,10 @@ Below is a mapping between `make` variables to E2E configuration flags. 1. `MIGRATE_FROM_VELERO_VERSION `: `-migrate-from-velero-version`. Optional. 1. `ADDITIONAL_BSL_PLUGINS `: `-additional-bsl-plugins`. Optional. 1. `ADDITIONAL_OBJECT_STORE_PROVIDER`: `-additional-bsl-object-store-provider`. Optional. -1. `ADDITIONAL_CREDS_FILE`: `-additional-bsl-bucket`. Optional. -1. `ADDITIONAL_BSL_BUCKET`: `-additional-bsl-prefix`. Optional. -1. `ADDITIONAL_BSL_PREFIX`: `-additional-bsl-config`. Optional. -1. `ADDITIONAL_BSL_CONFIG`: `-additional-bsl-credentials-file`. Optional. +1. `ADDITIONAL_CREDS_FILE`: `-additional-bsl-credentials-file`. Optional. +1. `ADDITIONAL_BSL_BUCKET`: `-additional-bsl-bucket`. Optional. +1. `ADDITIONAL_BSL_PREFIX`: `-additional-bsl-prefix`. Optional. +1. `ADDITIONAL_BSL_CONFIG`: `-additional-bsl-config`. Optional. 1. `FEATURES`: `-features`. Optional. 1. `REGISTRY_CREDENTIAL_FILE`: `-registry-credential-file`. Optional. 1. `KIBISHII_DIRECTORY`: `-kibishii-directory`. Optional. @@ -127,14 +127,14 @@ Below is a mapping between `make` variables to E2E configuration flags. 1. `SNAPSHOT_MOVE_DATA`: `-snapshot-move-data`. Optional. 1. `DATA_MOVER_plugin`: `-data-mover-plugin`. Optional. 1. `STANDBY_CLUSTER_CLOUD_PROVIDER`: `-standby-cluster-cloud-provider`. Optional. -1. `STANDBY_CLUSTER_PLUGINS`: `-dstandby-cluster-plugins`. Optional. +1. `STANDBY_CLUSTER_PLUGINS`: `-standby-cluster-plugins`. Optional. 1. `STANDBY_CLUSTER_OBJECT_STORE_PROVIDER`: `-standby-cluster-object-store-provider`. Optional. 1. `INSTALL_VELERO `: `-install-velero`. Optional. 1. `DEBUG_VELERO_POD_RESTART`: `-debug-velero-pod-restart`. Optional. 1. `FAIL_FAST`: `--fail-fast`. Optional. 1. `HAS_VSPHERE_PLUGIN`: `--has-vsphere-plugin`. Optional. 1. `WORKER_OS`: `--worker-os`. Optional. -1. `IMAGE_REGISTRY_PROXY`: `--image-registry-proxy.` Optional. +1. `IMAGE_REGISTRY_PROXY`: `--image-registry-proxy` Optional. ### Examples @@ -270,7 +270,7 @@ OBJECT_STORE_PROVIDER=aws \ CREDS_FILE= \ BSL_CONFIG=region= \ BSL_BUCKET= \ -BSL_PREFIX= \ +BSL_PREFIX= \ VSL_CONFIG=region= \ SNAPSHOT_MOVE_DATA=true \ STANDBY_CLUSTER_CLOUD_PROVIDER=aws \ @@ -369,7 +369,7 @@ there're some tests need to be run in a single execution or pipeline with specif Following pipelines should cover all E2E tests along with proper filters: 1. **CSI pipeline:** As we can see lots of labels in E2E test code, there're many snapshot-labeled test scripts. To cover CSI scenario, a pipeline with CSI enabled should be a good choice, otherwise, we will double all the snapshot cases for CSI scenario, it's very time-wasting. By providing `FEATURES=EnableCSI` and `PLUGINS=`, a CSI pipeline is ready for testing. -1. **Data mover pipeline:** Data mover scenario is the same scenario with migaration test except the restriction of migaration between different providers, so it better to separated it out from other pipelines. Please refer the example in previous. +1. **Data mover pipeline:** Data mover scenario is the same scenario with migration test except the restriction of migration between different providers, so it better to separated it out from other pipelines. Please refer the example in previous. 1. **File system backup pipeline:** Set `UPLOADER_TYPE` to `kopia` for all file system backup test cases; 1. **Long time pipeline:** Long time cases should be group into one pipeline, currently these test cases with labels `Scale`, `Schedule` or `TTL` can be group into a pipeline, and make sure to skip them off in any other pipelines. @@ -381,7 +381,7 @@ Following pipelines should cover all E2E tests along with proper filters: When adding a test, aim to instantiate an API client only once at the beginning of the test. There is a constructor `newTestClient` that facilitates the configuration and instantiation of clients. Also, please use the `kubebuilder` runtime controller client for any new test, as we will phase out usage of `client-go` API clients. ## 8. TestCase frame related -TestCase frame provide a serials of interface to concatenate one complete e2e test. it's makes the testing be concise and explicit. +TestCase frame provide a series of interfaces to concatenate one complete e2e test. it makes the testing be concise and explicit. ### VeleroBackupRestoreTest interface VeleroBackupRestoreTest interface provided a standard workflow of backup and restore, which makes the whole testing process clearer and code reusability. diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index a8bbbce4c..57121c378 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -392,7 +392,7 @@ var _ = Describe( APIGroupVersionsTest, ) var _ = Describe( - "CRD of apiextentions v1beta1 should be B/R successfully from cluster(k8s version < 1.22) to cluster(k8s version >= 1.22)", + "CRD of apiextensions v1beta1 should be B/R successfully from cluster(k8s version < 1.22) to cluster(k8s version >= 1.22)", Label("APIGroup", "APIExtensions", "SKIP_KIND"), APIExtensionsVersionsTest, ) diff --git a/test/e2e/nodeagentconfig/node-agent-config.go b/test/e2e/nodeagentconfig/node-agent-config.go index 3eb508234..cab65dfb5 100644 --- a/test/e2e/nodeagentconfig/node-agent-config.go +++ b/test/e2e/nodeagentconfig/node-agent-config.go @@ -62,7 +62,7 @@ var LoadAffinities func() = TestFunc(&NodeAgentConfigTestCase{ { NodeSelector: metav1.LabelSelector{ MatchLabels: map[string]string{ - "kubernetes.io/arch": "amd64", + corev1api.LabelArchStable: "amd64", }, }, StorageClass: test.StorageClassName2, diff --git a/test/testdata/volume-snapshot-class/kind.yaml b/test/testdata/volume-snapshot-class/kind.yaml new file mode 100644 index 000000000..35aa60ad4 --- /dev/null +++ b/test/testdata/volume-snapshot-class/kind.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: snapshot.storage.k8s.io/v1 +deletionPolicy: Delete +driver: hostpath.csi.k8s.io +kind: VolumeSnapshotClass +metadata: + labels: + velero.io/csi-volumesnapshot-class: "true" + name: e2e-volume-snapshot-class diff --git a/test/util/k8s/deployment.go b/test/util/k8s/deployment.go index 42e2d6ac7..01209aeb7 100644 --- a/test/util/k8s/deployment.go +++ b/test/util/k8s/deployment.go @@ -102,7 +102,7 @@ func NewDeployment( { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Values: []string{common.WorkerOSWindows}, Operator: corev1api.NodeSelectorOpIn, }, diff --git a/test/util/k8s/pod.go b/test/util/k8s/pod.go index 718beab98..ce8580eaa 100644 --- a/test/util/k8s/pod.go +++ b/test/util/k8s/pod.go @@ -84,7 +84,7 @@ func CreatePod( { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Values: []string{common.WorkerOSWindows}, Operator: corev1api.NodeSelectorOpIn, }, diff --git a/test/util/k8s/pvc.go b/test/util/k8s/pvc.go index 7d9610141..36de6fa4c 100644 --- a/test/util/k8s/pvc.go +++ b/test/util/k8s/pvc.go @@ -68,19 +68,19 @@ func (p *PVCBuilder) WithResourceStorage(q resource.Quantity) *PVCBuilder { } func CreatePVC(client TestClient, ns, name, sc string, ann map[string]string) (*corev1api.PersistentVolumeClaim, error) { - pvcBulder := NewPVC(ns, name) + pvcBuilder := NewPVC(ns, name) if ann != nil { - pvcBulder.WithAnnotation(ann) + pvcBuilder.WithAnnotation(ann) } if sc != "" { - pvcBulder.WithStorageClass(sc) + pvcBuilder.WithStorageClass(sc) } - return client.ClientGo.CoreV1().PersistentVolumeClaims(ns).Create(context.TODO(), pvcBulder.Result(), metav1.CreateOptions{}) + return client.ClientGo.CoreV1().PersistentVolumeClaims(ns).Create(context.TODO(), pvcBuilder.Result(), metav1.CreateOptions{}) } -func CreatePvc(client TestClient, pvcBulder *PVCBuilder) error { - _, err := client.ClientGo.CoreV1().PersistentVolumeClaims(pvcBulder.Namespace).Create(context.TODO(), pvcBulder.Result(), metav1.CreateOptions{}) +func CreatePvc(client TestClient, pvcBuilder *PVCBuilder) error { + _, err := client.ClientGo.CoreV1().PersistentVolumeClaims(pvcBuilder.Namespace).Create(context.TODO(), pvcBuilder.Result(), metav1.CreateOptions{}) return err }